History

Merge branch 'xarray' of git://git.infradead.org/users/willy/linux-dax

Pull XArray conversion from Matthew Wilcox: "The XArray provides an improved interface to the radix tree data structure, providing locking as part of the API, specifying GFP flags at allocation time, eliminating preloading, less re-walking the tree, more efficient iterations and not exposing RCU-protected pointers to its users. This patch set 1. Introduces the XArray implementation 2. Converts the pagecache to use it 3. Converts memremap to use it The page cache is the most complex and important user of the radix tree, so converting it was most important. Converting the memremap code removes the only other user of the multiorder code, which allows us to remove the radix tree code that supported it. I have 40+ followup patches to convert many other users of the radix tree over to the XArray, but I'd like to get this part in first. The other conversions haven't been in linux-next and aren't suitable for applying yet, but you can see them in the xarray-conv branch if you're interested" * 'xarray' of git://git.infradead.org/users/willy/linux-dax: (90 commits) radix tree: Remove multiorder support radix tree test: Convert multiorder tests to XArray radix tree tests: Convert item_delete_rcu to XArray radix tree tests: Convert item_kill_tree to XArray radix tree tests: Move item_insert_order radix tree test suite: Remove multiorder benchmarking radix tree test suite: Remove __item_insert memremap: Convert to XArray xarray: Add range store functionality xarray: Move multiorder_check to in-kernel tests xarray: Move multiorder_shrink to kernel tests xarray: Move multiorder account test in-kernel radix tree test suite: Convert iteration test to XArray radix tree test suite: Convert tag_tagged_items to XArray radix tree: Remove radix_tree_clear_tags radix tree: Remove radix_tree_maybe_preload_order radix tree: Remove split/join code radix tree: Remove radix_tree_update_node_t page cache: Finish XArray conversion dax: Convert page fault handlers to XArray ...

Linus Torvalds committed 8 years ago

Commit dad4f142 parents 69d5b97 3a08cd593 files · +7052 −3821Browse files

Unified · Split

93 files
  1. .clang-format +0 −1
  2. .mailmap +7 −0
  3. Documentation/core-api/index.rst +1 −0
  4. Documentation/core-api/xarray.rst +435 −0
  5. MAINTAINERS +14 −3
  6. arch/parisc/kernel/syscall.S +1 −1
  7. arch/powerpc/include/asm/book3s/64/pgtable.h +1 −3
  8. arch/powerpc/include/asm/nohash/64/pgtable.h +1 −3
  9. drivers/gpu/drm/i915/i915_gem.c +7 −10
  10. drivers/input/keyboard/hilkbd.c +1 −1
  11. drivers/pci/hotplug/acpiphp.h +1 −1
  12. drivers/pci/hotplug/acpiphp_core.c +2 −2
  13. drivers/pci/hotplug/acpiphp_glue.c +1 −1
  14. drivers/staging/erofs/utils.c +6 −12
  15. fs/btrfs/compression.c +2 −4
  16. fs/btrfs/extent_io.c +5 −7
  17. fs/buffer.c +7 −7
  18. fs/dax.c +388 −529
  19. fs/ext4/inode.c +1 −1
  20. fs/f2fs/data.c +3 −3
  21. fs/f2fs/dir.c +1 −1
  22. fs/f2fs/f2fs.h +1 −1
  23. fs/f2fs/inline.c +1 −1
  24. fs/f2fs/node.c +2 −4
  25. fs/fs-writeback.c +9 −16
  26. fs/gfs2/aops.c +1 −1
  27. fs/inode.c +1 −1
  28. fs/isofs/dir.c +1 −1
  29. fs/nfs/blocklayout/blocklayout.c +1 −1
  30. fs/nilfs2/btnode.c +9 −17
  31. fs/nilfs2/page.c +13 −16
  32. fs/proc/task_mmu.c +1 −1
  33. include/linux/fs.h +41 −22
  34. include/linux/idr.h +7 −11
  35. include/linux/pagemap.h +5 −5
  36. include/linux/pagevec.h +5 −3
  37. include/linux/radix-tree.h +24 −154
  38. include/linux/swap.h +9 −13
  39. include/linux/swapops.h +7 −12
  40. include/linux/xarray.h +1292 −1
  41. kernel/memremap.c +15 −60
  42. lib/Kconfig +4 −1
  43. lib/Kconfig.debug +3 −0
  44. lib/Makefile +2 −1
  45. lib/idr.c +207 −194
  46. lib/radix-tree.c +88 −746
  47. lib/test_xarray.c +1238 −0
  48. lib/xarray.c +2036 −0
  49. mm/Kconfig +2 −2
  50. mm/filemap.c +310 −414
  51. mm/huge_memory.c +7 −10
  52. mm/khugepaged.c +72 −106
  53. mm/madvise.c +1 −1
  54. mm/memcontrol.c +1 −1
  55. mm/memfd.c +43 −62
  56. mm/migrate.c +18 −30
  57. mm/mincore.c +1 −1
  58. mm/page-writeback.c +26 −46
  59. mm/readahead.c +4 −6
  60. mm/shmem.c +73 −120
  61. mm/swap.c +3 −3
  62. mm/swap_state.c +40 −79
  63. mm/truncate.c +12 −15
  64. mm/vmscan.c +5 −5
  65. mm/workingset.c +29 −39
  66. tools/include/asm-generic/bitops.h +1 −0
  67. tools/include/asm-generic/bitops/atomic.h +0 −9
  68. tools/include/asm-generic/bitops/non-atomic.h +109 −0
  69. tools/include/linux/bitmap.h +1 −0
  70. tools/include/linux/kernel.h +1 −0
  71. tools/include/linux/spinlock.h +10 −2
  72. tools/testing/radix-tree/.gitignore +1 −0
  73. tools/testing/radix-tree/Makefile +8 −3
  74. tools/testing/radix-tree/benchmark.c +21 −120
  75. tools/testing/radix-tree/bitmap.c +23 −0
  76. tools/testing/radix-tree/generated/autoconf.h +1 −1
  77. tools/testing/radix-tree/idr-test.c +66 −5
  78. tools/testing/radix-tree/iteration_check.c +53 −56
  79. tools/testing/radix-tree/linux/bug.h +1 −0
  80. tools/testing/radix-tree/linux/kconfig.h +1 −0
  81. tools/testing/radix-tree/linux/kernel.h +5 −0
  82. tools/testing/radix-tree/linux/lockdep.h +11 −0
  83. tools/testing/radix-tree/linux/radix-tree.h +0 −1
  84. tools/testing/radix-tree/linux/rcupdate.h +2 −0
  85. tools/testing/radix-tree/main.c +3 −63
  86. tools/testing/radix-tree/multiorder.c +64 −545
  87. tools/testing/radix-tree/regression1.c +27 −48
  88. tools/testing/radix-tree/regression2.c +4 −4
  89. tools/testing/radix-tree/regression3.c +0 −23
  90. tools/testing/radix-tree/tag_check.c +2 −31
  91. tools/testing/radix-tree/test.c +42 −89
  92. tools/testing/radix-tree/test.h +5 −8
  93. tools/testing/radix-tree/xarray.c +35 −0
.clang-format+0−1
@@ -323,7 +323,6 @@ ForEachMacros: · 322 unmodified lines
323323 - 'protocol_for_each_card'
324324 - 'protocol_for_each_dev'
325325 - 'queue_for_each_hw_ctx'
326− - 'radix_tree_for_each_contig'
327326 - 'radix_tree_for_each_slot'
328327 - 'radix_tree_for_each_tagged'
329328 - 'rbtree_postorder_for_each_entry_safe'
.mailmap+7−0
@@ -119,6 +119,13 @@ Mark Brown <broonie@sirena.org.uk> · 118 unmodified lines
119119 Mark Yao <markyao0591@gmail.com> <mark.yao@rock-chips.com>
120120 Martin Kepplinger <martink@posteo.de> <martin.kepplinger@theobroma-systems.com>
121121 Martin Kepplinger <martink@posteo.de> <martin.kepplinger@ginzinger.com>
122+Matthew Wilcox <willy@infradead.org> <matthew.r.wilcox@intel.com>
123+Matthew Wilcox <willy@infradead.org> <matthew@wil.cx>
124+Matthew Wilcox <willy@infradead.org> <mawilcox@linuxonhyperv.com>
125+Matthew Wilcox <willy@infradead.org> <mawilcox@microsoft.com>
126+Matthew Wilcox <willy@infradead.org> <willy@debian.org>
127+Matthew Wilcox <willy@infradead.org> <willy@linux.intel.com>
128+Matthew Wilcox <willy@infradead.org> <willy@parisc-linux.org>
122129 Matthieu CASTET <castet.matthieu@free.fr>
123130 Mauro Carvalho Chehab <mchehab@kernel.org> <mchehab@brturbo.com.br>
124131 Mauro Carvalho Chehab <mchehab@kernel.org> <maurochehab@gmail.com>
Documentation/core-api/index.rst+1−0
@@ -21,6 +21,7 @@ Core utilities · 20 unmodified lines
2121 local_ops
2222 workqueue
2323 genericirq
24+ xarray
2425 flexible-arrays
2526 librs
2627 genalloc
Documentation/core-api/xarray.rst+435−0
@@ -0,0 +1,435 @@
1+.. SPDX-License-Identifier: GPL-2.0+
2+
3+======
4+XArray
5+======
6+
7+:Author: Matthew Wilcox
8+
9+Overview
10+========
11+
12+The XArray is an abstract data type which behaves like a very large array
13+of pointers. It meets many of the same needs as a hash or a conventional
14+resizable array. Unlike a hash, it allows you to sensibly go to the
15+next or previous entry in a cache-efficient manner. In contrast to a
16+resizable array, there is no need to copy data or change MMU mappings in
17+order to grow the array. It is more memory-efficient, parallelisable
18+and cache friendly than a doubly-linked list. It takes advantage of
19+RCU to perform lookups without locking.
20+
21+The XArray implementation is efficient when the indices used are densely
22+clustered; hashing the object and using the hash as the index will not
23+perform well. The XArray is optimised for small indices, but still has
24+good performance with large indices. If your index can be larger than
25+``ULONG_MAX`` then the XArray is not the data type for you. The most
26+important user of the XArray is the page cache.
27+
28+Each non-``NULL`` entry in the array has three bits associated with
29+it called marks. Each mark may be set or cleared independently of
30+the others. You can iterate over entries which are marked.
31+
32+Normal pointers may be stored in the XArray directly. They must be 4-byte
33+aligned, which is true for any pointer returned from :c:func:`kmalloc` and
34+:c:func:`alloc_page`. It isn't true for arbitrary user-space pointers,
35+nor for function pointers. You can store pointers to statically allocated
36+objects, as long as those objects have an alignment of at least 4.
37+
38+You can also store integers between 0 and ``LONG_MAX`` in the XArray.
39+You must first convert it into an entry using :c:func:`xa_mk_value`.
40+When you retrieve an entry from the XArray, you can check whether it is
41+a value entry by calling :c:func:`xa_is_value`, and convert it back to
42+an integer by calling :c:func:`xa_to_value`.
43+
44+Some users want to store tagged pointers instead of using the marks
45+described above. They can call :c:func:`xa_tag_pointer` to create an
46+entry with a tag, :c:func:`xa_untag_pointer` to turn a tagged entry
47+back into an untagged pointer and :c:func:`xa_pointer_tag` to retrieve
48+the tag of an entry. Tagged pointers use the same bits that are used
49+to distinguish value entries from normal pointers, so each user must
50+decide whether they want to store value entries or tagged pointers in
51+any particular XArray.
52+
53+The XArray does not support storing :c:func:`IS_ERR` pointers as some
54+conflict with value entries or internal entries.
55+
56+An unusual feature of the XArray is the ability to create entries which
57+occupy a range of indices. Once stored to, looking up any index in
58+the range will return the same entry as looking up any other index in
59+the range. Setting a mark on one index will set it on all of them.
60+Storing to any index will store to all of them. Multi-index entries can
61+be explicitly split into smaller entries, or storing ``NULL`` into any
62+entry will cause the XArray to forget about the range.
63+
64+Normal API
65+==========
66+
67+Start by initialising an XArray, either with :c:func:`DEFINE_XARRAY`
68+for statically allocated XArrays or :c:func:`xa_init` for dynamically
69+allocated ones. A freshly-initialised XArray contains a ``NULL``
70+pointer at every index.
71+
72+You can then set entries using :c:func:`xa_store` and get entries
73+using :c:func:`xa_load`. xa_store will overwrite any entry with the
74+new entry and return the previous entry stored at that index. You can
75+use :c:func:`xa_erase` instead of calling :c:func:`xa_store` with a
76+``NULL`` entry. There is no difference between an entry that has never
77+been stored to and one that has most recently had ``NULL`` stored to it.
78+
79+You can conditionally replace an entry at an index by using
80+:c:func:`xa_cmpxchg`. Like :c:func:`cmpxchg`, it will only succeed if
81+the entry at that index has the 'old' value. It also returns the entry
82+which was at that index; if it returns the same entry which was passed as
83+'old', then :c:func:`xa_cmpxchg` succeeded.
84+
85+If you want to only store a new entry to an index if the current entry
86+at that index is ``NULL``, you can use :c:func:`xa_insert` which
87+returns ``-EEXIST`` if the entry is not empty.
88+
89+You can enquire whether a mark is set on an entry by using
90+:c:func:`xa_get_mark`. If the entry is not ``NULL``, you can set a mark
91+on it by using :c:func:`xa_set_mark` and remove the mark from an entry by
92+calling :c:func:`xa_clear_mark`. You can ask whether any entry in the
93+XArray has a particular mark set by calling :c:func:`xa_marked`.
94+
95+You can copy entries out of the XArray into a plain array by calling
96+:c:func:`xa_extract`. Or you can iterate over the present entries in
97+the XArray by calling :c:func:`xa_for_each`. You may prefer to use
98+:c:func:`xa_find` or :c:func:`xa_find_after` to move to the next present
99+entry in the XArray.
100+
101+Calling :c:func:`xa_store_range` stores the same entry in a range
102+of indices. If you do this, some of the other operations will behave
103+in a slightly odd way. For example, marking the entry at one index
104+may result in the entry being marked at some, but not all of the other
105+indices. Storing into one index may result in the entry retrieved by
106+some, but not all of the other indices changing.
107+
108+Finally, you can remove all entries from an XArray by calling
109+:c:func:`xa_destroy`. If the XArray entries are pointers, you may wish
110+to free the entries first. You can do this by iterating over all present
111+entries in the XArray using the :c:func:`xa_for_each` iterator.
112+
113+ID assignment
114+-------------
115+
116+You can call :c:func:`xa_alloc` to store the entry at any unused index
117+in the XArray. If you need to modify the array from interrupt context,
118+you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable
119+interrupts while allocating the ID. Unlike :c:func:`xa_store`, allocating
120+a ``NULL`` pointer does not delete an entry. Instead it reserves an
121+entry like :c:func:`xa_reserve` and you can release it using either
122+:c:func:`xa_erase` or :c:func:`xa_release`. To use ID assignment, the
123+XArray must be defined with :c:func:`DEFINE_XARRAY_ALLOC`, or initialised
124+by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`,
125+
126+Memory allocation
127+-----------------
128+
129+The :c:func:`xa_store`, :c:func:`xa_cmpxchg`, :c:func:`xa_alloc`,
130+:c:func:`xa_reserve` and :c:func:`xa_insert` functions take a gfp_t
131+parameter in case the XArray needs to allocate memory to store this entry.
132+If the entry is being deleted, no memory allocation needs to be performed,
133+and the GFP flags specified will be ignored.
134+
135+It is possible for no memory to be allocatable, particularly if you pass
136+a restrictive set of GFP flags. In that case, the functions return a
137+special value which can be turned into an errno using :c:func:`xa_err`.
138+If you don't need to know exactly which error occurred, using
139+:c:func:`xa_is_err` is slightly more efficient.
140+
141+Locking
142+-------
143+
144+When using the Normal API, you do not have to worry about locking.
145+The XArray uses RCU and an internal spinlock to synchronise access:
146+
147+No lock needed:
148+ * :c:func:`xa_empty`
149+ * :c:func:`xa_marked`
150+
151+Takes RCU read lock:
152+ * :c:func:`xa_load`
153+ * :c:func:`xa_for_each`
154+ * :c:func:`xa_find`
155+ * :c:func:`xa_find_after`
156+ * :c:func:`xa_extract`
157+ * :c:func:`xa_get_mark`
158+
159+Takes xa_lock internally:
160+ * :c:func:`xa_store`
161+ * :c:func:`xa_insert`
162+ * :c:func:`xa_erase`
163+ * :c:func:`xa_erase_bh`
164+ * :c:func:`xa_erase_irq`
165+ * :c:func:`xa_cmpxchg`
166+ * :c:func:`xa_store_range`
167+ * :c:func:`xa_alloc`
168+ * :c:func:`xa_alloc_bh`
169+ * :c:func:`xa_alloc_irq`
170+ * :c:func:`xa_destroy`
171+ * :c:func:`xa_set_mark`
172+ * :c:func:`xa_clear_mark`
173+
174+Assumes xa_lock held on entry:
175+ * :c:func:`__xa_store`
176+ * :c:func:`__xa_insert`
177+ * :c:func:`__xa_erase`
178+ * :c:func:`__xa_cmpxchg`
179+ * :c:func:`__xa_alloc`
180+ * :c:func:`__xa_set_mark`
181+ * :c:func:`__xa_clear_mark`
182+
183+If you want to take advantage of the lock to protect the data structures
184+that you are storing in the XArray, you can call :c:func:`xa_lock`
185+before calling :c:func:`xa_load`, then take a reference count on the
186+object you have found before calling :c:func:`xa_unlock`. This will
187+prevent stores from removing the object from the array between looking
188+up the object and incrementing the refcount. You can also use RCU to
189+avoid dereferencing freed memory, but an explanation of that is beyond
190+the scope of this document.
191+
192+The XArray does not disable interrupts or softirqs while modifying
193+the array. It is safe to read the XArray from interrupt or softirq
194+context as the RCU lock provides enough protection.
195+
196+If, for example, you want to store entries in the XArray in process
197+context and then erase them in softirq context, you can do that this way::
198+
199+ void foo_init(struct foo *foo)
200+ {
201+ xa_init_flags(&foo->array, XA_FLAGS_LOCK_BH);
202+ }
203+
204+ int foo_store(struct foo *foo, unsigned long index, void *entry)
205+ {
206+ int err;
207+
208+ xa_lock_bh(&foo->array);
209+ err = xa_err(__xa_store(&foo->array, index, entry, GFP_KERNEL));
210+ if (!err)
211+ foo->count++;
212+ xa_unlock_bh(&foo->array);
213+ return err;
214+ }
215+
216+ /* foo_erase() is only called from softirq context */
217+ void foo_erase(struct foo *foo, unsigned long index)
218+ {
219+ xa_lock(&foo->array);
220+ __xa_erase(&foo->array, index);
221+ foo->count--;
222+ xa_unlock(&foo->array);
223+ }
224+
225+If you are going to modify the XArray from interrupt or softirq context,
226+you need to initialise the array using :c:func:`xa_init_flags`, passing
227+``XA_FLAGS_LOCK_IRQ`` or ``XA_FLAGS_LOCK_BH``.
228+
229+The above example also shows a common pattern of wanting to extend the
230+coverage of the xa_lock on the store side to protect some statistics
231+associated with the array.
232+
233+Sharing the XArray with interrupt context is also possible, either
234+using :c:func:`xa_lock_irqsave` in both the interrupt handler and process
235+context, or :c:func:`xa_lock_irq` in process context and :c:func:`xa_lock`
236+in the interrupt handler. Some of the more common patterns have helper
237+functions such as :c:func:`xa_erase_bh` and :c:func:`xa_erase_irq`.
238+
239+Sometimes you need to protect access to the XArray with a mutex because
240+that lock sits above another mutex in the locking hierarchy. That does
241+not entitle you to use functions like :c:func:`__xa_erase` without taking
242+the xa_lock; the xa_lock is used for lockdep validation and will be used
243+for other purposes in the future.
244+
245+The :c:func:`__xa_set_mark` and :c:func:`__xa_clear_mark` functions are also
246+available for situations where you look up an entry and want to atomically
247+set or clear a mark. It may be more efficient to use the advanced API
248+in this case, as it will save you from walking the tree twice.
249+
250+Advanced API
251+============
252+
253+The advanced API offers more flexibility and better performance at the
254+cost of an interface which can be harder to use and has fewer safeguards.
255+No locking is done for you by the advanced API, and you are required
256+to use the xa_lock while modifying the array. You can choose whether
257+to use the xa_lock or the RCU lock while doing read-only operations on
258+the array. You can mix advanced and normal operations on the same array;
259+indeed the normal API is implemented in terms of the advanced API. The
260+advanced API is only available to modules with a GPL-compatible license.
261+
262+The advanced API is based around the xa_state. This is an opaque data
263+structure which you declare on the stack using the :c:func:`XA_STATE`
264+macro. This macro initialises the xa_state ready to start walking
265+around the XArray. It is used as a cursor to maintain the position
266+in the XArray and let you compose various operations together without
267+having to restart from the top every time.
268+
269+The xa_state is also used to store errors. You can call
270+:c:func:`xas_error` to retrieve the error. All operations check whether
271+the xa_state is in an error state before proceeding, so there's no need
272+for you to check for an error after each call; you can make multiple
273+calls in succession and only check at a convenient point. The only
274+errors currently generated by the XArray code itself are ``ENOMEM`` and
275+``EINVAL``, but it supports arbitrary errors in case you want to call
276+:c:func:`xas_set_err` yourself.
277+
278+If the xa_state is holding an ``ENOMEM`` error, calling :c:func:`xas_nomem`
279+will attempt to allocate more memory using the specified gfp flags and
280+cache it in the xa_state for the next attempt. The idea is that you take
281+the xa_lock, attempt the operation and drop the lock. The operation
282+attempts to allocate memory while holding the lock, but it is more
283+likely to fail. Once you have dropped the lock, :c:func:`xas_nomem`
284+can try harder to allocate more memory. It will return ``true`` if it
285+is worth retrying the operation (i.e. that there was a memory error *and*
286+more memory was allocated). If it has previously allocated memory, and
287+that memory wasn't used, and there is no error (or some error that isn't
288+``ENOMEM``), then it will free the memory previously allocated.
289+
290+Internal Entries
291+----------------
292+
293+The XArray reserves some entries for its own purposes. These are never
294+exposed through the normal API, but when using the advanced API, it's
295+possible to see them. Usually the best way to handle them is to pass them
296+to :c:func:`xas_retry`, and retry the operation if it returns ``true``.
297+
298+.. flat-table::
299+ :widths: 1 1 6
300+
301+ * - Name
302+ - Test
303+ - Usage
304+
305+ * - Node
306+ - :c:func:`xa_is_node`
307+ - An XArray node. May be visible when using a multi-index xa_state.
308+
309+ * - Sibling
310+ - :c:func:`xa_is_sibling`
311+ - A non-canonical entry for a multi-index entry. The value indicates
312+ which slot in this node has the canonical entry.
313+
314+ * - Retry
315+ - :c:func:`xa_is_retry`
316+ - This entry is currently being modified by a thread which has the
317+ xa_lock. The node containing this entry may be freed at the end
318+ of this RCU period. You should restart the lookup from the head
319+ of the array.
320+
321+ * - Zero
322+ - :c:func:`xa_is_zero`
323+ - Zero entries appear as ``NULL`` through the Normal API, but occupy
324+ an entry in the XArray which can be used to reserve the index for
325+ future use.
326+
327+Other internal entries may be added in the future. As far as possible, they
328+will be handled by :c:func:`xas_retry`.
329+
330+Additional functionality
331+------------------------
332+
333+The :c:func:`xas_create_range` function allocates all the necessary memory
334+to store every entry in a range. It will set ENOMEM in the xa_state if
335+it cannot allocate memory.
336+
337+You can use :c:func:`xas_init_marks` to reset the marks on an entry
338+to their default state. This is usually all marks clear, unless the
339+XArray is marked with ``XA_FLAGS_TRACK_FREE``, in which case mark 0 is set
340+and all other marks are clear. Replacing one entry with another using
341+:c:func:`xas_store` will not reset the marks on that entry; if you want
342+the marks reset, you should do that explicitly.
343+
344+The :c:func:`xas_load` will walk the xa_state as close to the entry
345+as it can. If you know the xa_state has already been walked to the
346+entry and need to check that the entry hasn't changed, you can use
347+:c:func:`xas_reload` to save a function call.
348+
349+If you need to move to a different index in the XArray, call
350+:c:func:`xas_set`. This resets the cursor to the top of the tree, which
351+will generally make the next operation walk the cursor to the desired
352+spot in the tree. If you want to move to the next or previous index,
353+call :c:func:`xas_next` or :c:func:`xas_prev`. Setting the index does
354+not walk the cursor around the array so does not require a lock to be
355+held, while moving to the next or previous index does.
356+
357+You can search for the next present entry using :c:func:`xas_find`. This
358+is the equivalent of both :c:func:`xa_find` and :c:func:`xa_find_after`;
359+if the cursor has been walked to an entry, then it will find the next
360+entry after the one currently referenced. If not, it will return the
361+entry at the index of the xa_state. Using :c:func:`xas_next_entry` to
362+move to the next present entry instead of :c:func:`xas_find` will save
363+a function call in the majority of cases at the expense of emitting more
364+inline code.
365+
366+The :c:func:`xas_find_marked` function is similar. If the xa_state has
367+not been walked, it will return the entry at the index of the xa_state,
368+if it is marked. Otherwise, it will return the first marked entry after
369+the entry referenced by the xa_state. The :c:func:`xas_next_marked`
370+function is the equivalent of :c:func:`xas_next_entry`.
371+
372+When iterating over a range of the XArray using :c:func:`xas_for_each`
373+or :c:func:`xas_for_each_marked`, it may be necessary to temporarily stop
374+the iteration. The :c:func:`xas_pause` function exists for this purpose.
375+After you have done the necessary work and wish to resume, the xa_state
376+is in an appropriate state to continue the iteration after the entry
377+you last processed. If you have interrupts disabled while iterating,
378+then it is good manners to pause the iteration and reenable interrupts
379+every ``XA_CHECK_SCHED`` entries.
380+
381+The :c:func:`xas_get_mark`, :c:func:`xas_set_mark` and
382+:c:func:`xas_clear_mark` functions require the xa_state cursor to have
383+been moved to the appropriate location in the xarray; they will do
384+nothing if you have called :c:func:`xas_pause` or :c:func:`xas_set`
385+immediately before.
386+
387+You can call :c:func:`xas_set_update` to have a callback function
388+called each time the XArray updates a node. This is used by the page
389+cache workingset code to maintain its list of nodes which contain only
390+shadow entries.
391+
392+Multi-Index Entries
393+-------------------
394+
395+The XArray has the ability to tie multiple indices together so that
396+operations on one index affect all indices. For example, storing into
397+any index will change the value of the entry retrieved from any index.
398+Setting or clearing a mark on any index will set or clear the mark
399+on every index that is tied together. The current implementation
400+only allows tying ranges which are aligned powers of two together;
401+eg indices 64-127 may be tied together, but 2-6 may not be. This may
402+save substantial quantities of memory; for example tying 512 entries
403+together will save over 4kB.
404+
405+You can create a multi-index entry by using :c:func:`XA_STATE_ORDER`
406+or :c:func:`xas_set_order` followed by a call to :c:func:`xas_store`.
407+Calling :c:func:`xas_load` with a multi-index xa_state will walk the
408+xa_state to the right location in the tree, but the return value is not
409+meaningful, potentially being an internal entry or ``NULL`` even when there
410+is an entry stored within the range. Calling :c:func:`xas_find_conflict`
411+will return the first entry within the range or ``NULL`` if there are no
412+entries in the range. The :c:func:`xas_for_each_conflict` iterator will
413+iterate over every entry which overlaps the specified range.
414+
415+If :c:func:`xas_load` encounters a multi-index entry, the xa_index
416+in the xa_state will not be changed. When iterating over an XArray
417+or calling :c:func:`xas_find`, if the initial index is in the middle
418+of a multi-index entry, it will not be altered. Subsequent calls
419+or iterations will move the index to the first index in the range.
420+Each entry will only be returned once, no matter how many indices it
421+occupies.
422+
423+Using :c:func:`xas_next` or :c:func:`xas_prev` with a multi-index xa_state
424+is not supported. Using either of these functions on a multi-index entry
425+will reveal sibling entries; these should be skipped over by the caller.
426+
427+Storing ``NULL`` into any index of a multi-index entry will set the entry
428+at every index to ``NULL`` and dissolve the tie. Splitting a multi-index
429+entry into entries occupying smaller ranges is not yet supported.
430+
431+Functions and structures
432+========================
433+
434+.. kernel-doc:: include/linux/xarray.h
435+.. kernel-doc:: lib/xarray.c
MAINTAINERS+14−3
@@ -535,7 +535,7 @@ F: Documentation/hwmon/adt7475 · 534 unmodified lines
535535 F: drivers/hwmon/adt7475.c
536536
537537 ADVANSYS SCSI DRIVER
538−M: Matthew Wilcox <matthew@wil.cx>
538+M: Matthew Wilcox <willy@infradead.org>
539539 M: Hannes Reinecke <hare@suse.com>
540540 L: linux-scsi@vger.kernel.org
541541 S: Maintained
@@ -4393,7 +4393,7 @@ S: Maintained · 3851 unmodified lines
43934393 F: drivers/i2c/busses/i2c-diolan-u2c.c
43944394
43954395 FILESYSTEM DIRECT ACCESS (DAX)
4396−M: Matthew Wilcox <mawilcox@microsoft.com>
4396+M: Matthew Wilcox <willy@infradead.org>
43974397 M: Ross Zwisler <zwisler@kernel.org>
43984398 M: Jan Kara <jack@suse.cz>
43994399 L: linux-fsdevel@vger.kernel.org
@@ -8697,7 +8697,7 @@ F: drivers/message/fusion/ · 4297 unmodified lines
86978697 F: drivers/scsi/mpt3sas/
86988698
86998699 LSILOGIC/SYMBIOS/NCR 53C8XX and 53C1010 PCI-SCSI drivers
8700−M: Matthew Wilcox <matthew@wil.cx>
8700+M: Matthew Wilcox <willy@infradead.org>
87018701 L: linux-scsi@vger.kernel.org
87028702 S: Maintained
87038703 F: drivers/scsi/sym53c8xx_2/
@@ -16137,6 +16137,17 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86/vdso · 7433 unmodified lines
1613716137 S: Maintained
1613816138 F: arch/x86/entry/vdso/
1613916139
16140+XARRAY
16141+M: Matthew Wilcox <willy@infradead.org>
16142+L: linux-fsdevel@vger.kernel.org
16143+S: Supported
16144+F: Documentation/core-api/xarray.rst
16145+F: lib/idr.c
16146+F: lib/xarray.c
16147+F: include/linux/idr.h
16148+F: include/linux/xarray.h
16149+F: tools/testing/radix-tree
16150+
1614016151 XC2028/3028 TUNER DRIVER
1614116152 M: Mauro Carvalho Chehab <mchehab@kernel.org>
1614216153 L: linux-media@vger.kernel.org
arch/parisc/kernel/syscall.S+1−1
@@ -2,7 +2,7 @@ · 1 unmodified line
22 * Linux/PA-RISC Project (http://www.parisc-linux.org/)
33 *
44 * System call entry code / Linux gateway page
5− * Copyright (c) Matthew Wilcox 1999 <willy@bofh.ai>
5+ * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org>
66 * Licensed under the GNU GPL.
77 * thanks to Philipp Rumpf, Mike Shaver and various others
88 * sorry about the wall, puffin..
arch/powerpc/include/asm/book3s/64/pgtable.h+1−3
@@ -716,9 +716,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) · 715 unmodified lines
716716 BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \
717717 BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \
718718 } while (0)
719−/*
720− * on pte we don't need handle RADIX_TREE_EXCEPTIONAL_SHIFT;
721− */
719+
722720 #define SWP_TYPE_BITS 5
723721 #define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \
724722 & ((1UL << SWP_TYPE_BITS) - 1))
arch/powerpc/include/asm/nohash/64/pgtable.h+1−3
@@ -350,9 +350,7 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, · 349 unmodified lines
350350 #define MAX_SWAPFILES_CHECK() do { \
351351 BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \
352352 } while (0)
353−/*
354− * on pte we don't need handle RADIX_TREE_EXCEPTIONAL_SHIFT;
355− */
353+
356354 #define SWP_TYPE_BITS 5
357355 #define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \
358356 & ((1UL << SWP_TYPE_BITS) - 1))
drivers/gpu/drm/i915/i915_gem.c+7−10
@@ -5996,7 +5996,8 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj, · 5995 unmodified lines
59965996 count = __sg_page_count(sg);
59975997
59985998 while (idx + count <= n) {
5999− unsigned long exception, i;
5999+ void *entry;
6000+ unsigned long i;
60006001 int ret;
60016002
60026003 /* If we cannot allocate and insert this entry, or the
@@ -6011,12 +6012,9 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj, · 8 unmodified lines
60116012 if (ret && ret != -EEXIST)
60126013 goto scan;
60136014
6014− exception =
6015− RADIX_TREE_EXCEPTIONAL_ENTRY |
6016− idx << RADIX_TREE_EXCEPTIONAL_SHIFT;
6015+ entry = xa_mk_value(idx);
60176016 for (i = 1; i < count; i++) {
6018− ret = radix_tree_insert(&iter->radix, idx + i,
6019− (void *)exception);
6017+ ret = radix_tree_insert(&iter->radix, idx + i, entry);
60206018 if (ret && ret != -EEXIST)
60216019 goto scan;
60226020 }
@@ -6054,15 +6052,14 @@ lookup: · 31 unmodified lines
60546052 GEM_BUG_ON(!sg);
60556053
60566054 /* If this index is in the middle of multi-page sg entry,
6057− * the radixtree will contain an exceptional entry that points
6055+ * the radix tree will contain a value entry that points
60586056 * to the start of that range. We will return the pointer to
60596057 * the base page and the offset of this page within the
60606058 * sg entry's range.
60616059 */
60626060 *offset = 0;
6063− if (unlikely(radix_tree_exception(sg))) {
6064− unsigned long base =
6065− (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
6061+ if (unlikely(xa_is_value(sg))) {
6062+ unsigned long base = xa_to_value(sg);
60666063
60676064 sg = radix_tree_lookup(&iter->radix, base);
60686065 GEM_BUG_ON(!sg);
drivers/input/keyboard/hilkbd.c+1−1
@@ -2,7 +2,7 @@ · 1 unmodified line
22 * linux/drivers/hil/hilkbd.c
33 *
44 * Copyright (C) 1998 Philip Blundell <philb@gnu.org>
5− * Copyright (C) 1999 Matthew Wilcox <willy@bofh.ai>
5+ * Copyright (C) 1999 Matthew Wilcox <willy@infradead.org>
66 * Copyright (C) 1999-2007 Helge Deller <deller@gmx.de>
77 *
88 * Very basic HP Human Interface Loop (HIL) driver.
drivers/pci/hotplug/acpiphp.h+1−1
@@ -8,7 +8,7 @@ · 7 unmodified lines
88 * Copyright (C) 2002 Hiroshi Aono (h-aono@ap.jp.nec.com)
99 * Copyright (C) 2002,2003 Takayoshi Kochi (t-kochi@bq.jp.nec.com)
1010 * Copyright (C) 2002,2003 NEC Corporation
11− * Copyright (C) 2003-2005 Matthew Wilcox (matthew.wilcox@hp.com)
11+ * Copyright (C) 2003-2005 Matthew Wilcox (willy@infradead.org)
1212 * Copyright (C) 2003-2005 Hewlett Packard
1313 *
1414 * All rights reserved.
drivers/pci/hotplug/acpiphp_core.c+2−2
@@ -8,7 +8,7 @@ · 7 unmodified lines
88 * Copyright (C) 2002 Hiroshi Aono (h-aono@ap.jp.nec.com)
99 * Copyright (C) 2002,2003 Takayoshi Kochi (t-kochi@bq.jp.nec.com)
1010 * Copyright (C) 2002,2003 NEC Corporation
11− * Copyright (C) 2003-2005 Matthew Wilcox (matthew.wilcox@hp.com)
11+ * Copyright (C) 2003-2005 Matthew Wilcox (willy@infradead.org)
1212 * Copyright (C) 2003-2005 Hewlett Packard
1313 *
1414 * All rights reserved.
@@ -40,7 +40,7 @@ bool acpiphp_disabled; · 25 unmodified lines
4040 static struct acpiphp_attention_info *attention_info;
4141
4242 #define DRIVER_VERSION "0.5"
43−#define DRIVER_AUTHOR "Greg Kroah-Hartman <gregkh@us.ibm.com>, Takayoshi Kochi <t-kochi@bq.jp.nec.com>, Matthew Wilcox <willy@hp.com>"
43+#define DRIVER_AUTHOR "Greg Kroah-Hartman <gregkh@us.ibm.com>, Takayoshi Kochi <t-kochi@bq.jp.nec.com>, Matthew Wilcox <willy@infradead.org>"
4444 #define DRIVER_DESC "ACPI Hot Plug PCI Controller Driver"
4545
4646 MODULE_AUTHOR(DRIVER_AUTHOR);
drivers/pci/hotplug/acpiphp_glue.c+1−1
@@ -5,7 +5,7 @@ · 4 unmodified lines
55 * Copyright (C) 2002,2003 Takayoshi Kochi (t-kochi@bq.jp.nec.com)
66 * Copyright (C) 2002 Hiroshi Aono (h-aono@ap.jp.nec.com)
77 * Copyright (C) 2002,2003 NEC Corporation
8− * Copyright (C) 2003-2005 Matthew Wilcox (matthew.wilcox@hp.com)
8+ * Copyright (C) 2003-2005 Matthew Wilcox (willy@infradead.org)
99 * Copyright (C) 2003-2005 Hewlett Packard
1010 * Copyright (C) 2005 Rajesh Shah (rajesh.shah@intel.com)
1111 * Copyright (C) 2005 Intel Corporation
drivers/staging/erofs/utils.c+6−12
@@ -35,7 +35,6 @@ static atomic_long_t erofs_global_shrink_cnt; · 34 unmodified lines
3535
3636 #ifdef CONFIG_EROFS_FS_ZIP
3737
38−/* radix_tree and the future XArray both don't use tagptr_t yet */
3938 struct erofs_workgroup *erofs_find_workgroup(
4039 struct super_block *sb, pgoff_t index, bool *tag)
4140 {
@@ -47,9 +46,8 @@ repeat: · 5 unmodified lines
4746 rcu_read_lock();
4847 grp = radix_tree_lookup(&sbi->workstn_tree, index);
4948 if (grp != NULL) {
50− *tag = radix_tree_exceptional_entry(grp);
51− grp = (void *)((unsigned long)grp &
52− ~RADIX_TREE_EXCEPTIONAL_ENTRY);
49+ *tag = xa_pointer_tag(grp);
50+ grp = xa_untag_pointer(grp);
5351
5452 if (erofs_workgroup_get(grp, &oldcount)) {
5553 /* prefer to relax rcu read side */
@@ -83,9 +81,7 @@ int erofs_register_workgroup(struct super_block *sb, · 27 unmodified lines
8381 sbi = EROFS_SB(sb);
8482 erofs_workstn_lock(sbi);
8583
86− if (tag)
87− grp = (void *)((unsigned long)grp |
88− 1UL << RADIX_TREE_EXCEPTIONAL_SHIFT);
84+ grp = xa_tag_pointer(grp, tag);
8985
9086 err = radix_tree_insert(&sbi->workstn_tree,
9187 grp->index, grp);
@@ -131,9 +127,7 @@ repeat: · 39 unmodified lines
131127
132128 for (i = 0; i < found; ++i) {
133129 int cnt;
134− struct erofs_workgroup *grp = (void *)
135− ((unsigned long)batch[i] &
136− ~RADIX_TREE_EXCEPTIONAL_ENTRY);
130+ struct erofs_workgroup *grp = xa_untag_pointer(batch[i]);
137131
138132 first_index = grp->index + 1;
139133
@@ -150,8 +144,8 @@ repeat: · 10 unmodified lines
150144 #endif
151145 continue;
152146
153− if (radix_tree_delete(&sbi->workstn_tree,
154− grp->index) != grp) {
147+ if (xa_untag_pointer(radix_tree_delete(&sbi->workstn_tree,
148+ grp->index)) != grp) {
155149 #ifdef EROFS_FS_HAS_MANAGED_CACHE
156150 skip:
157151 erofs_workgroup_unfreeze(grp, 1);
fs/btrfs/compression.c+2−4
@@ -437,10 +437,8 @@ static noinline int add_ra_bio_pages(struct inode *inode, · 436 unmodified lines
437437 if (pg_index > end_index)
438438 break;
439439
440− rcu_read_lock();
441− page = radix_tree_lookup(&mapping->i_pages, pg_index);
442− rcu_read_unlock();
443− if (page && !radix_tree_exceptional_entry(page)) {
440+ page = xa_load(&mapping->i_pages, pg_index);
441+ if (page && !xa_is_value(page)) {
444442 misses++;
445443 if (misses > 4)
446444 break;
fs/btrfs/extent_io.c+5−7
@@ -3784,7 +3784,7 @@ int btree_write_cache_pages(struct address_space *mapping, · 3783 unmodified lines
37843784 pgoff_t index;
37853785 pgoff_t end; /* Inclusive */
37863786 int scanned = 0;
3787− int tag;
3787+ xa_mark_t tag;
37883788
37893789 pagevec_init(&pvec);
37903790 if (wbc->range_cyclic) {
@@ -3909,7 +3909,7 @@ static int extent_write_cache_pages(struct address_space *mapping, · 118 unmodified lines
39093909 pgoff_t done_index;
39103910 int range_whole = 0;
39113911 int scanned = 0;
3912− int tag;
3912+ xa_mark_t tag;
39133913
39143914 /*
39153915 * We have to hold onto the inode so that ordered extents can do their
@@ -5159,11 +5159,9 @@ void clear_extent_buffer_dirty(struct extent_buffer *eb) · 1243 unmodified lines
51595159
51605160 clear_page_dirty_for_io(page);
51615161 xa_lock_irq(&page->mapping->i_pages);
5162− if (!PageDirty(page)) {
5163− radix_tree_tag_clear(&page->mapping->i_pages,
5164− page_index(page),
5165− PAGECACHE_TAG_DIRTY);
5166− }
5162+ if (!PageDirty(page))
5163+ __xa_clear_mark(&page->mapping->i_pages,
5164+ page_index(page), PAGECACHE_TAG_DIRTY);
51675165 xa_unlock_irq(&page->mapping->i_pages);
51685166 ClearPageError(page);
51695167 unlock_page(page);
fs/buffer.c+7−7
@@ -562,7 +562,7 @@ void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) · 561 unmodified lines
562562 EXPORT_SYMBOL(mark_buffer_dirty_inode);
563563
564564 /*
565− * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
565+ * Mark the page dirty, and set it dirty in the page cache, and mark the inode
566566 * dirty.
567567 *
568568 * If warn is true, then emit a warning if the page is not uptodate and has
@@ -579,8 +579,8 @@ void __set_page_dirty(struct page *page, struct address_space *mapping, · 10 unmodified lines
579579 if (page->mapping) { /* Race with truncate? */
580580 WARN_ON_ONCE(warn && !PageUptodate(page));
581581 account_page_dirtied(page, mapping);
582− radix_tree_tag_set(&mapping->i_pages,
583− page_index(page), PAGECACHE_TAG_DIRTY);
582+ __xa_set_mark(&mapping->i_pages, page_index(page),
583+ PAGECACHE_TAG_DIRTY);
584584 }
585585 xa_unlock_irqrestore(&mapping->i_pages, flags);
586586 }
@@ -1050,7 +1050,7 @@ __getblk_slow(struct block_device *bdev, sector_t block, · 463 unmodified lines
10501050 * The relationship between dirty buffers and dirty pages:
10511051 *
10521052 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1053− * the page is tagged dirty in its radix tree.
1053+ * the page is tagged dirty in the page cache.
10541054 *
10551055 * At all times, the dirtiness of the buffers represents the dirtiness of
10561056 * subsections of the page. If the page has buffers, the page dirty bit is
@@ -1073,9 +1073,9 @@ __getblk_slow(struct block_device *bdev, sector_t block, · 16 unmodified lines
10731073 * mark_buffer_dirty - mark a buffer_head as needing writeout
10741074 * @bh: the buffer_head to mark dirty
10751075 *
1076− * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1077− * backing page dirty, then tag the page as dirty in its address_space's radix
1078− * tree and then attach the address_space's inode to its superblock's dirty
1076+ * mark_buffer_dirty() will set the dirty bit against the buffer, then set
1077+ * its backing page dirty, then tag the page as dirty in the page cache
1078+ * and then attach the address_space's inode to its superblock's dirty
10791079 * inode list.
10801080 *
10811081 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
fs/dax.c+388−529
@@ -38,6 +38,17 @@ · 37 unmodified lines
3838 #define CREATE_TRACE_POINTS
3939 #include <trace/events/fs_dax.h>
4040
41+static inline unsigned int pe_order(enum page_entry_size pe_size)
42+{
43+ if (pe_size == PE_SIZE_PTE)
44+ return PAGE_SHIFT - PAGE_SHIFT;
45+ if (pe_size == PE_SIZE_PMD)
46+ return PMD_SHIFT - PAGE_SHIFT;
47+ if (pe_size == PE_SIZE_PUD)
48+ return PUD_SHIFT - PAGE_SHIFT;
49+ return ~0;
50+}
51+
4152 /* We choose 4096 entries - same as per-zone page wait tables */
4253 #define DAX_WAIT_TABLE_BITS 12
4354 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
@@ -46,6 +57,9 @@ · 2 unmodified lines
4657 #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
4758 #define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT)
4859
60+/* The order of a PMD entry */
61+#define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)
62+
4963 static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
5064
5165 static int __init init_dax_wait_table(void)
@@ -59,63 +73,74 @@ static int __init init_dax_wait_table(void) · 7 unmodified lines
5973 fs_initcall(init_dax_wait_table);
6074
6175 /*
62− * We use lowest available bit in exceptional entry for locking, one bit for
63− * the entry size (PMD) and two more to tell us if the entry is a zero page or
64− * an empty entry that is just used for locking. In total four special bits.
76+ * DAX pagecache entries use XArray value entries so they can't be mistaken
77+ * for pages. We use one bit for locking, one bit for the entry size (PMD)
78+ * and two more to tell us if the entry is a zero page or an empty entry that
79+ * is just used for locking. In total four special bits.
6580 *
6681 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
6782 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
6883 * block allocation.
6984 */
70−#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
71−#define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
72−#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
73−#define RADIX_DAX_ZERO_PAGE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
74−#define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))
85+#define DAX_SHIFT (4)
86+#define DAX_LOCKED (1UL << 0)
87+#define DAX_PMD (1UL << 1)
88+#define DAX_ZERO_PAGE (1UL << 2)
89+#define DAX_EMPTY (1UL << 3)
7590
76−static unsigned long dax_radix_pfn(void *entry)
91+static unsigned long dax_to_pfn(void *entry)
7792 {
78− return (unsigned long)entry >> RADIX_DAX_SHIFT;
93+ return xa_to_value(entry) >> DAX_SHIFT;
7994 }
8095
81−static void *dax_radix_locked_entry(unsigned long pfn, unsigned long flags)
96+static void *dax_make_entry(pfn_t pfn, unsigned long flags)
8297 {
83− return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |
84− (pfn << RADIX_DAX_SHIFT) | RADIX_DAX_ENTRY_LOCK);
98+ return xa_mk_value(flags | (pfn_t_to_pfn(pfn) << DAX_SHIFT));
8599 }
86100
87−static unsigned int dax_radix_order(void *entry)
101+static void *dax_make_page_entry(struct page *page)
88102 {
89− if ((unsigned long)entry & RADIX_DAX_PMD)
90− return PMD_SHIFT - PAGE_SHIFT;
103+ pfn_t pfn = page_to_pfn_t(page);
104+ return dax_make_entry(pfn, PageHead(page) ? DAX_PMD : 0);
105+}
106+
107+static bool dax_is_locked(void *entry)
108+{
109+ return xa_to_value(entry) & DAX_LOCKED;
110+}
111+
112+static unsigned int dax_entry_order(void *entry)
113+{
114+ if (xa_to_value(entry) & DAX_PMD)
115+ return PMD_ORDER;
91116 return 0;
92117 }
93118
94119 static int dax_is_pmd_entry(void *entry)
95120 {
96− return (unsigned long)entry & RADIX_DAX_PMD;
121+ return xa_to_value(entry) & DAX_PMD;
97122 }
98123
99124 static int dax_is_pte_entry(void *entry)
100125 {
101− return !((unsigned long)entry & RADIX_DAX_PMD);
126+ return !(xa_to_value(entry) & DAX_PMD);
102127 }
103128
104129 static int dax_is_zero_entry(void *entry)
105130 {
106− return (unsigned long)entry & RADIX_DAX_ZERO_PAGE;
131+ return xa_to_value(entry) & DAX_ZERO_PAGE;
107132 }
108133
109134 static int dax_is_empty_entry(void *entry)
110135 {
111− return (unsigned long)entry & RADIX_DAX_EMPTY;
136+ return xa_to_value(entry) & DAX_EMPTY;
112137 }
113138
114139 /*
115− * DAX radix tree locking
140+ * DAX page cache entry locking
116141 */
117142 struct exceptional_entry_key {
118− struct address_space *mapping;
143+ struct xarray *xa;
119144 pgoff_t entry_start;
120145 };
121146
@@ -124,10 +149,11 @@ struct wait_exceptional_entry_queue { · 2 unmodified lines
124149 struct exceptional_entry_key key;
125150 };
126151
127−static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
128− pgoff_t index, void *entry, struct exceptional_entry_key *key)
152+static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas,
153+ void *entry, struct exceptional_entry_key *key)
129154 {
130155 unsigned long hash;
156+ unsigned long index = xas->xa_index;
131157
132158 /*
133159 * If 'entry' is a PMD, align the 'index' that we use for the wait
@@ -136,22 +162,21 @@ static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping, · 2 unmodified lines
136162 */
137163 if (dax_is_pmd_entry(entry))
138164 index &= ~PG_PMD_COLOUR;
139−
140− key->mapping = mapping;
165+ key->xa = xas->xa;
141166 key->entry_start = index;
142167
143− hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
168+ hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS);
144169 return wait_table + hash;
145170 }
146171
147−static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode,
148− int sync, void *keyp)
172+static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
173+ unsigned int mode, int sync, void *keyp)
149174 {
150175 struct exceptional_entry_key *key = keyp;
151176 struct wait_exceptional_entry_queue *ewait =
152177 container_of(wait, struct wait_exceptional_entry_queue, wait);
153178
154− if (key->mapping != ewait->key.mapping ||
179+ if (key->xa != ewait->key.xa ||
155180 key->entry_start != ewait->key.entry_start)
156181 return 0;
157182 return autoremove_wake_function(wait, mode, sync, NULL);
@@ -162,13 +187,12 @@ static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mo · 4 unmodified lines
162187 * The important information it's conveying is whether the entry at
163188 * this index used to be a PMD entry.
164189 */
165−static void dax_wake_mapping_entry_waiter(struct address_space *mapping,
166− pgoff_t index, void *entry, bool wake_all)
190+static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all)
167191 {
168192 struct exceptional_entry_key key;
169193 wait_queue_head_t *wq;
170194
171− wq = dax_entry_waitqueue(mapping, index, entry, &key);
195+ wq = dax_entry_waitqueue(xas, entry, &key);
172196
173197 /*
174198 * Checking for locked entry and prepare_to_wait_exclusive() happens
@@ -181,55 +205,16 @@ static void dax_wake_mapping_entry_waiter(struct address_space *mapping, · 6 unmodified lines
181205 }
182206
183207 /*
184− * Check whether the given slot is locked. Must be called with the i_pages
185− * lock held.
186− */
187−static inline int slot_locked(struct address_space *mapping, void **slot)
188−{
189− unsigned long entry = (unsigned long)
190− radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock);
191− return entry & RADIX_DAX_ENTRY_LOCK;
192−}
193−
194−/*
195− * Mark the given slot as locked. Must be called with the i_pages lock held.
196− */
197−static inline void *lock_slot(struct address_space *mapping, void **slot)
198−{
199− unsigned long entry = (unsigned long)
200− radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock);
201−
202− entry |= RADIX_DAX_ENTRY_LOCK;
203− radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry);
204− return (void *)entry;
205−}
206−
207−/*
208− * Mark the given slot as unlocked. Must be called with the i_pages lock held.
209− */
210−static inline void *unlock_slot(struct address_space *mapping, void **slot)
211−{
212− unsigned long entry = (unsigned long)
213− radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock);
214−
215− entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
216− radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry);
217− return (void *)entry;
218−}
219−
220−/*
221− * Lookup entry in radix tree, wait for it to become unlocked if it is
222− * exceptional entry and return it. The caller must call
223− * put_unlocked_mapping_entry() when he decided not to lock the entry or
224− * put_locked_mapping_entry() when he locked the entry and now wants to
225− * unlock it.
208+ * Look up entry in page cache, wait for it to become unlocked if it
209+ * is a DAX entry and return it. The caller must subsequently call
210+ * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry()
211+ * if it did.
226212 *
227213 * Must be called with the i_pages lock held.
228214 */
229−static void *__get_unlocked_mapping_entry(struct address_space *mapping,
230− pgoff_t index, void ***slotp, bool (*wait_fn)(void))
215+static void *get_unlocked_entry(struct xa_state *xas)
231216 {
232− void *entry, **slot;
217+ void *entry;
233218 struct wait_exceptional_entry_queue ewait;
234219 wait_queue_head_t *wq;
235220
@@ -237,80 +222,54 @@ static void *__get_unlocked_mapping_entry(struct address_space *mapping, · 1 unmodified line
237222 ewait.wait.func = wake_exceptional_entry_func;
238223
239224 for (;;) {
240− bool revalidate;
241−
242− entry = __radix_tree_lookup(&mapping->i_pages, index, NULL,
243− &slot);
244− if (!entry ||
245− WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) ||
246− !slot_locked(mapping, slot)) {
247− if (slotp)
248− *slotp = slot;
225+ entry = xas_load(xas);
226+ if (!entry || xa_is_internal(entry) ||
227+ WARN_ON_ONCE(!xa_is_value(entry)) ||
228+ !dax_is_locked(entry))
249229 return entry;
250− }
251230
252− wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
231+ wq = dax_entry_waitqueue(xas, entry, &ewait.key);
253232 prepare_to_wait_exclusive(wq, &ewait.wait,
254233 TASK_UNINTERRUPTIBLE);
255− xa_unlock_irq(&mapping->i_pages);
256− revalidate = wait_fn();
234+ xas_unlock_irq(xas);
235+ xas_reset(xas);
236+ schedule();
257237 finish_wait(wq, &ewait.wait);
258− xa_lock_irq(&mapping->i_pages);
259− if (revalidate)
260− return ERR_PTR(-EAGAIN);
238+ xas_lock_irq(xas);
261239 }
262240 }
263241
264−static bool entry_wait(void)
265−{
266− schedule();
267− /*
268− * Never return an ERR_PTR() from
269− * __get_unlocked_mapping_entry(), just keep looping.
270− */
271− return false;
272−}
273−
274−static void *get_unlocked_mapping_entry(struct address_space *mapping,
275− pgoff_t index, void ***slotp)
242+static void put_unlocked_entry(struct xa_state *xas, void *entry)
276243 {
277− return __get_unlocked_mapping_entry(mapping, index, slotp, entry_wait);
278−}
279−
280−static void unlock_mapping_entry(struct address_space *mapping, pgoff_t index)
281−{
282− void *entry, **slot;
283−
284− xa_lock_irq(&mapping->i_pages);
285− entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot);
286− if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
287− !slot_locked(mapping, slot))) {
288− xa_unlock_irq(&mapping->i_pages);
289− return;
290− }
291− unlock_slot(mapping, slot);
292− xa_unlock_irq(&mapping->i_pages);
293− dax_wake_mapping_entry_waiter(mapping, index, entry, false);
244+ /* If we were the only waiter woken, wake the next one */
245+ if (entry)
246+ dax_wake_entry(xas, entry, false);
294247 }
295248
296−static void put_locked_mapping_entry(struct address_space *mapping,
297− pgoff_t index)
249+/*
250+ * We used the xa_state to get the entry, but then we locked the entry and
251+ * dropped the xa_lock, so we know the xa_state is stale and must be reset
252+ * before use.
253+ */
254+static void dax_unlock_entry(struct xa_state *xas, void *entry)
298255 {
299− unlock_mapping_entry(mapping, index);
256+ void *old;
257+
258+ xas_reset(xas);
259+ xas_lock_irq(xas);
260+ old = xas_store(xas, entry);
261+ xas_unlock_irq(xas);
262+ BUG_ON(!dax_is_locked(old));
263+ dax_wake_entry(xas, entry, false);
300264 }
301265
302266 /*
303− * Called when we are done with radix tree entry we looked up via
304− * get_unlocked_mapping_entry() and which we didn't lock in the end.
267+ * Return: The entry stored at this location before it was locked.
305268 */
306−static void put_unlocked_mapping_entry(struct address_space *mapping,
307− pgoff_t index, void *entry)
269+static void *dax_lock_entry(struct xa_state *xas, void *entry)
308270 {
309− if (!entry)
310− return;
311−
312− /* We have to wake up next waiter for the radix tree entry lock */
313− dax_wake_mapping_entry_waiter(mapping, index, entry, false);
271+ unsigned long v = xa_to_value(entry);
272+ return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
314273 }
315274
316275 static unsigned long dax_entry_size(void *entry)
@@ -325,9 +284,9 @@ static unsigned long dax_entry_size(void *entry) · 8 unmodified lines
325284 return PAGE_SIZE;
326285 }
327286
328−static unsigned long dax_radix_end_pfn(void *entry)
287+static unsigned long dax_end_pfn(void *entry)
329288 {
330− return dax_radix_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
289+ return dax_to_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
331290 }
332291
333292 /*
@@ -335,8 +294,8 @@ static unsigned long dax_radix_end_pfn(void *entry) · 1 unmodified line
335294 * 'empty' and 'zero' entries.
336295 */
337296 #define for_each_mapped_pfn(entry, pfn) \
338− for (pfn = dax_radix_pfn(entry); \
339− pfn < dax_radix_end_pfn(entry); pfn++)
297+ for (pfn = dax_to_pfn(entry); \
298+ pfn < dax_end_pfn(entry); pfn++)
340299
341300 /*
342301 * TODO: for reflink+dax we need a way to associate a single page with
@@ -393,33 +352,16 @@ static struct page *dax_busy_page(void *entry) · 50 unmodified lines
393352 return NULL;
394353 }
395354
396−static bool entry_wait_revalidate(void)
397−{
398− rcu_read_unlock();
399− schedule();
400− rcu_read_lock();
401−
402− /*
403− * Tell __get_unlocked_mapping_entry() to take a break, we need
404− * to revalidate page->mapping after dropping locks
405− */
406− return true;
407−}
408−
409355 bool dax_lock_mapping_entry(struct page *page)
410356 {
411− pgoff_t index;
412− struct inode *inode;
413− bool did_lock = false;
414− void *entry = NULL, **slot;
415− struct address_space *mapping;
357+ XA_STATE(xas, NULL, 0);
358+ void *entry;
416359
417− rcu_read_lock();
418360 for (;;) {
419− mapping = READ_ONCE(page->mapping);
361+ struct address_space *mapping = READ_ONCE(page->mapping);
420362
421363 if (!dax_mapping(mapping))
422− break;
364+ return false;
423365
424366 /*
425367 * In the device-dax case there's no need to lock, a
@@ -428,98 +370,94 @@ bool dax_lock_mapping_entry(struct page *page) · 2 unmodified lines
428370 * otherwise we would not have a valid pfn_to_page()
429371 * translation.
430372 */
431− inode = mapping->host;
432− if (S_ISCHR(inode->i_mode)) {
433− did_lock = true;
434− break;
435− }
373+ if (S_ISCHR(mapping->host->i_mode))
374+ return true;
436375
437− xa_lock_irq(&mapping->i_pages);
376+ xas.xa = &mapping->i_pages;
377+ xas_lock_irq(&xas);
438378 if (mapping != page->mapping) {
439− xa_unlock_irq(&mapping->i_pages);
379+ xas_unlock_irq(&xas);
440380 continue;
441381 }
442− index = page->index;
443−
444− entry = __get_unlocked_mapping_entry(mapping, index, &slot,
445− entry_wait_revalidate);
446− if (!entry) {
447− xa_unlock_irq(&mapping->i_pages);
448− break;
449− } else if (IS_ERR(entry)) {
450− xa_unlock_irq(&mapping->i_pages);
451− WARN_ON_ONCE(PTR_ERR(entry) != -EAGAIN);
452− continue;
382+ xas_set(&xas, page->index);
383+ entry = xas_load(&xas);
384+ if (dax_is_locked(entry)) {
385+ entry = get_unlocked_entry(&xas);
386+ /* Did the page move while we slept? */
387+ if (dax_to_pfn(entry) != page_to_pfn(page)) {
388+ xas_unlock_irq(&xas);
389+ continue;
390+ }
453391 }
454− lock_slot(mapping, slot);
455− did_lock = true;
456− xa_unlock_irq(&mapping->i_pages);
457− break;
392+ dax_lock_entry(&xas, entry);
393+ xas_unlock_irq(&xas);
394+ return true;
458395 }
459− rcu_read_unlock();
460−
461− return did_lock;
462396 }
463397
464398 void dax_unlock_mapping_entry(struct page *page)
465399 {
466400 struct address_space *mapping = page->mapping;
467− struct inode *inode = mapping->host;
401+ XA_STATE(xas, &mapping->i_pages, page->index);
468402
469− if (S_ISCHR(inode->i_mode))
403+ if (S_ISCHR(mapping->host->i_mode))
470404 return;
471405
472− unlock_mapping_entry(mapping, page->index);
406+ dax_unlock_entry(&xas, dax_make_page_entry(page));
473407 }
474408
475409 /*
476− * Find radix tree entry at given index. If it points to an exceptional entry,
477− * return it with the radix tree entry locked. If the radix tree doesn't
478− * contain given index, create an empty exceptional entry for the index and
479− * return with it locked.
410+ * Find page cache entry at given index. If it is a DAX entry, return it
411+ * with the entry locked. If the page cache doesn't contain an entry at
412+ * that index, add a locked empty entry.
480413 *
481− * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
482− * either return that locked entry or will return an error. This error will
483− * happen if there are any 4k entries within the 2MiB range that we are
484− * requesting.
414+ * When requesting an entry with size DAX_PMD, grab_mapping_entry() will
415+ * either return that locked entry or will return VM_FAULT_FALLBACK.
416+ * This will happen if there are any PTE entries within the PMD range
417+ * that we are requesting.
485418 *
486− * We always favor 4k entries over 2MiB entries. There isn't a flow where we
487− * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB
488− * insertion will fail if it finds any 4k entries already in the tree, and a
489− * 4k insertion will cause an existing 2MiB entry to be unmapped and
490− * downgraded to 4k entries. This happens for both 2MiB huge zero pages as
491− * well as 2MiB empty entries.
419+ * We always favor PTE entries over PMD entries. There isn't a flow where we
420+ * evict PTE entries in order to 'upgrade' them to a PMD entry. A PMD
421+ * insertion will fail if it finds any PTE entries already in the tree, and a
422+ * PTE insertion will cause an existing PMD entry to be unmapped and
423+ * downgraded to PTE entries. This happens for both PMD zero pages as
424+ * well as PMD empty entries.
492425 *
493− * The exception to this downgrade path is for 2MiB DAX PMD entries that have
494− * real storage backing them. We will leave these real 2MiB DAX entries in
495− * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
426+ * The exception to this downgrade path is for PMD entries that have
427+ * real storage backing them. We will leave these real PMD entries in
428+ * the tree, and PTE writes will simply dirty the entire PMD entry.
496429 *
497430 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
498431 * persistent memory the benefit is doubtful. We can add that later if we can
499432 * show it helps.
433+ *
434+ * On error, this function does not return an ERR_PTR. Instead it returns
435+ * a VM_FAULT code, encoded as an xarray internal entry. The ERR_PTR values
436+ * overlap with xarray value entries.
500437 */
501−static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
502− unsigned long size_flag)
438+static void *grab_mapping_entry(struct xa_state *xas,
439+ struct address_space *mapping, unsigned long size_flag)
503440 {
504− bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
505− void *entry, **slot;
506−
507−restart:
508− xa_lock_irq(&mapping->i_pages);
509− entry = get_unlocked_mapping_entry(mapping, index, &slot);
441+ unsigned long index = xas->xa_index;
442+ bool pmd_downgrade = false; /* splitting PMD entry into PTE entries? */
443+ void *entry;
510444
511− if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) {
512− entry = ERR_PTR(-EIO);
513− goto out_unlock;
514− }
445+retry:
446+ xas_lock_irq(xas);
447+ entry = get_unlocked_entry(xas);
448+ if (xa_is_internal(entry))
449+ goto fallback;
515450
516451 if (entry) {
517− if (size_flag & RADIX_DAX_PMD) {
452+ if (WARN_ON_ONCE(!xa_is_value(entry))) {
453+ xas_set_err(xas, EIO);
454+ goto out_unlock;
455+ }
456+
457+ if (size_flag & DAX_PMD) {
518458 if (dax_is_pte_entry(entry)) {
519− put_unlocked_mapping_entry(mapping, index,
520− entry);
521− entry = ERR_PTR(-EEXIST);
522− goto out_unlock;
459+ put_unlocked_entry(xas, entry);
460+ goto fallback;
523461 }
524462 } else { /* trying to grab a PTE entry */
525463 if (dax_is_pmd_entry(entry) &&
@@ -530,87 +468,57 @@ restart: · 4 unmodified lines
530468 }
531469 }
532470
533− /* No entry for given index? Make sure radix tree is big enough. */
534− if (!entry || pmd_downgrade) {
535− int err;
536−
537− if (pmd_downgrade) {
538− /*
539− * Make sure 'entry' remains valid while we drop
540− * the i_pages lock.
541− */
542− entry = lock_slot(mapping, slot);
543− }
471+ if (pmd_downgrade) {
472+ /*
473+ * Make sure 'entry' remains valid while we drop
474+ * the i_pages lock.
475+ */
476+ dax_lock_entry(xas, entry);
544477
545− xa_unlock_irq(&mapping->i_pages);
546478 /*
547479 * Besides huge zero pages the only other thing that gets
548480 * downgraded are empty entries which don't need to be
549481 * unmapped.
550482 */
551− if (pmd_downgrade && dax_is_zero_entry(entry))
552− unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
553− PG_PMD_NR, false);
554−
555− err = radix_tree_preload(
556− mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
557− if (err) {
558− if (pmd_downgrade)
559− put_locked_mapping_entry(mapping, index);
560− return ERR_PTR(err);
561− }
562− xa_lock_irq(&mapping->i_pages);
563−
564− if (!entry) {
565− /*
566− * We needed to drop the i_pages lock while calling
567− * radix_tree_preload() and we didn't have an entry to
568− * lock. See if another thread inserted an entry at
569− * our index during this time.
570− */
571− entry = __radix_tree_lookup(&mapping->i_pages, index,
572− NULL, &slot);
573− if (entry) {
574− radix_tree_preload_end();
575− xa_unlock_irq(&mapping->i_pages);
576− goto restart;
577− }
483+ if (dax_is_zero_entry(entry)) {
484+ xas_unlock_irq(xas);
485+ unmap_mapping_pages(mapping,
486+ xas->xa_index & ~PG_PMD_COLOUR,
487+ PG_PMD_NR, false);
488+ xas_reset(xas);
489+ xas_lock_irq(xas);
578490 }
579491
580− if (pmd_downgrade) {
581− dax_disassociate_entry(entry, mapping, false);
582− radix_tree_delete(&mapping->i_pages, index);
583− mapping->nrexceptional--;
584− dax_wake_mapping_entry_waiter(mapping, index, entry,
585− true);
586− }
492+ dax_disassociate_entry(entry, mapping, false);
493+ xas_store(xas, NULL); /* undo the PMD join */
494+ dax_wake_entry(xas, entry, true);
495+ mapping->nrexceptional--;
496+ entry = NULL;
497+ xas_set(xas, index);
498+ }
587499
588− entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);
589−
590− err = __radix_tree_insert(&mapping->i_pages, index,
591− dax_radix_order(entry), entry);
592− radix_tree_preload_end();
593− if (err) {
594− xa_unlock_irq(&mapping->i_pages);
595− /*
596− * Our insertion of a DAX entry failed, most likely
597− * because we were inserting a PMD entry and it
598− * collided with a PTE sized entry at a different
599− * index in the PMD range. We haven't inserted
600− * anything into the radix tree and have no waiters to
601− * wake.
602− */
603− return ERR_PTR(err);
604− }
605− /* Good, we have inserted empty locked entry into the tree. */
500+ if (entry) {
501+ dax_lock_entry(xas, entry);
502+ } else {
503+ entry = dax_make_entry(pfn_to_pfn_t(0), size_flag | DAX_EMPTY);
504+ dax_lock_entry(xas, entry);
505+ if (xas_error(xas))
506+ goto out_unlock;
606507 mapping->nrexceptional++;
607− xa_unlock_irq(&mapping->i_pages);
608− return entry;
609508 }
610− entry = lock_slot(mapping, slot);
611− out_unlock:
612− xa_unlock_irq(&mapping->i_pages);
509+
510+out_unlock:
511+ xas_unlock_irq(xas);
512+ if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))
513+ goto retry;
514+ if (xas->xa_node == XA_ERROR(-ENOMEM))
515+ return xa_mk_internal(VM_FAULT_OOM);
516+ if (xas_error(xas))
517+ return xa_mk_internal(VM_FAULT_SIGBUS);
613518 return entry;
519+fallback:
520+ xas_unlock_irq(xas);
521+ return xa_mk_internal(VM_FAULT_FALLBACK);
614522 }
615523
616524 /**
@@ -630,11 +538,10 @@ restart: · 13 unmodified lines
630538 */
631539 struct page *dax_layout_busy_page(struct address_space *mapping)
632540 {
633− pgoff_t indices[PAGEVEC_SIZE];
541+ XA_STATE(xas, &mapping->i_pages, 0);
542+ void *entry;
543+ unsigned int scanned = 0;
634544 struct page *page = NULL;
635− struct pagevec pvec;
636− pgoff_t index, end;
637− unsigned i;
638545
639546 /*
640547 * In the 'limited' case get_user_pages() for dax is disabled.
@@ -645,13 +552,9 @@ struct page *dax_layout_busy_page(struct address_space *mapping) · 4 unmodified lines
645552 if (!dax_mapping(mapping) || !mapping_mapped(mapping))
646553 return NULL;
647554
648− pagevec_init(&pvec);
649− index = 0;
650− end = -1;
651−
652555 /*
653556 * If we race get_user_pages_fast() here either we'll see the
654− * elevated page count in the pagevec_lookup and wait, or
557+ * elevated page count in the iteration and wait, or
655558 * get_user_pages_fast() will see that the page it took a reference
656559 * against is no longer mapped in the page tables and bail to the
657560 * get_user_pages() slow path. The slow path is protected by
@@ -663,94 +566,68 @@ struct page *dax_layout_busy_page(struct address_space *mapping) · 5 unmodified lines
663566 */
664567 unmap_mapping_range(mapping, 0, 0, 1);
665568
666− while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
667− min(end - index, (pgoff_t)PAGEVEC_SIZE),
668− indices)) {
669− pgoff_t nr_pages = 1;
670−
671− for (i = 0; i < pagevec_count(&pvec); i++) {
672− struct page *pvec_ent = pvec.pages[i];
673− void *entry;
674−
675− index = indices[i];
676− if (index >= end)
677− break;
678−
679− if (WARN_ON_ONCE(
680− !radix_tree_exceptional_entry(pvec_ent)))
681− continue;
682−
683− xa_lock_irq(&mapping->i_pages);
684− entry = get_unlocked_mapping_entry(mapping, index, NULL);
685− if (entry) {
686− page = dax_busy_page(entry);
687− /*
688− * Account for multi-order entries at
689− * the end of the pagevec.
690− */
691− if (i + 1 >= pagevec_count(&pvec))
692− nr_pages = 1UL << dax_radix_order(entry);
693− }
694− put_unlocked_mapping_entry(mapping, index, entry);
695− xa_unlock_irq(&mapping->i_pages);
696− if (page)
697− break;
698− }
699−
700− /*
701− * We don't expect normal struct page entries to exist in our
702− * tree, but we keep these pagevec calls so that this code is
703− * consistent with the common pattern for handling pagevecs
704− * throughout the kernel.
705− */
706− pagevec_remove_exceptionals(&pvec);
707− pagevec_release(&pvec);
708− index += nr_pages;
709−
569+ xas_lock_irq(&xas);
570+ xas_for_each(&xas, entry, ULONG_MAX) {
571+ if (WARN_ON_ONCE(!xa_is_value(entry)))
572+ continue;
573+ if (unlikely(dax_is_locked(entry)))
574+ entry = get_unlocked_entry(&xas);
575+ if (entry)
576+ page = dax_busy_page(entry);
577+ put_unlocked_entry(&xas, entry);
710578 if (page)
711579 break;
580+ if (++scanned % XA_CHECK_SCHED)
581+ continue;
582+
583+ xas_pause(&xas);
584+ xas_unlock_irq(&xas);
585+ cond_resched();
586+ xas_lock_irq(&xas);
712587 }
588+ xas_unlock_irq(&xas);
713589 return page;
714590 }
715591 EXPORT_SYMBOL_GPL(dax_layout_busy_page);
716592
717−static int __dax_invalidate_mapping_entry(struct address_space *mapping,
593+static int __dax_invalidate_entry(struct address_space *mapping,
718594 pgoff_t index, bool trunc)
719595 {
596+ XA_STATE(xas, &mapping->i_pages, index);
720597 int ret = 0;
721598 void *entry;
722− struct radix_tree_root *pages = &mapping->i_pages;
723599
724− xa_lock_irq(pages);
725− entry = get_unlocked_mapping_entry(mapping, index, NULL);
726− if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)))
600+ xas_lock_irq(&xas);
601+ entry = get_unlocked_entry(&xas);
602+ if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
727603 goto out;
728604 if (!trunc &&
729− (radix_tree_tag_get(pages, index, PAGECACHE_TAG_DIRTY) ||
730− radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE)))
605+ (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
606+ xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
731607 goto out;
732608 dax_disassociate_entry(entry, mapping, trunc);
733− radix_tree_delete(pages, index);
609+ xas_store(&xas, NULL);
734610 mapping->nrexceptional--;
735611 ret = 1;
736612 out:
737− put_unlocked_mapping_entry(mapping, index, entry);
738− xa_unlock_irq(pages);
613+ put_unlocked_entry(&xas, entry);
614+ xas_unlock_irq(&xas);
739615 return ret;
740616 }
617+
741618 /*
742− * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
743− * entry to get unlocked before deleting it.
619+ * Delete DAX entry at @index from @mapping. Wait for it
620+ * to be unlocked before deleting it.
744621 */
745622 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
746623 {
747− int ret = __dax_invalidate_mapping_entry(mapping, index, true);
624+ int ret = __dax_invalidate_entry(mapping, index, true);
748625
749626 /*
750627 * This gets called from truncate / punch_hole path. As such, the caller
751628 * must hold locks protecting against concurrent modifications of the
752− * radix tree (usually fs-private i_mmap_sem for writing). Since the
753− * caller has seen exceptional entry for this index, we better find it
629+ * page cache (usually fs-private i_mmap_sem for writing). Since the
630+ * caller has seen a DAX entry for this index, we better find it
754631 * at that index as well...
755632 */
756633 WARN_ON_ONCE(!ret);
@@ -758,12 +635,12 @@ int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) · 1 unmodified line
758635 }
759636
760637 /*
761− * Invalidate exceptional DAX entry if it is clean.
638+ * Invalidate DAX entry if it is clean.
762639 */
763640 int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
764641 pgoff_t index)
765642 {
766− return __dax_invalidate_mapping_entry(mapping, index, false);
643+ return __dax_invalidate_entry(mapping, index, false);
767644 }
768645
769646 static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
@@ -799,30 +676,27 @@ static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev, · 29 unmodified lines
799676 * already in the tree, we will skip the insertion and just dirty the PMD as
800677 * appropriate.
801678 */
802−static void *dax_insert_mapping_entry(struct address_space *mapping,
803− struct vm_fault *vmf,
804− void *entry, pfn_t pfn_t,
805− unsigned long flags, bool dirty)
679+static void *dax_insert_entry(struct xa_state *xas,
680+ struct address_space *mapping, struct vm_fault *vmf,
681+ void *entry, pfn_t pfn, unsigned long flags, bool dirty)
806682 {
807− struct radix_tree_root *pages = &mapping->i_pages;
808− unsigned long pfn = pfn_t_to_pfn(pfn_t);
809− pgoff_t index = vmf->pgoff;
810− void *new_entry;
683+ void *new_entry = dax_make_entry(pfn, flags);
811684
812685 if (dirty)
813686 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
814687
815− if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) {
688+ if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) {
689+ unsigned long index = xas->xa_index;
816690 /* we are replacing a zero page with block mapping */
817691 if (dax_is_pmd_entry(entry))
818692 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
819− PG_PMD_NR, false);
693+ PG_PMD_NR, false);
820694 else /* pte entry */
821− unmap_mapping_pages(mapping, vmf->pgoff, 1, false);
695+ unmap_mapping_pages(mapping, index, 1, false);
822696 }
823697
824− xa_lock_irq(pages);
825− new_entry = dax_radix_locked_entry(pfn, flags);
698+ xas_reset(xas);
699+ xas_lock_irq(xas);
826700 if (dax_entry_size(entry) != dax_entry_size(new_entry)) {
827701 dax_disassociate_entry(entry, mapping, false);
828702 dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
@@ -830,33 +704,30 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, · 1 unmodified line
830704
831705 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
832706 /*
833− * Only swap our new entry into the radix tree if the current
707+ * Only swap our new entry into the page cache if the current
834708 * entry is a zero page or an empty entry. If a normal PTE or
835− * PMD entry is already in the tree, we leave it alone. This
709+ * PMD entry is already in the cache, we leave it alone. This
836710 * means that if we are trying to insert a PTE and the
837711 * existing entry is a PMD, we will just leave the PMD in the
838712 * tree and dirty it if necessary.
839713 */
840− struct radix_tree_node *node;
841− void **slot;
842− void *ret;
843−
844− ret = __radix_tree_lookup(pages, index, &node, &slot);
845− WARN_ON_ONCE(ret != entry);
846− __radix_tree_replace(pages, node, slot,
847− new_entry, NULL);
714+ void *old = dax_lock_entry(xas, new_entry);
715+ WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) |
716+ DAX_LOCKED));
848717 entry = new_entry;
718+ } else {
719+ xas_load(xas); /* Walk the xa_state */
849720 }
850721
851722 if (dirty)
852− radix_tree_tag_set(pages, index, PAGECACHE_TAG_DIRTY);
723+ xas_set_mark(xas, PAGECACHE_TAG_DIRTY);
853724
854− xa_unlock_irq(pages);
725+ xas_unlock_irq(xas);
855726 return entry;
856727 }
857728
858−static inline unsigned long
859−pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
729+static inline
730+unsigned long pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
860731 {
861732 unsigned long address;
862733
@@ -866,8 +737,8 @@ pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma) · 3 unmodified lines
866737 }
867738
868739 /* Walk all mappings of a given index of a file and writeprotect them */
869−static void dax_mapping_entry_mkclean(struct address_space *mapping,
870− pgoff_t index, unsigned long pfn)
740+static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
741+ unsigned long pfn)
871742 {
872743 struct vm_area_struct *vma;
873744 pte_t pte, *ptep = NULL;
@@ -937,11 +808,9 @@ unlock_pte: · 63 unmodified lines
937808 i_mmap_unlock_read(mapping);
938809 }
939810
940−static int dax_writeback_one(struct dax_device *dax_dev,
941− struct address_space *mapping, pgoff_t index, void *entry)
811+static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
812+ struct address_space *mapping, void *entry)
942813 {
943− struct radix_tree_root *pages = &mapping->i_pages;
944− void *entry2, **slot;
945814 unsigned long pfn;
946815 long ret = 0;
947816 size_t size;
@@ -950,32 +819,38 @@ static int dax_writeback_one(struct dax_device *dax_dev, · 2 unmodified lines
950819 * A page got tagged dirty in DAX mapping? Something is seriously
951820 * wrong.
952821 */
953− if (WARN_ON(!radix_tree_exceptional_entry(entry)))
822+ if (WARN_ON(!xa_is_value(entry)))
954823 return -EIO;
955824
956− xa_lock_irq(pages);
957− entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
958− /* Entry got punched out / reallocated? */
959− if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2)))
960− goto put_unlocked;
961− /*
962− * Entry got reallocated elsewhere? No need to writeback. We have to
963− * compare pfns as we must not bail out due to difference in lockbit
964− * or entry type.
965− */
966− if (dax_radix_pfn(entry2) != dax_radix_pfn(entry))
967− goto put_unlocked;
968− if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
969− dax_is_zero_entry(entry))) {
970− ret = -EIO;
971− goto put_unlocked;
825+ if (unlikely(dax_is_locked(entry))) {
826+ void *old_entry = entry;
827+
828+ entry = get_unlocked_entry(xas);
829+
830+ /* Entry got punched out / reallocated? */
831+ if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
832+ goto put_unlocked;
833+ /*
834+ * Entry got reallocated elsewhere? No need to writeback.
835+ * We have to compare pfns as we must not bail out due to
836+ * difference in lockbit or entry type.
837+ */
838+ if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
839+ goto put_unlocked;
840+ if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
841+ dax_is_zero_entry(entry))) {
842+ ret = -EIO;
843+ goto put_unlocked;
844+ }
845+
846+ /* Another fsync thread may have already done this entry */
847+ if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
848+ goto put_unlocked;
972849 }
973850
974− /* Another fsync thread may have already written back this entry */
975− if (!radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE))
976− goto put_unlocked;
977851 /* Lock the entry to serialize with page faults */
978− entry = lock_slot(mapping, slot);
852+ dax_lock_entry(xas, entry);
853+
979854 /*
980855 * We can clear the tag now but we have to be careful so that concurrent
981856 * dax_writeback_one() calls for the same index cannot finish before we
@@ -983,8 +858,8 @@ static int dax_writeback_one(struct dax_device *dax_dev, · 1 unmodified line
983858 * at the entry only under the i_pages lock and once they do that
984859 * they will see the entry locked and wait for it to unlock.
985860 */
986− radix_tree_tag_clear(pages, index, PAGECACHE_TAG_TOWRITE);
987− xa_unlock_irq(pages);
861+ xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
862+ xas_unlock_irq(xas);
988863
989864 /*
990865 * Even if dax_writeback_mapping_range() was given a wbc->range_start
@@ -993,10 +868,10 @@ static int dax_writeback_one(struct dax_device *dax_dev, · 2 unmodified lines
993868 * This allows us to flush for PMD_SIZE and not have to worry about
994869 * partial PMD writebacks.
995870 */
996− pfn = dax_radix_pfn(entry);
997− size = PAGE_SIZE << dax_radix_order(entry);
871+ pfn = dax_to_pfn(entry);
872+ size = PAGE_SIZE << dax_entry_order(entry);
998873
999− dax_mapping_entry_mkclean(mapping, index, pfn);
874+ dax_entry_mkclean(mapping, xas->xa_index, pfn);
1000875 dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size);
1001876 /*
1002877 * After we have flushed the cache, we can clear the dirty tag. There
@@ -1004,16 +879,18 @@ static int dax_writeback_one(struct dax_device *dax_dev, · 1 unmodified line
1004879 * the pfn mappings are writeprotected and fault waits for mapping
1005880 * entry lock.
1006881 */
1007− xa_lock_irq(pages);
1008− radix_tree_tag_clear(pages, index, PAGECACHE_TAG_DIRTY);
1009− xa_unlock_irq(pages);
1010− trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT);
1011− put_locked_mapping_entry(mapping, index);
882+ xas_reset(xas);
883+ xas_lock_irq(xas);
884+ xas_store(xas, entry);
885+ xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
886+ dax_wake_entry(xas, entry, false);
887+
888+ trace_dax_writeback_one(mapping->host, xas->xa_index,
889+ size >> PAGE_SHIFT);
1012890 return ret;
1013891
1014892 put_unlocked:
1015− put_unlocked_mapping_entry(mapping, index, entry2);
1016− xa_unlock_irq(pages);
893+ put_unlocked_entry(xas, entry);
1017894 return ret;
1018895 }
1019896
@@ -1025,13 +902,13 @@ static int dax_writeback_one(struct dax_device *dax_dev, · 5 unmodified lines
1025902 int dax_writeback_mapping_range(struct address_space *mapping,
1026903 struct block_device *bdev, struct writeback_control *wbc)
1027904 {
905+ XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
1028906 struct inode *inode = mapping->host;
1029− pgoff_t start_index, end_index;
1030− pgoff_t indices[PAGEVEC_SIZE];
907+ pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
1031908 struct dax_device *dax_dev;
1032− struct pagevec pvec;
1033− bool done = false;
1034− int i, ret = 0;
909+ void *entry;
910+ int ret = 0;
911+ unsigned int scanned = 0;
1035912
1036913 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
1037914 return -EIO;
@@ -1043,41 +920,29 @@ int dax_writeback_mapping_range(struct address_space *mapping, · 5 unmodified lines
1043920 if (!dax_dev)
1044921 return -EIO;
1045922
1046− start_index = wbc->range_start >> PAGE_SHIFT;
1047− end_index = wbc->range_end >> PAGE_SHIFT;
1048−
1049− trace_dax_writeback_range(inode, start_index, end_index);
923+ trace_dax_writeback_range(inode, xas.xa_index, end_index);
1050924
1051− tag_pages_for_writeback(mapping, start_index, end_index);
925+ tag_pages_for_writeback(mapping, xas.xa_index, end_index);
1052926
1053− pagevec_init(&pvec);
1054− while (!done) {
1055− pvec.nr = find_get_entries_tag(mapping, start_index,
1056− PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
1057− pvec.pages, indices);
1058−
1059− if (pvec.nr == 0)
927+ xas_lock_irq(&xas);
928+ xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
929+ ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
930+ if (ret < 0) {
931+ mapping_set_error(mapping, ret);
1060932 break;
1061−
1062− for (i = 0; i < pvec.nr; i++) {
1063− if (indices[i] > end_index) {
1064− done = true;
1065− break;
1066− }
1067−
1068− ret = dax_writeback_one(dax_dev, mapping, indices[i],
1069− pvec.pages[i]);
1070− if (ret < 0) {
1071− mapping_set_error(mapping, ret);
1072− goto out;
1073− }
1074933 }
1075− start_index = indices[pvec.nr - 1] + 1;
934+ if (++scanned % XA_CHECK_SCHED)
935+ continue;
936+
937+ xas_pause(&xas);
938+ xas_unlock_irq(&xas);
939+ cond_resched();
940+ xas_lock_irq(&xas);
1076941 }
1077−out:
942+ xas_unlock_irq(&xas);
1078943 put_dax(dax_dev);
1079− trace_dax_writeback_range_done(inode, start_index, end_index);
1080− return (ret < 0 ? ret : 0);
944+ trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
945+ return ret;
1081946 }
1082947 EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
1083948
@@ -1125,16 +990,18 @@ out: · 41 unmodified lines
1125990 * If this page is ever written to we will re-fault and change the mapping to
1126991 * point to real DAX storage instead.
1127992 */
1128−static vm_fault_t dax_load_hole(struct address_space *mapping, void *entry,
1129− struct vm_fault *vmf)
993+static vm_fault_t dax_load_hole(struct xa_state *xas,
994+ struct address_space *mapping, void **entry,
995+ struct vm_fault *vmf)
1130996 {
1131997 struct inode *inode = mapping->host;
1132998 unsigned long vaddr = vmf->address;
1133999 pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr));
11341000 vm_fault_t ret;
11351001
1136− dax_insert_mapping_entry(mapping, vmf, entry, pfn, RADIX_DAX_ZERO_PAGE,
1137− false);
1002+ *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
1003+ DAX_ZERO_PAGE, false);
1004+
11381005 ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
11391006 trace_dax_load_hole(inode, vmf, ret);
11401007 return ret;
@@ -1342,6 +1209,7 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, · 201 unmodified lines
13421209 {
13431210 struct vm_area_struct *vma = vmf->vma;
13441211 struct address_space *mapping = vma->vm_file->f_mapping;
1212+ XA_STATE(xas, &mapping->i_pages, vmf->pgoff);
13451213 struct inode *inode = mapping->host;
13461214 unsigned long vaddr = vmf->address;
13471215 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
@@ -1368,9 +1236,9 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, · 20 unmodified lines
13681236 if (write && !vmf->cow_page)
13691237 flags |= IOMAP_WRITE;
13701238
1371− entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
1372− if (IS_ERR(entry)) {
1373− ret = dax_fault_return(PTR_ERR(entry));
1239+ entry = grab_mapping_entry(&xas, mapping, 0);
1240+ if (xa_is_internal(entry)) {
1241+ ret = xa_to_internal(entry);
13741242 goto out;
13751243 }
13761244
@@ -1443,7 +1311,7 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, · 66 unmodified lines
14431311 if (error < 0)
14441312 goto error_finish_iomap;
14451313
1446− entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
1314+ entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
14471315 0, write && !sync);
14481316
14491317 /*
@@ -1471,7 +1339,7 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, · 21 unmodified lines
14711339 case IOMAP_UNWRITTEN:
14721340 case IOMAP_HOLE:
14731341 if (!write) {
1474− ret = dax_load_hole(mapping, entry, vmf);
1342+ ret = dax_load_hole(&xas, mapping, &entry, vmf);
14751343 goto finish_iomap;
14761344 }
14771345 /*FALLTHRU*/
@@ -1498,21 +1366,20 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, · 20 unmodified lines
14981366 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
14991367 }
15001368 unlock_entry:
1501− put_locked_mapping_entry(mapping, vmf->pgoff);
1369+ dax_unlock_entry(&xas, entry);
15021370 out:
15031371 trace_dax_pte_fault_done(inode, vmf, ret);
15041372 return ret | major;
15051373 }
15061374
15071375 #ifdef CONFIG_FS_DAX_PMD
1508−static vm_fault_t dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap,
1509− void *entry)
1376+static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,
1377+ struct iomap *iomap, void **entry)
15101378 {
15111379 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
15121380 unsigned long pmd_addr = vmf->address & PMD_MASK;
15131381 struct inode *inode = mapping->host;
15141382 struct page *zero_page;
1515− void *ret = NULL;
15161383 spinlock_t *ptl;
15171384 pmd_t pmd_entry;
15181385 pfn_t pfn;
@@ -1523,8 +1390,8 @@ static vm_fault_t dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap, · 4 unmodified lines
15231390 goto fallback;
15241391
15251392 pfn = page_to_pfn_t(zero_page);
1526− ret = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
1527− RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false);
1393+ *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
1394+ DAX_PMD | DAX_ZERO_PAGE, false);
15281395
15291396 ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
15301397 if (!pmd_none(*(vmf->pmd))) {
@@ -1536,11 +1403,11 @@ static vm_fault_t dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap, · 5 unmodified lines
15361403 pmd_entry = pmd_mkhuge(pmd_entry);
15371404 set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
15381405 spin_unlock(ptl);
1539− trace_dax_pmd_load_hole(inode, vmf, zero_page, ret);
1406+ trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);
15401407 return VM_FAULT_NOPAGE;
15411408
15421409 fallback:
1543− trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret);
1410+ trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);
15441411 return VM_FAULT_FALLBACK;
15451412 }
15461413
@@ -1549,6 +1416,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 2 unmodified lines
15491416 {
15501417 struct vm_area_struct *vma = vmf->vma;
15511418 struct address_space *mapping = vma->vm_file->f_mapping;
1419+ XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);
15521420 unsigned long pmd_addr = vmf->address & PMD_MASK;
15531421 bool write = vmf->flags & FAULT_FLAG_WRITE;
15541422 bool sync;
@@ -1556,7 +1424,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 1 unmodified line
15561424 struct inode *inode = mapping->host;
15571425 vm_fault_t result = VM_FAULT_FALLBACK;
15581426 struct iomap iomap = { 0 };
1559− pgoff_t max_pgoff, pgoff;
1427+ pgoff_t max_pgoff;
15601428 void *entry;
15611429 loff_t pos;
15621430 int error;
@@ -1567,7 +1435,6 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 4 unmodified lines
15671435 * supposed to hold locks serializing us with truncate / punch hole so
15681436 * this is a reliable test.
15691437 */
1570− pgoff = linear_page_index(vma, pmd_addr);
15711438 max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
15721439
15731440 trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
@@ -1576,7 +1443,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 2 unmodified lines
15761443 * Make sure that the faulting address's PMD offset (color) matches
15771444 * the PMD offset from the start of the file. This is necessary so
15781445 * that a PMD range in the page table overlaps exactly with a PMD
1579− * range in the radix tree.
1446+ * range in the page cache.
15801447 */
15811448 if ((vmf->pgoff & PG_PMD_COLOUR) !=
15821449 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
@@ -1592,24 +1459,26 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 9 unmodified lines
15921459 if ((pmd_addr + PMD_SIZE) > vma->vm_end)
15931460 goto fallback;
15941461
1595− if (pgoff >= max_pgoff) {
1462+ if (xas.xa_index >= max_pgoff) {
15961463 result = VM_FAULT_SIGBUS;
15971464 goto out;
15981465 }
15991466
16001467 /* If the PMD would extend beyond the file size */
1601− if ((pgoff | PG_PMD_COLOUR) >= max_pgoff)
1468+ if ((xas.xa_index | PG_PMD_COLOUR) >= max_pgoff)
16021469 goto fallback;
16031470
16041471 /*
1605− * grab_mapping_entry() will make sure we get a 2MiB empty entry, a
1606− * 2MiB zero page entry or a DAX PMD. If it can't (because a 4k page
1607− * is already in the tree, for instance), it will return -EEXIST and
1608− * we just fall back to 4k entries.
1472+ * grab_mapping_entry() will make sure we get an empty PMD entry,
1473+ * a zero PMD entry or a DAX PMD. If it can't (because a PTE
1474+ * entry is already in the array, for instance), it will return
1475+ * VM_FAULT_FALLBACK.
16091476 */
1610− entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
1611− if (IS_ERR(entry))
1477+ entry = grab_mapping_entry(&xas, mapping, DAX_PMD);
1478+ if (xa_is_internal(entry)) {
1479+ result = xa_to_internal(entry);
16121480 goto fallback;
1481+ }
16131482
16141483 /*
16151484 * It is possible, particularly with mixed reads & writes to private
@@ -1628,7 +1497,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 12 unmodified lines
16281497 * setting up a mapping, so really we're using iomap_begin() as a way
16291498 * to look up our filesystem block.
16301499 */
1631− pos = (loff_t)pgoff << PAGE_SHIFT;
1500+ pos = (loff_t)xas.xa_index << PAGE_SHIFT;
16321501 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
16331502 if (error)
16341503 goto unlock_entry;
@@ -1644,8 +1513,8 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 9 unmodified lines
16441513 if (error < 0)
16451514 goto finish_iomap;
16461515
1647− entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn,
1648− RADIX_DAX_PMD, write && !sync);
1516+ entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
1517+ DAX_PMD, write && !sync);
16491518
16501519 /*
16511520 * If we are doing synchronous page fault and inode needs fsync,
@@ -1669,7 +1538,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 17 unmodified lines
16691538 case IOMAP_HOLE:
16701539 if (WARN_ON_ONCE(write))
16711540 break;
1672− result = dax_pmd_load_hole(vmf, &iomap, entry);
1541+ result = dax_pmd_load_hole(&xas, vmf, &iomap, &entry);
16731542 break;
16741543 default:
16751544 WARN_ON_ONCE(1);
@@ -1692,7 +1561,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, · 16 unmodified lines
16921561 &iomap);
16931562 }
16941563 unlock_entry:
1695− put_locked_mapping_entry(mapping, pgoff);
1564+ dax_unlock_entry(&xas, entry);
16961565 fallback:
16971566 if (result == VM_FAULT_FALLBACK) {
16981567 split_huge_pmd(vma, vmf->pmd, vmf->address);
@@ -1737,54 +1606,49 @@ vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size, · 38 unmodified lines
17371606 }
17381607 EXPORT_SYMBOL_GPL(dax_iomap_fault);
17391608
1740−/**
1609+/*
17411610 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
17421611 * @vmf: The description of the fault
1743− * @pe_size: Size of entry to be inserted
17441612 * @pfn: PFN to insert
1613+ * @order: Order of entry to insert.
17451614 *
1746− * This function inserts writeable PTE or PMD entry into page tables for mmaped
1747− * DAX file. It takes care of marking corresponding radix tree entry as dirty
1748− * as well.
1615+ * This function inserts a writeable PTE or PMD entry into the page tables
1616+ * for an mmaped DAX file. It also marks the page cache entry as dirty.
17491617 */
1750−static vm_fault_t dax_insert_pfn_mkwrite(struct vm_fault *vmf,
1751− enum page_entry_size pe_size,
1752− pfn_t pfn)
1618+static vm_fault_t
1619+dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)
17531620 {
17541621 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1755− void *entry, **slot;
1756− pgoff_t index = vmf->pgoff;
1622+ XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);
1623+ void *entry;
17571624 vm_fault_t ret;
17581625
1759− xa_lock_irq(&mapping->i_pages);
1760− entry = get_unlocked_mapping_entry(mapping, index, &slot);
1626+ xas_lock_irq(&xas);
1627+ entry = get_unlocked_entry(&xas);
17611628 /* Did we race with someone splitting entry or so? */
17621629 if (!entry ||
1763− (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) ||
1764− (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) {
1765− put_unlocked_mapping_entry(mapping, index, entry);
1766− xa_unlock_irq(&mapping->i_pages);
1630+ (order == 0 && !dax_is_pte_entry(entry)) ||
1631+ (order == PMD_ORDER && (xa_is_internal(entry) ||
1632+ !dax_is_pmd_entry(entry)))) {
1633+ put_unlocked_entry(&xas, entry);
1634+ xas_unlock_irq(&xas);
17671635 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
17681636 VM_FAULT_NOPAGE);
17691637 return VM_FAULT_NOPAGE;
17701638 }
1771− radix_tree_tag_set(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY);
1772− entry = lock_slot(mapping, slot);
1773− xa_unlock_irq(&mapping->i_pages);
1774− switch (pe_size) {
1775− case PE_SIZE_PTE:
1639+ xas_set_mark(&xas, PAGECACHE_TAG_DIRTY);
1640+ dax_lock_entry(&xas, entry);
1641+ xas_unlock_irq(&xas);
1642+ if (order == 0)
17761643 ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
1777− break;
17781644 #ifdef CONFIG_FS_DAX_PMD
1779− case PE_SIZE_PMD:
1645+ else if (order == PMD_ORDER)
17801646 ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
17811647 pfn, true);
1782− break;
17831648 #endif
1784− default:
1649+ else
17851650 ret = VM_FAULT_FALLBACK;
1786− }
1787− put_locked_mapping_entry(mapping, index);
1651+ dax_unlock_entry(&xas, entry);
17881652 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
17891653 return ret;
17901654 }
@@ -1804,17 +1668,12 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, · 13 unmodified lines
18041668 {
18051669 int err;
18061670 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
1807− size_t len = 0;
1671+ unsigned int order = pe_order(pe_size);
1672+ size_t len = PAGE_SIZE << order;
18081673
1809− if (pe_size == PE_SIZE_PTE)
1810− len = PAGE_SIZE;
1811− else if (pe_size == PE_SIZE_PMD)
1812− len = PMD_SIZE;
1813− else
1814− WARN_ON_ONCE(1);
18151674 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
18161675 if (err)
18171676 return VM_FAULT_SIGBUS;
1818− return dax_insert_pfn_mkwrite(vmf, pe_size, pfn);
1677+ return dax_insert_pfn_mkwrite(vmf, pfn, order);
18191678 }
18201679 EXPORT_SYMBOL_GPL(dax_finish_sync_fault);
fs/ext4/inode.c+1−1
@@ -2643,7 +2643,7 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) · 2642 unmodified lines
26432643 long left = mpd->wbc->nr_to_write;
26442644 pgoff_t index = mpd->first_page;
26452645 pgoff_t end = mpd->last_page;
2646− int tag;
2646+ xa_mark_t tag;
26472647 int i, err = 0;
26482648 int blkbits = mpd->inode->i_blkbits;
26492649 ext4_lblk_t lblk;
fs/f2fs/data.c+3−3
@@ -2071,7 +2071,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping, · 2070 unmodified lines
20712071 pgoff_t done_index;
20722072 int cycled;
20732073 int range_whole = 0;
2074− int tag;
2074+ xa_mark_t tag;
20752075 int nwritten = 0;
20762076
20772077 pagevec_init(&pvec);
@@ -2787,13 +2787,13 @@ const struct address_space_operations f2fs_dblock_aops = { · 709 unmodified lines
27872787 #endif
27882788 };
27892789
2790−void f2fs_clear_radix_tree_dirty_tag(struct page *page)
2790+void f2fs_clear_page_cache_dirty_tag(struct page *page)
27912791 {
27922792 struct address_space *mapping = page_mapping(page);
27932793 unsigned long flags;
27942794
27952795 xa_lock_irqsave(&mapping->i_pages, flags);
2796− radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2796+ __xa_clear_mark(&mapping->i_pages, page_index(page),
27972797 PAGECACHE_TAG_DIRTY);
27982798 xa_unlock_irqrestore(&mapping->i_pages, flags);
27992799 }
fs/f2fs/dir.c+1−1
@@ -726,7 +726,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, · 725 unmodified lines
726726
727727 if (bit_pos == NR_DENTRY_IN_BLOCK &&
728728 !f2fs_truncate_hole(dir, page->index, page->index + 1)) {
729− f2fs_clear_radix_tree_dirty_tag(page);
729+ f2fs_clear_page_cache_dirty_tag(page);
730730 clear_page_dirty_for_io(page);
731731 ClearPagePrivate(page);
732732 ClearPageUptodate(page);
fs/f2fs/f2fs.h+1−1
@@ -3108,7 +3108,7 @@ int f2fs_migrate_page(struct address_space *mapping, struct page *newpage, · 3107 unmodified lines
31083108 struct page *page, enum migrate_mode mode);
31093109 #endif
31103110 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len);
3111−void f2fs_clear_radix_tree_dirty_tag(struct page *page);
3111+void f2fs_clear_page_cache_dirty_tag(struct page *page);
31123112
31133113 /*
31143114 * gc.c
fs/f2fs/inline.c+1−1
@@ -243,7 +243,7 @@ int f2fs_write_inline_data(struct inode *inode, struct page *page) · 242 unmodified lines
243243 kunmap_atomic(src_addr);
244244 set_page_dirty(dn.inode_page);
245245
246− f2fs_clear_radix_tree_dirty_tag(page);
246+ f2fs_clear_page_cache_dirty_tag(page);
247247
248248 set_inode_flag(inode, FI_APPEND_WRITE);
249249 set_inode_flag(inode, FI_DATA_EXIST);
fs/f2fs/node.c+2−4
@@ -101,7 +101,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) · 100 unmodified lines
101101 static void clear_node_page_dirty(struct page *page)
102102 {
103103 if (PageDirty(page)) {
104− f2fs_clear_radix_tree_dirty_tag(page);
104+ f2fs_clear_page_cache_dirty_tag(page);
105105 clear_page_dirty_for_io(page);
106106 dec_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES);
107107 }
@@ -1306,9 +1306,7 @@ void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) · 1198 unmodified lines
13061306 if (f2fs_check_nid_range(sbi, nid))
13071307 return;
13081308
1309− rcu_read_lock();
1310− apage = radix_tree_lookup(&NODE_MAPPING(sbi)->i_pages, nid);
1311− rcu_read_unlock();
1309+ apage = xa_load(&NODE_MAPPING(sbi)->i_pages, nid);
13121310 if (apage)
13131311 return;
13141312
fs/fs-writeback.c+9−16
@@ -339,9 +339,9 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) · 338 unmodified lines
339339 struct address_space *mapping = inode->i_mapping;
340340 struct bdi_writeback *old_wb = inode->i_wb;
341341 struct bdi_writeback *new_wb = isw->new_wb;
342− struct radix_tree_iter iter;
342+ XA_STATE(xas, &mapping->i_pages, 0);
343+ struct page *page;
343344 bool switched = false;
344− void **slot;
345345
346346 /*
347347 * By the time control reaches here, RCU grace period has passed
@@ -375,25 +375,18 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) · 27 unmodified lines
375375 * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to
376376 * pages actually under writeback.
377377 */
378− radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0,
379− PAGECACHE_TAG_DIRTY) {
380− struct page *page = radix_tree_deref_slot_protected(slot,
381− &mapping->i_pages.xa_lock);
382− if (likely(page) && PageDirty(page)) {
378+ xas_for_each_marked(&xas, page, ULONG_MAX, PAGECACHE_TAG_DIRTY) {
379+ if (PageDirty(page)) {
383380 dec_wb_stat(old_wb, WB_RECLAIMABLE);
384381 inc_wb_stat(new_wb, WB_RECLAIMABLE);
385382 }
386383 }
387384
388− radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0,
389− PAGECACHE_TAG_WRITEBACK) {
390− struct page *page = radix_tree_deref_slot_protected(slot,
391− &mapping->i_pages.xa_lock);
392− if (likely(page)) {
393− WARN_ON_ONCE(!PageWriteback(page));
394− dec_wb_stat(old_wb, WB_WRITEBACK);
395− inc_wb_stat(new_wb, WB_WRITEBACK);
396− }
385+ xas_set(&xas, 0);
386+ xas_for_each_marked(&xas, page, ULONG_MAX, PAGECACHE_TAG_WRITEBACK) {
387+ WARN_ON_ONCE(!PageWriteback(page));
388+ dec_wb_stat(old_wb, WB_WRITEBACK);
389+ inc_wb_stat(new_wb, WB_WRITEBACK);
397390 }
398391
399392 wb_get(new_wb);
fs/gfs2/aops.c+1−1
@@ -366,7 +366,7 @@ static int gfs2_write_cache_jdata(struct address_space *mapping, · 365 unmodified lines
366366 pgoff_t done_index;
367367 int cycled;
368368 int range_whole = 0;
369− int tag;
369+ xa_mark_t tag;
370370
371371 pagevec_init(&pvec);
372372 if (wbc->range_cyclic) {
fs/inode.c+1−1
@@ -349,7 +349,7 @@ EXPORT_SYMBOL(inc_nlink); · 348 unmodified lines
349349
350350 static void __address_space_init_once(struct address_space *mapping)
351351 {
352− INIT_RADIX_TREE(&mapping->i_pages, GFP_ATOMIC | __GFP_ACCOUNT);
352+ xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ);
353353 init_rwsem(&mapping->i_mmap_rwsem);
354354 INIT_LIST_HEAD(&mapping->private_list);
355355 spin_lock_init(&mapping->private_lock);
fs/isofs/dir.c+1−1
@@ -46,7 +46,7 @@ int isofs_name_translate(struct iso_directory_record *de, char *new, struct inod · 45 unmodified lines
4646 return i;
4747 }
4848
49−/* Acorn extensions written by Matthew Wilcox <willy@bofh.ai> 1998 */
49+/* Acorn extensions written by Matthew Wilcox <willy@infradead.org> 1998 */
5050 int get_acorn_filename(struct iso_directory_record *de,
5151 char *retname, struct inode *inode)
5252 {
fs/nfs/blocklayout/blocklayout.c+1−1
@@ -896,7 +896,7 @@ static u64 pnfs_num_cont_bytes(struct inode *inode, pgoff_t idx) · 895 unmodified lines
896896 end = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
897897 if (end != inode->i_mapping->nrpages) {
898898 rcu_read_lock();
899− end = page_cache_next_hole(mapping, idx + 1, ULONG_MAX);
899+ end = page_cache_next_miss(mapping, idx + 1, ULONG_MAX);
900900 rcu_read_unlock();
901901 }
902902
fs/nilfs2/btnode.c+9−17
@@ -168,24 +168,18 @@ int nilfs_btnode_prepare_change_key(struct address_space *btnc, · 167 unmodified lines
168168 ctxt->newbh = NULL;
169169
170170 if (inode->i_blkbits == PAGE_SHIFT) {
171− lock_page(obh->b_page);
172− /*
173− * We cannot call radix_tree_preload for the kernels older
174− * than 2.6.23, because it is not exported for modules.
175− */
171+ struct page *opage = obh->b_page;
172+ lock_page(opage);
176173 retry:
177− err = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
178− if (err)
179− goto failed_unlock;
180174 /* BUG_ON(oldkey != obh->b_page->index); */
181− if (unlikely(oldkey != obh->b_page->index))
182− NILFS_PAGE_BUG(obh->b_page,
175+ if (unlikely(oldkey != opage->index))
176+ NILFS_PAGE_BUG(opage,
183177 "invalid oldkey %lld (newkey=%lld)",
184178 (unsigned long long)oldkey,
185179 (unsigned long long)newkey);
186180
187181 xa_lock_irq(&btnc->i_pages);
188− err = radix_tree_insert(&btnc->i_pages, newkey, obh->b_page);
182+ err = __xa_insert(&btnc->i_pages, newkey, opage, GFP_NOFS);
189183 xa_unlock_irq(&btnc->i_pages);
190184 /*
191185 * Note: page->index will not change to newkey until
@@ -193,7 +187,6 @@ retry: · 1 unmodified line
193187 * To protect the page in intermediate state, the page lock
194188 * is held.
195189 */
196− radix_tree_preload_end();
197190 if (!err)
198191 return 0;
199192 else if (err != -EEXIST)
@@ -203,7 +196,7 @@ retry: · 3 unmodified lines
203196 if (!err)
204197 goto retry;
205198 /* fallback to copy mode */
206− unlock_page(obh->b_page);
199+ unlock_page(opage);
207200 }
208201
209202 nbh = nilfs_btnode_create_block(btnc, newkey);
@@ -243,9 +236,8 @@ void nilfs_btnode_commit_change_key(struct address_space *btnc, · 33 unmodified lines
243236 mark_buffer_dirty(obh);
244237
245238 xa_lock_irq(&btnc->i_pages);
246− radix_tree_delete(&btnc->i_pages, oldkey);
247− radix_tree_tag_set(&btnc->i_pages, newkey,
248− PAGECACHE_TAG_DIRTY);
239+ __xa_erase(&btnc->i_pages, oldkey);
240+ __xa_set_mark(&btnc->i_pages, newkey, PAGECACHE_TAG_DIRTY);
249241 xa_unlock_irq(&btnc->i_pages);
250242
251243 opage->index = obh->b_blocknr = newkey;
@@ -275,7 +267,7 @@ void nilfs_btnode_abort_change_key(struct address_space *btnc, · 23 unmodified lines
275267
276268 if (nbh == NULL) { /* blocksize == pagesize */
277269 xa_lock_irq(&btnc->i_pages);
278− radix_tree_delete(&btnc->i_pages, newkey);
270+ __xa_erase(&btnc->i_pages, newkey);
279271 xa_unlock_irq(&btnc->i_pages);
280272 unlock_page(ctxt->bh->b_page);
281273 } else
fs/nilfs2/page.c+13−16
@@ -289,7 +289,7 @@ repeat: · 288 unmodified lines
289289 * @dmap: destination page cache
290290 * @smap: source page cache
291291 *
292− * No pages must no be added to the cache during this process.
292+ * No pages must be added to the cache during this process.
293293 * This must be ensured by the caller.
294294 */
295295 void nilfs_copy_back_pages(struct address_space *dmap,
@@ -298,7 +298,6 @@ void nilfs_copy_back_pages(struct address_space *dmap, · 2 unmodified lines
298298 struct pagevec pvec;
299299 unsigned int i, n;
300300 pgoff_t index = 0;
301− int err;
302301
303302 pagevec_init(&pvec);
304303 repeat:
@@ -313,35 +312,34 @@ repeat: · 8 unmodified lines
313312 lock_page(page);
314313 dpage = find_lock_page(dmap, offset);
315314 if (dpage) {
316− /* override existing page on the destination cache */
315+ /* overwrite existing page in the destination cache */
317316 WARN_ON(PageDirty(dpage));
318317 nilfs_copy_page(dpage, page, 0);
319318 unlock_page(dpage);
320319 put_page(dpage);
320+ /* Do we not need to remove page from smap here? */
321321 } else {
322− struct page *page2;
322+ struct page *p;
323323
324324 /* move the page to the destination cache */
325325 xa_lock_irq(&smap->i_pages);
326− page2 = radix_tree_delete(&smap->i_pages, offset);
327− WARN_ON(page2 != page);
328−
326+ p = __xa_erase(&smap->i_pages, offset);
327+ WARN_ON(page != p);
329328 smap->nrpages--;
330329 xa_unlock_irq(&smap->i_pages);
331330
332331 xa_lock_irq(&dmap->i_pages);
333− err = radix_tree_insert(&dmap->i_pages, offset, page);
334− if (unlikely(err < 0)) {
335− WARN_ON(err == -EEXIST);
332+ p = __xa_store(&dmap->i_pages, offset, page, GFP_NOFS);
333+ if (unlikely(p)) {
334+ /* Probably -ENOMEM */
336335 page->mapping = NULL;
337− put_page(page); /* for cache */
336+ put_page(page);
338337 } else {
339338 page->mapping = dmap;
340339 dmap->nrpages++;
341340 if (PageDirty(page))
342− radix_tree_tag_set(&dmap->i_pages,
343− offset,
344− PAGECACHE_TAG_DIRTY);
341+ __xa_set_mark(&dmap->i_pages, offset,
342+ PAGECACHE_TAG_DIRTY);
345343 }
346344 xa_unlock_irq(&dmap->i_pages);
347345 }
@@ -467,8 +465,7 @@ int __nilfs_clear_page_dirty(struct page *page) · 119 unmodified lines
467465 if (mapping) {
468466 xa_lock_irq(&mapping->i_pages);
469467 if (test_bit(PG_dirty, &page->flags)) {
470− radix_tree_tag_clear(&mapping->i_pages,
471− page_index(page),
468+ __xa_clear_mark(&mapping->i_pages, page_index(page),
472469 PAGECACHE_TAG_DIRTY);
473470 xa_unlock_irq(&mapping->i_pages);
474471 return clear_page_dirty_for_io(page);
fs/proc/task_mmu.c+1−1
@@ -521,7 +521,7 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, · 520 unmodified lines
521521 if (!page)
522522 return;
523523
524− if (radix_tree_exceptional_entry(page))
524+ if (xa_is_value(page))
525525 mss->swap += PAGE_SIZE;
526526 else
527527 put_page(page);
include/linux/fs.h+41−22
@@ -403,24 +403,40 @@ int pagecache_write_end(struct file *, struct address_space *mapping, · 402 unmodified lines
403403 loff_t pos, unsigned len, unsigned copied,
404404 struct page *page, void *fsdata);
405405
406+/**
407+ * struct address_space - Contents of a cacheable, mappable object.
408+ * @host: Owner, either the inode or the block_device.
409+ * @i_pages: Cached pages.
410+ * @gfp_mask: Memory allocation flags to use for allocating pages.
411+ * @i_mmap_writable: Number of VM_SHARED mappings.
412+ * @i_mmap: Tree of private and shared mappings.
413+ * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable.
414+ * @nrpages: Number of page entries, protected by the i_pages lock.
415+ * @nrexceptional: Shadow or DAX entries, protected by the i_pages lock.
416+ * @writeback_index: Writeback starts here.
417+ * @a_ops: Methods.
418+ * @flags: Error bits and flags (AS_*).
419+ * @wb_err: The most recent error which has occurred.
420+ * @private_lock: For use by the owner of the address_space.
421+ * @private_list: For use by the owner of the address_space.
422+ * @private_data: For use by the owner of the address_space.
423+ */
406424 struct address_space {
407− struct inode *host; /* owner: inode, block_device */
408− struct radix_tree_root i_pages; /* cached pages */
409− atomic_t i_mmap_writable;/* count VM_SHARED mappings */
410− struct rb_root_cached i_mmap; /* tree of private and shared mappings */
411− struct rw_semaphore i_mmap_rwsem; /* protect tree, count, list */
412− /* Protected by the i_pages lock */
413− unsigned long nrpages; /* number of total pages */
414− /* number of shadow or DAX exceptional entries */
425+ struct inode *host;
426+ struct xarray i_pages;
427+ gfp_t gfp_mask;
428+ atomic_t i_mmap_writable;
429+ struct rb_root_cached i_mmap;
430+ struct rw_semaphore i_mmap_rwsem;
431+ unsigned long nrpages;
415432 unsigned long nrexceptional;
416− pgoff_t writeback_index;/* writeback starts here */
417− const struct address_space_operations *a_ops; /* methods */
418− unsigned long flags; /* error bits */
419− spinlock_t private_lock; /* for use by the address_space */
420− gfp_t gfp_mask; /* implicit gfp mask for allocations */
421− struct list_head private_list; /* for use by the address_space */
422− void *private_data; /* ditto */
433+ pgoff_t writeback_index;
434+ const struct address_space_operations *a_ops;
435+ unsigned long flags;
423436 errseq_t wb_err;
437+ spinlock_t private_lock;
438+ struct list_head private_list;
439+ void *private_data;
424440 } __attribute__((aligned(sizeof(long)))) __randomize_layout;
425441 /*
426442 * On most architectures that alignment is already the case; but
@@ -467,15 +483,18 @@ struct block_device { · 40 unmodified lines
467483 struct mutex bd_fsfreeze_mutex;
468484 } __randomize_layout;
469485
486+/* XArray tags, for tagging dirty and writeback pages in the pagecache. */
487+#define PAGECACHE_TAG_DIRTY XA_MARK_0
488+#define PAGECACHE_TAG_WRITEBACK XA_MARK_1
489+#define PAGECACHE_TAG_TOWRITE XA_MARK_2
490+
470491 /*
471− * Radix-tree tags, for tagging dirty and writeback pages within the pagecache
472− * radix trees
492+ * Returns true if any of the pages in the mapping are marked with the tag.
473493 */
474−#define PAGECACHE_TAG_DIRTY 0
475−#define PAGECACHE_TAG_WRITEBACK 1
476−#define PAGECACHE_TAG_TOWRITE 2
477−
478−int mapping_tagged(struct address_space *mapping, int tag);
494+static inline bool mapping_tagged(struct address_space *mapping, xa_mark_t tag)
495+{
496+ return xa_marked(&mapping->i_pages, tag);
497+}
479498
480499 static inline void i_mmap_lock_write(struct address_space *mapping)
481500 {
include/linux/idr.h+7−11
@@ -214,8 +214,7 @@ static inline void idr_preload_end(void) · 213 unmodified lines
214214 ++id, (entry) = idr_get_next((idr), &(id)))
215215
216216 /*
217− * IDA - IDR based id allocator, use when translation from id to
218− * pointer isn't necessary.
217+ * IDA - ID Allocator, use when translation from id to pointer isn't necessary.
219218 */
220219 #define IDA_CHUNK_SIZE 128 /* 128 bytes per chunk */
221220 #define IDA_BITMAP_LONGS (IDA_CHUNK_SIZE / sizeof(long))
@@ -225,14 +224,14 @@ struct ida_bitmap { · 3 unmodified lines
225224 unsigned long bitmap[IDA_BITMAP_LONGS];
226225 };
227226
228−DECLARE_PER_CPU(struct ida_bitmap *, ida_bitmap);
229−
230227 struct ida {
231− struct radix_tree_root ida_rt;
228+ struct xarray xa;
232229 };
233230
231+#define IDA_INIT_FLAGS (XA_FLAGS_LOCK_IRQ | XA_FLAGS_ALLOC)
232+
234233 #define IDA_INIT(name) { \
235− .ida_rt = RADIX_TREE_INIT(name, IDR_RT_MARKER | GFP_NOWAIT), \
234+ .xa = XARRAY_INIT(name, IDA_INIT_FLAGS) \
236235 }
237236 #define DEFINE_IDA(name) struct ida name = IDA_INIT(name)
238237
@@ -292,7 +291,7 @@ static inline int ida_alloc_max(struct ida *ida, unsigned int max, gfp_t gfp) · 53 unmodified lines
292291
293292 static inline void ida_init(struct ida *ida)
294293 {
295− INIT_RADIX_TREE(&ida->ida_rt, IDR_RT_MARKER | GFP_NOWAIT);
294+ xa_init_flags(&ida->xa, IDA_INIT_FLAGS);
296295 }
297296
298297 #define ida_simple_get(ida, start, end, gfp) \
@@ -301,9 +300,6 @@ static inline void ida_init(struct ida *ida) · 2 unmodified lines
301300
302301 static inline bool ida_is_empty(const struct ida *ida)
303302 {
304− return radix_tree_empty(&ida->ida_rt);
303+ return xa_empty(&ida->xa);
305304 }
306−
307−/* in lib/radix-tree.c */
308−int ida_pre_get(struct ida *ida, gfp_t gfp_mask);
309305 #endif /* __IDR_H__ */
include/linux/pagemap.h+5−5
@@ -241,9 +241,9 @@ static inline gfp_t readahead_gfp_mask(struct address_space *x) · 240 unmodified lines
241241
242242 typedef int filler_t(void *, struct page *);
243243
244−pgoff_t page_cache_next_hole(struct address_space *mapping,
244+pgoff_t page_cache_next_miss(struct address_space *mapping,
245245 pgoff_t index, unsigned long max_scan);
246−pgoff_t page_cache_prev_hole(struct address_space *mapping,
246+pgoff_t page_cache_prev_miss(struct address_space *mapping,
247247 pgoff_t index, unsigned long max_scan);
248248
249249 #define FGP_ACCESSED 0x00000001
@@ -363,17 +363,17 @@ static inline unsigned find_get_pages(struct address_space *mapping, · 113 unmodified lines
363363 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start,
364364 unsigned int nr_pages, struct page **pages);
365365 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
366− pgoff_t end, int tag, unsigned int nr_pages,
366+ pgoff_t end, xa_mark_t tag, unsigned int nr_pages,
367367 struct page **pages);
368368 static inline unsigned find_get_pages_tag(struct address_space *mapping,
369− pgoff_t *index, int tag, unsigned int nr_pages,
369+ pgoff_t *index, xa_mark_t tag, unsigned int nr_pages,
370370 struct page **pages)
371371 {
372372 return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag,
373373 nr_pages, pages);
374374 }
375375 unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
376− int tag, unsigned int nr_entries,
376+ xa_mark_t tag, unsigned int nr_entries,
377377 struct page **entries, pgoff_t *indices);
378378
379379 struct page *grab_cache_page_write_begin(struct address_space *mapping,
include/linux/pagevec.h+5−3
@@ -9,6 +9,8 @@ · 8 unmodified lines
99 #ifndef _LINUX_PAGEVEC_H
1010 #define _LINUX_PAGEVEC_H
1111
12+#include <linux/xarray.h>
13+
1214 /* 15 pointers + header align the pagevec structure to a power of two */
1315 #define PAGEVEC_SIZE 15
1416
@@ -40,12 +42,12 @@ static inline unsigned pagevec_lookup(struct pagevec *pvec, · 25 unmodified lines
4042
4143 unsigned pagevec_lookup_range_tag(struct pagevec *pvec,
4244 struct address_space *mapping, pgoff_t *index, pgoff_t end,
43− int tag);
45+ xa_mark_t tag);
4446 unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec,
4547 struct address_space *mapping, pgoff_t *index, pgoff_t end,
46− int tag, unsigned max_pages);
48+ xa_mark_t tag, unsigned max_pages);
4749 static inline unsigned pagevec_lookup_tag(struct pagevec *pvec,
48− struct address_space *mapping, pgoff_t *index, int tag)
50+ struct address_space *mapping, pgoff_t *index, xa_mark_t tag)
4951 {
5052 return pagevec_lookup_range_tag(pvec, mapping, index, (pgoff_t)-1, tag);
5153 }
include/linux/radix-tree.h+24−154
@@ -28,34 +28,30 @@ · 27 unmodified lines
2828 #include <linux/rcupdate.h>
2929 #include <linux/spinlock.h>
3030 #include <linux/types.h>
31+#include <linux/xarray.h>
32+
33+/* Keep unconverted code working */
34+#define radix_tree_root xarray
35+#define radix_tree_node xa_node
3136
3237 /*
3338 * The bottom two bits of the slot determine how the remaining bits in the
3439 * slot are interpreted:
3540 *
3641 * 00 - data pointer
37− * 01 - internal entry
38− * 10 - exceptional entry
39− * 11 - this bit combination is currently unused/reserved
42+ * 10 - internal entry
43+ * x1 - value entry
4044 *
4145 * The internal entry may be a pointer to the next level in the tree, a
4246 * sibling entry, or an indicator that the entry in this slot has been moved
4347 * to another location in the tree and the lookup should be restarted. While
4448 * NULL fits the 'data pointer' pattern, it means that there is no entry in
4549 * the tree for this index (no matter what level of the tree it is found at).
46− * This means that you cannot store NULL in the tree as a value for the index.
50+ * This means that storing a NULL entry in the tree is the same as deleting
51+ * the entry from the tree.
4752 */
4853 #define RADIX_TREE_ENTRY_MASK 3UL
49−#define RADIX_TREE_INTERNAL_NODE 1UL
50−
51−/*
52− * Most users of the radix tree store pointers but shmem/tmpfs stores swap
53− * entries in the same tree. They are marked as exceptional entries to
54− * distinguish them from pointers to struct page.
55− * EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it.
56− */
57−#define RADIX_TREE_EXCEPTIONAL_ENTRY 2
58−#define RADIX_TREE_EXCEPTIONAL_SHIFT 2
54+#define RADIX_TREE_INTERNAL_NODE 2UL
5955
6056 static inline bool radix_tree_is_internal_node(void *ptr)
6157 {
@@ -65,75 +61,32 @@ static inline bool radix_tree_is_internal_node(void *ptr) · 3 unmodified lines
6561
6662 /*** radix-tree API starts here ***/
6763
68−#define RADIX_TREE_MAX_TAGS 3
69−
70−#ifndef RADIX_TREE_MAP_SHIFT
71−#define RADIX_TREE_MAP_SHIFT (CONFIG_BASE_SMALL ? 4 : 6)
72−#endif
73−
64+#define RADIX_TREE_MAP_SHIFT XA_CHUNK_SHIFT
7465 #define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
7566 #define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1)
7667
77−#define RADIX_TREE_TAG_LONGS \
78− ((RADIX_TREE_MAP_SIZE + BITS_PER_LONG - 1) / BITS_PER_LONG)
68+#define RADIX_TREE_MAX_TAGS XA_MAX_MARKS
69+#define RADIX_TREE_TAG_LONGS XA_MARK_LONGS
7970
8071 #define RADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long))
8172 #define RADIX_TREE_MAX_PATH (DIV_ROUND_UP(RADIX_TREE_INDEX_BITS, \
8273 RADIX_TREE_MAP_SHIFT))
8374
84−/*
85− * @count is the count of every non-NULL element in the ->slots array
86− * whether that is an exceptional entry, a retry entry, a user pointer,
87− * a sibling entry or a pointer to the next level of the tree.
88− * @exceptional is the count of every element in ->slots which is
89− * either radix_tree_exceptional_entry() or is a sibling entry for an
90− * exceptional entry.
91− */
92−struct radix_tree_node {
93− unsigned char shift; /* Bits remaining in each slot */
94− unsigned char offset; /* Slot offset in parent */
95− unsigned char count; /* Total entry count */
96− unsigned char exceptional; /* Exceptional entry count */
97− struct radix_tree_node *parent; /* Used when ascending tree */
98− struct radix_tree_root *root; /* The tree we belong to */
99− union {
100− struct list_head private_list; /* For tree user */
101− struct rcu_head rcu_head; /* Used when freeing node */
102− };
103− void __rcu *slots[RADIX_TREE_MAP_SIZE];
104− unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS];
105−};
106−
107−/* The IDR tag is stored in the low bits of the GFP flags */
75+/* The IDR tag is stored in the low bits of xa_flags */
10876 #define ROOT_IS_IDR ((__force gfp_t)4)
109−/* The top bits of gfp_mask are used to store the root tags */
77+/* The top bits of xa_flags are used to store the root tags */
11078 #define ROOT_TAG_SHIFT (__GFP_BITS_SHIFT)
11179
112−struct radix_tree_root {
113− spinlock_t xa_lock;
114− gfp_t gfp_mask;
115− struct radix_tree_node __rcu *rnode;
116−};
117−
118−#define RADIX_TREE_INIT(name, mask) { \
119− .xa_lock = __SPIN_LOCK_UNLOCKED(name.xa_lock), \
120− .gfp_mask = (mask), \
121− .rnode = NULL, \
122−}
80+#define RADIX_TREE_INIT(name, mask) XARRAY_INIT(name, mask)
12381
12482 #define RADIX_TREE(name, mask) \
12583 struct radix_tree_root name = RADIX_TREE_INIT(name, mask)
12684
127−#define INIT_RADIX_TREE(root, mask) \
128−do { \
129− spin_lock_init(&(root)->xa_lock); \
130− (root)->gfp_mask = (mask); \
131− (root)->rnode = NULL; \
132−} while (0)
85+#define INIT_RADIX_TREE(root, mask) xa_init_flags(root, mask)
13386
13487 static inline bool radix_tree_empty(const struct radix_tree_root *root)
13588 {
136− return root->rnode == NULL;
89+ return root->xa_head == NULL;
13790 }
13891
13992 /**
@@ -143,7 +96,6 @@ static inline bool radix_tree_empty(const struct radix_tree_root *root) · 3 unmodified lines
14396 * @next_index: one beyond the last index for this chunk
14497 * @tags: bit-mask for tag-iterating
14598 * @node: node that contains current slot
146− * @shift: shift for the node that holds our slots
14799 *
148100 * This radix tree iterator works in terms of "chunks" of slots. A chunk is a
149101 * subinterval of slots contained within one radix tree leaf node. It is
@@ -157,20 +109,8 @@ struct radix_tree_iter { · 7 unmodified lines
157109 unsigned long next_index;
158110 unsigned long tags;
159111 struct radix_tree_node *node;
160−#ifdef CONFIG_RADIX_TREE_MULTIORDER
161− unsigned int shift;
162−#endif
163112 };
164113
165−static inline unsigned int iter_shift(const struct radix_tree_iter *iter)
166−{
167−#ifdef CONFIG_RADIX_TREE_MULTIORDER
168− return iter->shift;
169−#else
170− return 0;
171−#endif
172−}
173−
174114 /**
175115 * Radix-tree synchronization
176116 *
@@ -194,12 +134,11 @@ static inline unsigned int iter_shift(const struct radix_tree_iter *iter) · 17 unmodified lines
194134 * radix_tree_lookup_slot
195135 * radix_tree_tag_get
196136 * radix_tree_gang_lookup
197− * radix_tree_gang_lookup_slot
198137 * radix_tree_gang_lookup_tag
199138 * radix_tree_gang_lookup_tag_slot
200139 * radix_tree_tagged
201140 *
202− * The first 8 functions are able to be called locklessly, using RCU. The
141+ * The first 7 functions are able to be called locklessly, using RCU. The
203142 * caller must ensure calls to these functions are made within rcu_read_lock()
204143 * regions. Other readers (lock-free or otherwise) and modifications may be
205144 * running concurrently.
@@ -268,17 +207,6 @@ static inline int radix_tree_deref_retry(void *arg) · 62 unmodified lines
268207 return unlikely(radix_tree_is_internal_node(arg));
269208 }
270209
271−/**
272− * radix_tree_exceptional_entry - radix_tree_deref_slot gave exceptional entry?
273− * @arg: value returned by radix_tree_deref_slot
274− * Returns: 0 if well-aligned pointer, non-0 if exceptional entry.
275− */
276−static inline int radix_tree_exceptional_entry(void *arg)
277−{
278− /* Not unlikely because radix_tree_exception often tested first */
279− return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY;
280−}
281−
282210 /**
283211 * radix_tree_exception - radix_tree_deref_slot returned either exception?
284212 * @arg: value returned by radix_tree_deref_slot
@@ -289,47 +217,28 @@ static inline int radix_tree_exception(void *arg) · 4 unmodified lines
289217 return unlikely((unsigned long)arg & RADIX_TREE_ENTRY_MASK);
290218 }
291219
292−int __radix_tree_create(struct radix_tree_root *, unsigned long index,
293− unsigned order, struct radix_tree_node **nodep,
294− void __rcu ***slotp);
295−int __radix_tree_insert(struct radix_tree_root *, unsigned long index,
296− unsigned order, void *);
297−static inline int radix_tree_insert(struct radix_tree_root *root,
298− unsigned long index, void *entry)
299−{
300− return __radix_tree_insert(root, index, 0, entry);
301−}
220+int radix_tree_insert(struct radix_tree_root *, unsigned long index,
221+ void *);
302222 void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index,
303223 struct radix_tree_node **nodep, void __rcu ***slotp);
304224 void *radix_tree_lookup(const struct radix_tree_root *, unsigned long);
305225 void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *,
306226 unsigned long index);
307−typedef void (*radix_tree_update_node_t)(struct radix_tree_node *);
308227 void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *,
309− void __rcu **slot, void *entry,
310− radix_tree_update_node_t update_node);
228+ void __rcu **slot, void *entry);
311229 void radix_tree_iter_replace(struct radix_tree_root *,
312230 const struct radix_tree_iter *, void __rcu **slot, void *entry);
313231 void radix_tree_replace_slot(struct radix_tree_root *,
314232 void __rcu **slot, void *entry);
315−void __radix_tree_delete_node(struct radix_tree_root *,
316− struct radix_tree_node *,
317− radix_tree_update_node_t update_node);
318233 void radix_tree_iter_delete(struct radix_tree_root *,
319234 struct radix_tree_iter *iter, void __rcu **slot);
320235 void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *);
321236 void *radix_tree_delete(struct radix_tree_root *, unsigned long);
322−void radix_tree_clear_tags(struct radix_tree_root *, struct radix_tree_node *,
323− void __rcu **slot);
324237 unsigned int radix_tree_gang_lookup(const struct radix_tree_root *,
325238 void **results, unsigned long first_index,
326239 unsigned int max_items);
327−unsigned int radix_tree_gang_lookup_slot(const struct radix_tree_root *,
328− void __rcu ***results, unsigned long *indices,
329− unsigned long first_index, unsigned int max_items);
330240 int radix_tree_preload(gfp_t gfp_mask);
331241 int radix_tree_maybe_preload(gfp_t gfp_mask);
332−int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order);
333242 void radix_tree_init(void);
334243 void *radix_tree_tag_set(struct radix_tree_root *,
335244 unsigned long index, unsigned int tag);
@@ -337,8 +246,6 @@ void *radix_tree_tag_clear(struct radix_tree_root *, · 1 unmodified line
337246 unsigned long index, unsigned int tag);
338247 int radix_tree_tag_get(const struct radix_tree_root *,
339248 unsigned long index, unsigned int tag);
340−void radix_tree_iter_tag_set(struct radix_tree_root *,
341− const struct radix_tree_iter *iter, unsigned int tag);
342249 void radix_tree_iter_tag_clear(struct radix_tree_root *,
343250 const struct radix_tree_iter *iter, unsigned int tag);
344251 unsigned int radix_tree_gang_lookup_tag(const struct radix_tree_root *,
@@ -354,12 +261,6 @@ static inline void radix_tree_preload_end(void) · 9 unmodified lines
354261 preempt_enable();
355262 }
356263
357−int radix_tree_split_preload(unsigned old_order, unsigned new_order, gfp_t);
358−int radix_tree_split(struct radix_tree_root *, unsigned long index,
359− unsigned new_order);
360−int radix_tree_join(struct radix_tree_root *, unsigned long index,
361− unsigned new_order, void *);
362−
363264 void __rcu **idr_get_free(struct radix_tree_root *root,
364265 struct radix_tree_iter *iter, gfp_t gfp,
365266 unsigned long max);
@@ -465,7 +366,7 @@ void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter) · 99 unmodified lines
465366 static inline unsigned long
466367 __radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots)
467368 {
468− return iter->index + (slots << iter_shift(iter));
369+ return iter->index + slots;
469370 }
470371
471372 /**
@@ -490,21 +391,9 @@ void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot, · 18 unmodified lines
490391 static __always_inline long
491392 radix_tree_chunk_size(struct radix_tree_iter *iter)
492393 {
493− return (iter->next_index - iter->index) >> iter_shift(iter);
394+ return iter->next_index - iter->index;
494395 }
495396
496−#ifdef CONFIG_RADIX_TREE_MULTIORDER
497−void __rcu **__radix_tree_next_slot(void __rcu **slot,
498− struct radix_tree_iter *iter, unsigned flags);
499−#else
500−/* Can't happen without sibling entries, but the compiler can't tell that */
501−static inline void __rcu **__radix_tree_next_slot(void __rcu **slot,
502− struct radix_tree_iter *iter, unsigned flags)
503−{
504− return slot;
505−}
506−#endif
507−
508397 /**
509398 * radix_tree_next_slot - find next slot in chunk
510399 *
@@ -563,8 +452,6 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot, · 52 unmodified lines
563452 return NULL;
564453
565454 found:
566− if (unlikely(radix_tree_is_internal_node(rcu_dereference_raw(*slot))))
567− return __radix_tree_next_slot(slot, iter, flags);
568455 return slot;
569456 }
570457
@@ -583,23 +470,6 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot, · 12 unmodified lines
583470 slot || (slot = radix_tree_next_chunk(root, iter, 0)) ; \
584471 slot = radix_tree_next_slot(slot, iter, 0))
585472
586−/**
587− * radix_tree_for_each_contig - iterate over contiguous slots
588− *
589− * @slot: the void** variable for pointer to slot
590− * @root: the struct radix_tree_root pointer
591− * @iter: the struct radix_tree_iter pointer
592− * @start: iteration starting index
593− *
594− * @slot points to radix tree slot, @iter->index contains its index.
595− */
596−#define radix_tree_for_each_contig(slot, root, iter, start) \
597− for (slot = radix_tree_iter_init(iter, start) ; \
598− slot || (slot = radix_tree_next_chunk(root, iter, \
599− RADIX_TREE_ITER_CONTIG)) ; \
600− slot = radix_tree_next_slot(slot, iter, \
601− RADIX_TREE_ITER_CONTIG))
602−
603473 /**
604474 * radix_tree_for_each_tagged - iterate over tagged slots
605475 *
include/linux/swap.h+9−13
@@ -300,17 +300,12 @@ void *workingset_eviction(struct address_space *mapping, struct page *page); · 299 unmodified lines
300300 void workingset_refault(struct page *page, void *shadow);
301301 void workingset_activation(struct page *page);
302302
303−/* Do not use directly, use workingset_lookup_update */
304−void workingset_update_node(struct radix_tree_node *node);
305−
306−/* Returns workingset_update_node() if the mapping has shadow entries. */
307−#define workingset_lookup_update(mapping) \
308−({ \
309− radix_tree_update_node_t __helper = workingset_update_node; \
310− if (dax_mapping(mapping) || shmem_mapping(mapping)) \
311− __helper = NULL; \
312− __helper; \
313−})
303+/* Only track the nodes of mappings with shadow entries */
304+void workingset_update_node(struct xa_node *node);
305+#define mapping_set_update(xas, mapping) do { \
306+ if (!dax_mapping(mapping) && !shmem_mapping(mapping)) \
307+ xas_set_update(xas, workingset_update_node); \
308+} while (0)
314309
315310 /* linux/mm/page_alloc.c */
316311 extern unsigned long totalram_pages;
@@ -409,7 +404,7 @@ extern void show_swap_cache_info(void); · 92 unmodified lines
409404 extern int add_to_swap(struct page *page);
410405 extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t);
411406 extern int __add_to_swap_cache(struct page *page, swp_entry_t entry);
412−extern void __delete_from_swap_cache(struct page *);
407+extern void __delete_from_swap_cache(struct page *, swp_entry_t entry);
413408 extern void delete_from_swap_cache(struct page *);
414409 extern void free_page_and_swap_cache(struct page *);
415410 extern void free_pages_and_swap_cache(struct page **, int);
@@ -563,7 +558,8 @@ static inline int add_to_swap_cache(struct page *page, swp_entry_t entry, · 147 unmodified lines
563558 return -1;
564559 }
565560
566−static inline void __delete_from_swap_cache(struct page *page)
561+static inline void __delete_from_swap_cache(struct page *page,
562+ swp_entry_t entry)
567563 {
568564 }
569565
include/linux/swapops.h+7−12
@@ -18,9 +18,8 @@ · 17 unmodified lines
1818 *
1919 * swp_entry_t's are *never* stored anywhere in their arch-dependent format.
2020 */
21−#define SWP_TYPE_SHIFT(e) ((sizeof(e.val) * 8) - \
22− (MAX_SWAPFILES_SHIFT + RADIX_TREE_EXCEPTIONAL_SHIFT))
23−#define SWP_OFFSET_MASK(e) ((1UL << SWP_TYPE_SHIFT(e)) - 1)
21+#define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
22+#define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1)
2423
2524 /*
2625 * Store a type+offset into a swp_entry_t in an arch-independent format
@@ -29,8 +28,7 @@ static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) · 2 unmodified lines
2928 {
3029 swp_entry_t ret;
3130
32− ret.val = (type << SWP_TYPE_SHIFT(ret)) |
33− (offset & SWP_OFFSET_MASK(ret));
31+ ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK);
3432 return ret;
3533 }
3634
@@ -40,7 +38,7 @@ static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) · 3 unmodified lines
4038 */
4139 static inline unsigned swp_type(swp_entry_t entry)
4240 {
43− return (entry.val >> SWP_TYPE_SHIFT(entry));
41+ return (entry.val >> SWP_TYPE_SHIFT);
4442 }
4543
4644 /*
@@ -49,7 +47,7 @@ static inline unsigned swp_type(swp_entry_t entry) · 2 unmodified lines
4947 */
5048 static inline pgoff_t swp_offset(swp_entry_t entry)
5149 {
52− return entry.val & SWP_OFFSET_MASK(entry);
50+ return entry.val & SWP_OFFSET_MASK;
5351 }
5452
5553 #ifdef CONFIG_MMU
@@ -90,16 +88,13 @@ static inline swp_entry_t radix_to_swp_entry(void *arg) · 34 unmodified lines
9088 {
9189 swp_entry_t entry;
9290
93− entry.val = (unsigned long)arg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
91+ entry.val = xa_to_value(arg);
9492 return entry;
9593 }
9694
9795 static inline void *swp_to_radix_entry(swp_entry_t entry)
9896 {
99− unsigned long value;
100−
101− value = entry.val << RADIX_TREE_EXCEPTIONAL_SHIFT;
102− return (void *)(value | RADIX_TREE_EXCEPTIONAL_ENTRY);
97+ return xa_mk_value(entry.val);
10398 }
10499
105100 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
include/linux/xarray.h+1292−1
@@ -4,10 +4,432 @@ · 3 unmodified lines
44 /*
55 * eXtensible Arrays
66 * Copyright (c) 2017 Microsoft Corporation
7− * Author: Matthew Wilcox <mawilcox@microsoft.com>
7+ * Author: Matthew Wilcox <willy@infradead.org>
8+ *
9+ * See Documentation/core-api/xarray.rst for how to use the XArray.
810 */
911
12+#include <linux/bug.h>
13+#include <linux/compiler.h>
14+#include <linux/gfp.h>
15+#include <linux/kconfig.h>
16+#include <linux/kernel.h>
17+#include <linux/rcupdate.h>
1018 #include <linux/spinlock.h>
19+#include <linux/types.h>
20+
21+/*
22+ * The bottom two bits of the entry determine how the XArray interprets
23+ * the contents:
24+ *
25+ * 00: Pointer entry
26+ * 10: Internal entry
27+ * x1: Value entry or tagged pointer
28+ *
29+ * Attempting to store internal entries in the XArray is a bug.
30+ *
31+ * Most internal entries are pointers to the next node in the tree.
32+ * The following internal entries have a special meaning:
33+ *
34+ * 0-62: Sibling entries
35+ * 256: Zero entry
36+ * 257: Retry entry
37+ *
38+ * Errors are also represented as internal entries, but use the negative
39+ * space (-4094 to -2). They're never stored in the slots array; only
40+ * returned by the normal API.
41+ */
42+
43+#define BITS_PER_XA_VALUE (BITS_PER_LONG - 1)
44+
45+/**
46+ * xa_mk_value() - Create an XArray entry from an integer.
47+ * @v: Value to store in XArray.
48+ *
49+ * Context: Any context.
50+ * Return: An entry suitable for storing in the XArray.
51+ */
52+static inline void *xa_mk_value(unsigned long v)
53+{
54+ WARN_ON((long)v < 0);
55+ return (void *)((v << 1) | 1);
56+}
57+
58+/**
59+ * xa_to_value() - Get value stored in an XArray entry.
60+ * @entry: XArray entry.
61+ *
62+ * Context: Any context.
63+ * Return: The value stored in the XArray entry.
64+ */
65+static inline unsigned long xa_to_value(const void *entry)
66+{
67+ return (unsigned long)entry >> 1;
68+}
69+
70+/**
71+ * xa_is_value() - Determine if an entry is a value.
72+ * @entry: XArray entry.
73+ *
74+ * Context: Any context.
75+ * Return: True if the entry is a value, false if it is a pointer.
76+ */
77+static inline bool xa_is_value(const void *entry)
78+{
79+ return (unsigned long)entry & 1;
80+}
81+
82+/**
83+ * xa_tag_pointer() - Create an XArray entry for a tagged pointer.
84+ * @p: Plain pointer.
85+ * @tag: Tag value (0, 1 or 3).
86+ *
87+ * If the user of the XArray prefers, they can tag their pointers instead
88+ * of storing value entries. Three tags are available (0, 1 and 3).
89+ * These are distinct from the xa_mark_t as they are not replicated up
90+ * through the array and cannot be searched for.
91+ *
92+ * Context: Any context.
93+ * Return: An XArray entry.
94+ */
95+static inline void *xa_tag_pointer(void *p, unsigned long tag)
96+{
97+ return (void *)((unsigned long)p | tag);
98+}
99+
100+/**
101+ * xa_untag_pointer() - Turn an XArray entry into a plain pointer.
102+ * @entry: XArray entry.
103+ *
104+ * If you have stored a tagged pointer in the XArray, call this function
105+ * to get the untagged version of the pointer.
106+ *
107+ * Context: Any context.
108+ * Return: A pointer.
109+ */
110+static inline void *xa_untag_pointer(void *entry)
111+{
112+ return (void *)((unsigned long)entry & ~3UL);
113+}
114+
115+/**
116+ * xa_pointer_tag() - Get the tag stored in an XArray entry.
117+ * @entry: XArray entry.
118+ *
119+ * If you have stored a tagged pointer in the XArray, call this function
120+ * to get the tag of that pointer.
121+ *
122+ * Context: Any context.
123+ * Return: A tag.
124+ */
125+static inline unsigned int xa_pointer_tag(void *entry)
126+{
127+ return (unsigned long)entry & 3UL;
128+}
129+
130+/*
131+ * xa_mk_internal() - Create an internal entry.
132+ * @v: Value to turn into an internal entry.
133+ *
134+ * Context: Any context.
135+ * Return: An XArray internal entry corresponding to this value.
136+ */
137+static inline void *xa_mk_internal(unsigned long v)
138+{
139+ return (void *)((v << 2) | 2);
140+}
141+
142+/*
143+ * xa_to_internal() - Extract the value from an internal entry.
144+ * @entry: XArray entry.
145+ *
146+ * Context: Any context.
147+ * Return: The value which was stored in the internal entry.
148+ */
149+static inline unsigned long xa_to_internal(const void *entry)
150+{
151+ return (unsigned long)entry >> 2;
152+}
153+
154+/*
155+ * xa_is_internal() - Is the entry an internal entry?
156+ * @entry: XArray entry.
157+ *
158+ * Context: Any context.
159+ * Return: %true if the entry is an internal entry.
160+ */
161+static inline bool xa_is_internal(const void *entry)
162+{
163+ return ((unsigned long)entry & 3) == 2;
164+}
165+
166+/**
167+ * xa_is_err() - Report whether an XArray operation returned an error
168+ * @entry: Result from calling an XArray function
169+ *
170+ * If an XArray operation cannot complete an operation, it will return
171+ * a special value indicating an error. This function tells you
172+ * whether an error occurred; xa_err() tells you which error occurred.
173+ *
174+ * Context: Any context.
175+ * Return: %true if the entry indicates an error.
176+ */
177+static inline bool xa_is_err(const void *entry)
178+{
179+ return unlikely(xa_is_internal(entry));
180+}
181+
182+/**
183+ * xa_err() - Turn an XArray result into an errno.
184+ * @entry: Result from calling an XArray function.
185+ *
186+ * If an XArray operation cannot complete an operation, it will return
187+ * a special pointer value which encodes an errno. This function extracts
188+ * the errno from the pointer value, or returns 0 if the pointer does not
189+ * represent an errno.
190+ *
191+ * Context: Any context.
192+ * Return: A negative errno or 0.
193+ */
194+static inline int xa_err(void *entry)
195+{
196+ /* xa_to_internal() would not do sign extension. */
197+ if (xa_is_err(entry))
198+ return (long)entry >> 2;
199+ return 0;
200+}
201+
202+typedef unsigned __bitwise xa_mark_t;
203+#define XA_MARK_0 ((__force xa_mark_t)0U)
204+#define XA_MARK_1 ((__force xa_mark_t)1U)
205+#define XA_MARK_2 ((__force xa_mark_t)2U)
206+#define XA_PRESENT ((__force xa_mark_t)8U)
207+#define XA_MARK_MAX XA_MARK_2
208+#define XA_FREE_MARK XA_MARK_0
209+
210+enum xa_lock_type {
211+ XA_LOCK_IRQ = 1,
212+ XA_LOCK_BH = 2,
213+};
214+
215+/*
216+ * Values for xa_flags. The radix tree stores its GFP flags in the xa_flags,
217+ * and we remain compatible with that.
218+ */
219+#define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ)
220+#define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH)
221+#define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U)
222+#define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \
223+ (__force unsigned)(mark)))
224+
225+#define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK))
226+
227+/**
228+ * struct xarray - The anchor of the XArray.
229+ * @xa_lock: Lock that protects the contents of the XArray.
230+ *
231+ * To use the xarray, define it statically or embed it in your data structure.
232+ * It is a very small data structure, so it does not usually make sense to
233+ * allocate it separately and keep a pointer to it in your data structure.
234+ *
235+ * You may use the xa_lock to protect your own data structures as well.
236+ */
237+/*
238+ * If all of the entries in the array are NULL, @xa_head is a NULL pointer.
239+ * If the only non-NULL entry in the array is at index 0, @xa_head is that
240+ * entry. If any other entry in the array is non-NULL, @xa_head points
241+ * to an @xa_node.
242+ */
243+struct xarray {
244+ spinlock_t xa_lock;
245+/* private: The rest of the data structure is not to be used directly. */
246+ gfp_t xa_flags;
247+ void __rcu * xa_head;
248+};
249+
250+#define XARRAY_INIT(name, flags) { \
251+ .xa_lock = __SPIN_LOCK_UNLOCKED(name.xa_lock), \
252+ .xa_flags = flags, \
253+ .xa_head = NULL, \
254+}
255+
256+/**
257+ * DEFINE_XARRAY_FLAGS() - Define an XArray with custom flags.
258+ * @name: A string that names your XArray.
259+ * @flags: XA_FLAG values.
260+ *
261+ * This is intended for file scope definitions of XArrays. It declares
262+ * and initialises an empty XArray with the chosen name and flags. It is
263+ * equivalent to calling xa_init_flags() on the array, but it does the
264+ * initialisation at compiletime instead of runtime.
265+ */
266+#define DEFINE_XARRAY_FLAGS(name, flags) \
267+ struct xarray name = XARRAY_INIT(name, flags)
268+
269+/**
270+ * DEFINE_XARRAY() - Define an XArray.
271+ * @name: A string that names your XArray.
272+ *
273+ * This is intended for file scope definitions of XArrays. It declares
274+ * and initialises an empty XArray with the chosen name. It is equivalent
275+ * to calling xa_init() on the array, but it does the initialisation at
276+ * compiletime instead of runtime.
277+ */
278+#define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0)
279+
280+/**
281+ * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs.
282+ * @name: A string that names your XArray.
283+ *
284+ * This is intended for file scope definitions of allocating XArrays.
285+ * See also DEFINE_XARRAY().
286+ */
287+#define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC)
288+
289+void xa_init_flags(struct xarray *, gfp_t flags);
290+void *xa_load(struct xarray *, unsigned long index);
291+void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
292+void *xa_cmpxchg(struct xarray *, unsigned long index,
293+ void *old, void *entry, gfp_t);
294+int xa_reserve(struct xarray *, unsigned long index, gfp_t);
295+void *xa_store_range(struct xarray *, unsigned long first, unsigned long last,
296+ void *entry, gfp_t);
297+bool xa_get_mark(struct xarray *, unsigned long index, xa_mark_t);
298+void xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
299+void xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
300+void *xa_find(struct xarray *xa, unsigned long *index,
301+ unsigned long max, xa_mark_t) __attribute__((nonnull(2)));
302+void *xa_find_after(struct xarray *xa, unsigned long *index,
303+ unsigned long max, xa_mark_t) __attribute__((nonnull(2)));
304+unsigned int xa_extract(struct xarray *, void **dst, unsigned long start,
305+ unsigned long max, unsigned int n, xa_mark_t);
306+void xa_destroy(struct xarray *);
307+
308+/**
309+ * xa_init() - Initialise an empty XArray.
310+ * @xa: XArray.
311+ *
312+ * An empty XArray is full of NULL entries.
313+ *
314+ * Context: Any context.
315+ */
316+static inline void xa_init(struct xarray *xa)
317+{
318+ xa_init_flags(xa, 0);
319+}
320+
321+/**
322+ * xa_empty() - Determine if an array has any present entries.
323+ * @xa: XArray.
324+ *
325+ * Context: Any context.
326+ * Return: %true if the array contains only NULL pointers.
327+ */
328+static inline bool xa_empty(const struct xarray *xa)
329+{
330+ return xa->xa_head == NULL;
331+}
332+
333+/**
334+ * xa_marked() - Inquire whether any entry in this array has a mark set
335+ * @xa: Array
336+ * @mark: Mark value
337+ *
338+ * Context: Any context.
339+ * Return: %true if any entry has this mark set.
340+ */
341+static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark)
342+{
343+ return xa->xa_flags & XA_FLAGS_MARK(mark);
344+}
345+
346+/**
347+ * xa_erase() - Erase this entry from the XArray.
348+ * @xa: XArray.
349+ * @index: Index of entry.
350+ *
351+ * This function is the equivalent of calling xa_store() with %NULL as
352+ * the third argument. The XArray does not need to allocate memory, so
353+ * the user does not need to provide GFP flags.
354+ *
355+ * Context: Process context. Takes and releases the xa_lock.
356+ * Return: The entry which used to be at this index.
357+ */
358+static inline void *xa_erase(struct xarray *xa, unsigned long index)
359+{
360+ return xa_store(xa, index, NULL, 0);
361+}
362+
363+/**
364+ * xa_insert() - Store this entry in the XArray unless another entry is
365+ * already present.
366+ * @xa: XArray.
367+ * @index: Index into array.
368+ * @entry: New entry.
369+ * @gfp: Memory allocation flags.
370+ *
371+ * If you would rather see the existing entry in the array, use xa_cmpxchg().
372+ * This function is for users who don't care what the entry is, only that
373+ * one is present.
374+ *
375+ * Context: Process context. Takes and releases the xa_lock.
376+ * May sleep if the @gfp flags permit.
377+ * Return: 0 if the store succeeded. -EEXIST if another entry was present.
378+ * -ENOMEM if memory could not be allocated.
379+ */
380+static inline int xa_insert(struct xarray *xa, unsigned long index,
381+ void *entry, gfp_t gfp)
382+{
383+ void *curr = xa_cmpxchg(xa, index, NULL, entry, gfp);
384+ if (!curr)
385+ return 0;
386+ if (xa_is_err(curr))
387+ return xa_err(curr);
388+ return -EEXIST;
389+}
390+
391+/**
392+ * xa_release() - Release a reserved entry.
393+ * @xa: XArray.
394+ * @index: Index of entry.
395+ *
396+ * After calling xa_reserve(), you can call this function to release the
397+ * reservation. If the entry at @index has been stored to, this function
398+ * will do nothing.
399+ */
400+static inline void xa_release(struct xarray *xa, unsigned long index)
401+{
402+ xa_cmpxchg(xa, index, NULL, NULL, 0);
403+}
404+
405+/**
406+ * xa_for_each() - Iterate over a portion of an XArray.
407+ * @xa: XArray.
408+ * @entry: Entry retrieved from array.
409+ * @index: Index of @entry.
410+ * @max: Maximum index to retrieve from array.
411+ * @filter: Selection criterion.
412+ *
413+ * Initialise @index to the lowest index you want to retrieve from the
414+ * array. During the iteration, @entry will have the value of the entry
415+ * stored in @xa at @index. The iteration will skip all entries in the
416+ * array which do not match @filter. You may modify @index during the
417+ * iteration if you want to skip or reprocess indices. It is safe to modify
418+ * the array during the iteration. At the end of the iteration, @entry will
419+ * be set to NULL and @index will have a value less than or equal to max.
420+ *
421+ * xa_for_each() is O(n.log(n)) while xas_for_each() is O(n). You have
422+ * to handle your own locking with xas_for_each(), and if you have to unlock
423+ * after each iteration, it will also end up being O(n.log(n)). xa_for_each()
424+ * will spin if it hits a retry entry; if you intend to see retry entries,
425+ * you should use the xas_for_each() iterator instead. The xas_for_each()
426+ * iterator will expand into more inline code than xa_for_each().
427+ *
428+ * Context: Any context. Takes and releases the RCU lock.
429+ */
430+#define xa_for_each(xa, entry, index, max, filter) \
431+ for (entry = xa_find(xa, &index, max, filter); entry; \
432+ entry = xa_find_after(xa, &index, max, filter))
11433
12434 #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock)
13435 #define xa_lock(xa) spin_lock(&(xa)->xa_lock)
@@ -21,4 +443,873 @@ · 7 unmodified lines
21443 #define xa_unlock_irqrestore(xa, flags) \
22444 spin_unlock_irqrestore(&(xa)->xa_lock, flags)
23445
446+/*
447+ * Versions of the normal API which require the caller to hold the
448+ * xa_lock. If the GFP flags allow it, they will drop the lock to
449+ * allocate memory, then reacquire it afterwards. These functions
450+ * may also re-enable interrupts if the XArray flags indicate the
451+ * locking should be interrupt safe.
452+ */
453+void *__xa_erase(struct xarray *, unsigned long index);
454+void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
455+void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old,
456+ void *entry, gfp_t);
457+int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t);
458+void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
459+void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
460+
461+/**
462+ * __xa_insert() - Store this entry in the XArray unless another entry is
463+ * already present.
464+ * @xa: XArray.
465+ * @index: Index into array.
466+ * @entry: New entry.
467+ * @gfp: Memory allocation flags.
468+ *
469+ * If you would rather see the existing entry in the array, use __xa_cmpxchg().
470+ * This function is for users who don't care what the entry is, only that
471+ * one is present.
472+ *
473+ * Context: Any context. Expects xa_lock to be held on entry. May
474+ * release and reacquire xa_lock if the @gfp flags permit.
475+ * Return: 0 if the store succeeded. -EEXIST if another entry was present.
476+ * -ENOMEM if memory could not be allocated.
477+ */
478+static inline int __xa_insert(struct xarray *xa, unsigned long index,
479+ void *entry, gfp_t gfp)
480+{
481+ void *curr = __xa_cmpxchg(xa, index, NULL, entry, gfp);
482+ if (!curr)
483+ return 0;
484+ if (xa_is_err(curr))
485+ return xa_err(curr);
486+ return -EEXIST;
487+}
488+
489+/**
490+ * xa_erase_bh() - Erase this entry from the XArray.
491+ * @xa: XArray.
492+ * @index: Index of entry.
493+ *
494+ * This function is the equivalent of calling xa_store() with %NULL as
495+ * the third argument. The XArray does not need to allocate memory, so
496+ * the user does not need to provide GFP flags.
497+ *
498+ * Context: Process context. Takes and releases the xa_lock while
499+ * disabling softirqs.
500+ * Return: The entry which used to be at this index.
501+ */
502+static inline void *xa_erase_bh(struct xarray *xa, unsigned long index)
503+{
504+ void *entry;
505+
506+ xa_lock_bh(xa);
507+ entry = __xa_erase(xa, index);
508+ xa_unlock_bh(xa);
509+
510+ return entry;
511+}
512+
513+/**
514+ * xa_erase_irq() - Erase this entry from the XArray.
515+ * @xa: XArray.
516+ * @index: Index of entry.
517+ *
518+ * This function is the equivalent of calling xa_store() with %NULL as
519+ * the third argument. The XArray does not need to allocate memory, so
520+ * the user does not need to provide GFP flags.
521+ *
522+ * Context: Process context. Takes and releases the xa_lock while
523+ * disabling interrupts.
524+ * Return: The entry which used to be at this index.
525+ */
526+static inline void *xa_erase_irq(struct xarray *xa, unsigned long index)
527+{
528+ void *entry;
529+
530+ xa_lock_irq(xa);
531+ entry = __xa_erase(xa, index);
532+ xa_unlock_irq(xa);
533+
534+ return entry;
535+}
536+
537+/**
538+ * xa_alloc() - Find somewhere to store this entry in the XArray.
539+ * @xa: XArray.
540+ * @id: Pointer to ID.
541+ * @max: Maximum ID to allocate (inclusive).
542+ * @entry: New entry.
543+ * @gfp: Memory allocation flags.
544+ *
545+ * Allocates an unused ID in the range specified by @id and @max.
546+ * Updates the @id pointer with the index, then stores the entry at that
547+ * index. A concurrent lookup will not see an uninitialised @id.
548+ *
549+ * Context: Process context. Takes and releases the xa_lock. May sleep if
550+ * the @gfp flags permit.
551+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
552+ * there is no more space in the XArray.
553+ */
554+static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry,
555+ gfp_t gfp)
556+{
557+ int err;
558+
559+ xa_lock(xa);
560+ err = __xa_alloc(xa, id, max, entry, gfp);
561+ xa_unlock(xa);
562+
563+ return err;
564+}
565+
566+/**
567+ * xa_alloc_bh() - Find somewhere to store this entry in the XArray.
568+ * @xa: XArray.
569+ * @id: Pointer to ID.
570+ * @max: Maximum ID to allocate (inclusive).
571+ * @entry: New entry.
572+ * @gfp: Memory allocation flags.
573+ *
574+ * Allocates an unused ID in the range specified by @id and @max.
575+ * Updates the @id pointer with the index, then stores the entry at that
576+ * index. A concurrent lookup will not see an uninitialised @id.
577+ *
578+ * Context: Process context. Takes and releases the xa_lock while
579+ * disabling softirqs. May sleep if the @gfp flags permit.
580+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
581+ * there is no more space in the XArray.
582+ */
583+static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry,
584+ gfp_t gfp)
585+{
586+ int err;
587+
588+ xa_lock_bh(xa);
589+ err = __xa_alloc(xa, id, max, entry, gfp);
590+ xa_unlock_bh(xa);
591+
592+ return err;
593+}
594+
595+/**
596+ * xa_alloc_irq() - Find somewhere to store this entry in the XArray.
597+ * @xa: XArray.
598+ * @id: Pointer to ID.
599+ * @max: Maximum ID to allocate (inclusive).
600+ * @entry: New entry.
601+ * @gfp: Memory allocation flags.
602+ *
603+ * Allocates an unused ID in the range specified by @id and @max.
604+ * Updates the @id pointer with the index, then stores the entry at that
605+ * index. A concurrent lookup will not see an uninitialised @id.
606+ *
607+ * Context: Process context. Takes and releases the xa_lock while
608+ * disabling interrupts. May sleep if the @gfp flags permit.
609+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
610+ * there is no more space in the XArray.
611+ */
612+static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry,
613+ gfp_t gfp)
614+{
615+ int err;
616+
617+ xa_lock_irq(xa);
618+ err = __xa_alloc(xa, id, max, entry, gfp);
619+ xa_unlock_irq(xa);
620+
621+ return err;
622+}
623+
624+/* Everything below here is the Advanced API. Proceed with caution. */
625+
626+/*
627+ * The xarray is constructed out of a set of 'chunks' of pointers. Choosing
628+ * the best chunk size requires some tradeoffs. A power of two recommends
629+ * itself so that we can walk the tree based purely on shifts and masks.
630+ * Generally, the larger the better; as the number of slots per level of the
631+ * tree increases, the less tall the tree needs to be. But that needs to be
632+ * balanced against the memory consumption of each node. On a 64-bit system,
633+ * xa_node is currently 576 bytes, and we get 7 of them per 4kB page. If we
634+ * doubled the number of slots per node, we'd get only 3 nodes per 4kB page.
635+ */
636+#ifndef XA_CHUNK_SHIFT
637+#define XA_CHUNK_SHIFT (CONFIG_BASE_SMALL ? 4 : 6)
638+#endif
639+#define XA_CHUNK_SIZE (1UL << XA_CHUNK_SHIFT)
640+#define XA_CHUNK_MASK (XA_CHUNK_SIZE - 1)
641+#define XA_MAX_MARKS 3
642+#define XA_MARK_LONGS DIV_ROUND_UP(XA_CHUNK_SIZE, BITS_PER_LONG)
643+
644+/*
645+ * @count is the count of every non-NULL element in the ->slots array
646+ * whether that is a value entry, a retry entry, a user pointer,
647+ * a sibling entry or a pointer to the next level of the tree.
648+ * @nr_values is the count of every element in ->slots which is
649+ * either a value entry or a sibling of a value entry.
650+ */
651+struct xa_node {
652+ unsigned char shift; /* Bits remaining in each slot */
653+ unsigned char offset; /* Slot offset in parent */
654+ unsigned char count; /* Total entry count */
655+ unsigned char nr_values; /* Value entry count */
656+ struct xa_node __rcu *parent; /* NULL at top of tree */
657+ struct xarray *array; /* The array we belong to */
658+ union {
659+ struct list_head private_list; /* For tree user */
660+ struct rcu_head rcu_head; /* Used when freeing node */
661+ };
662+ void __rcu *slots[XA_CHUNK_SIZE];
663+ union {
664+ unsigned long tags[XA_MAX_MARKS][XA_MARK_LONGS];
665+ unsigned long marks[XA_MAX_MARKS][XA_MARK_LONGS];
666+ };
667+};
668+
669+void xa_dump(const struct xarray *);
670+void xa_dump_node(const struct xa_node *);
671+
672+#ifdef XA_DEBUG
673+#define XA_BUG_ON(xa, x) do { \
674+ if (x) { \
675+ xa_dump(xa); \
676+ BUG(); \
677+ } \
678+ } while (0)
679+#define XA_NODE_BUG_ON(node, x) do { \
680+ if (x) { \
681+ if (node) xa_dump_node(node); \
682+ BUG(); \
683+ } \
684+ } while (0)
685+#else
686+#define XA_BUG_ON(xa, x) do { } while (0)
687+#define XA_NODE_BUG_ON(node, x) do { } while (0)
688+#endif
689+
690+/* Private */
691+static inline void *xa_head(const struct xarray *xa)
692+{
693+ return rcu_dereference_check(xa->xa_head,
694+ lockdep_is_held(&xa->xa_lock));
695+}
696+
697+/* Private */
698+static inline void *xa_head_locked(const struct xarray *xa)
699+{
700+ return rcu_dereference_protected(xa->xa_head,
701+ lockdep_is_held(&xa->xa_lock));
702+}
703+
704+/* Private */
705+static inline void *xa_entry(const struct xarray *xa,
706+ const struct xa_node *node, unsigned int offset)
707+{
708+ XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE);
709+ return rcu_dereference_check(node->slots[offset],
710+ lockdep_is_held(&xa->xa_lock));
711+}
712+
713+/* Private */
714+static inline void *xa_entry_locked(const struct xarray *xa,
715+ const struct xa_node *node, unsigned int offset)
716+{
717+ XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE);
718+ return rcu_dereference_protected(node->slots[offset],
719+ lockdep_is_held(&xa->xa_lock));
720+}
721+
722+/* Private */
723+static inline struct xa_node *xa_parent(const struct xarray *xa,
724+ const struct xa_node *node)
725+{
726+ return rcu_dereference_check(node->parent,
727+ lockdep_is_held(&xa->xa_lock));
728+}
729+
730+/* Private */
731+static inline struct xa_node *xa_parent_locked(const struct xarray *xa,
732+ const struct xa_node *node)
733+{
734+ return rcu_dereference_protected(node->parent,
735+ lockdep_is_held(&xa->xa_lock));
736+}
737+
738+/* Private */
739+static inline void *xa_mk_node(const struct xa_node *node)
740+{
741+ return (void *)((unsigned long)node | 2);
742+}
743+
744+/* Private */
745+static inline struct xa_node *xa_to_node(const void *entry)
746+{
747+ return (struct xa_node *)((unsigned long)entry - 2);
748+}
749+
750+/* Private */
751+static inline bool xa_is_node(const void *entry)
752+{
753+ return xa_is_internal(entry) && (unsigned long)entry > 4096;
754+}
755+
756+/* Private */
757+static inline void *xa_mk_sibling(unsigned int offset)
758+{
759+ return xa_mk_internal(offset);
760+}
761+
762+/* Private */
763+static inline unsigned long xa_to_sibling(const void *entry)
764+{
765+ return xa_to_internal(entry);
766+}
767+
768+/**
769+ * xa_is_sibling() - Is the entry a sibling entry?
770+ * @entry: Entry retrieved from the XArray
771+ *
772+ * Return: %true if the entry is a sibling entry.
773+ */
774+static inline bool xa_is_sibling(const void *entry)
775+{
776+ return IS_ENABLED(CONFIG_XARRAY_MULTI) && xa_is_internal(entry) &&
777+ (entry < xa_mk_sibling(XA_CHUNK_SIZE - 1));
778+}
779+
780+#define XA_ZERO_ENTRY xa_mk_internal(256)
781+#define XA_RETRY_ENTRY xa_mk_internal(257)
782+
783+/**
784+ * xa_is_zero() - Is the entry a zero entry?
785+ * @entry: Entry retrieved from the XArray
786+ *
787+ * Return: %true if the entry is a zero entry.
788+ */
789+static inline bool xa_is_zero(const void *entry)
790+{
791+ return unlikely(entry == XA_ZERO_ENTRY);
792+}
793+
794+/**
795+ * xa_is_retry() - Is the entry a retry entry?
796+ * @entry: Entry retrieved from the XArray
797+ *
798+ * Return: %true if the entry is a retry entry.
799+ */
800+static inline bool xa_is_retry(const void *entry)
801+{
802+ return unlikely(entry == XA_RETRY_ENTRY);
803+}
804+
805+/**
806+ * typedef xa_update_node_t - A callback function from the XArray.
807+ * @node: The node which is being processed
808+ *
809+ * This function is called every time the XArray updates the count of
810+ * present and value entries in a node. It allows advanced users to
811+ * maintain the private_list in the node.
812+ *
813+ * Context: The xa_lock is held and interrupts may be disabled.
814+ * Implementations should not drop the xa_lock, nor re-enable
815+ * interrupts.
816+ */
817+typedef void (*xa_update_node_t)(struct xa_node *node);
818+
819+/*
820+ * The xa_state is opaque to its users. It contains various different pieces
821+ * of state involved in the current operation on the XArray. It should be
822+ * declared on the stack and passed between the various internal routines.
823+ * The various elements in it should not be accessed directly, but only
824+ * through the provided accessor functions. The below documentation is for
825+ * the benefit of those working on the code, not for users of the XArray.
826+ *
827+ * @xa_node usually points to the xa_node containing the slot we're operating
828+ * on (and @xa_offset is the offset in the slots array). If there is a
829+ * single entry in the array at index 0, there are no allocated xa_nodes to
830+ * point to, and so we store %NULL in @xa_node. @xa_node is set to
831+ * the value %XAS_RESTART if the xa_state is not walked to the correct
832+ * position in the tree of nodes for this operation. If an error occurs
833+ * during an operation, it is set to an %XAS_ERROR value. If we run off the
834+ * end of the allocated nodes, it is set to %XAS_BOUNDS.
835+ */
836+struct xa_state {
837+ struct xarray *xa;
838+ unsigned long xa_index;
839+ unsigned char xa_shift;
840+ unsigned char xa_sibs;
841+ unsigned char xa_offset;
842+ unsigned char xa_pad; /* Helps gcc generate better code */
843+ struct xa_node *xa_node;
844+ struct xa_node *xa_alloc;
845+ xa_update_node_t xa_update;
846+};
847+
848+/*
849+ * We encode errnos in the xas->xa_node. If an error has happened, we need to
850+ * drop the lock to fix it, and once we've done so the xa_state is invalid.
851+ */
852+#define XA_ERROR(errno) ((struct xa_node *)(((unsigned long)errno << 2) | 2UL))
853+#define XAS_BOUNDS ((struct xa_node *)1UL)
854+#define XAS_RESTART ((struct xa_node *)3UL)
855+
856+#define __XA_STATE(array, index, shift, sibs) { \
857+ .xa = array, \
858+ .xa_index = index, \
859+ .xa_shift = shift, \
860+ .xa_sibs = sibs, \
861+ .xa_offset = 0, \
862+ .xa_pad = 0, \
863+ .xa_node = XAS_RESTART, \
864+ .xa_alloc = NULL, \
865+ .xa_update = NULL \
866+}
867+
868+/**
869+ * XA_STATE() - Declare an XArray operation state.
870+ * @name: Name of this operation state (usually xas).
871+ * @array: Array to operate on.
872+ * @index: Initial index of interest.
873+ *
874+ * Declare and initialise an xa_state on the stack.
875+ */
876+#define XA_STATE(name, array, index) \
877+ struct xa_state name = __XA_STATE(array, index, 0, 0)
878+
879+/**
880+ * XA_STATE_ORDER() - Declare an XArray operation state.
881+ * @name: Name of this operation state (usually xas).
882+ * @array: Array to operate on.
883+ * @index: Initial index of interest.
884+ * @order: Order of entry.
885+ *
886+ * Declare and initialise an xa_state on the stack. This variant of
887+ * XA_STATE() allows you to specify the 'order' of the element you
888+ * want to operate on.`
889+ */
890+#define XA_STATE_ORDER(name, array, index, order) \
891+ struct xa_state name = __XA_STATE(array, \
892+ (index >> order) << order, \
893+ order - (order % XA_CHUNK_SHIFT), \
894+ (1U << (order % XA_CHUNK_SHIFT)) - 1)
895+
896+#define xas_marked(xas, mark) xa_marked((xas)->xa, (mark))
897+#define xas_trylock(xas) xa_trylock((xas)->xa)
898+#define xas_lock(xas) xa_lock((xas)->xa)
899+#define xas_unlock(xas) xa_unlock((xas)->xa)
900+#define xas_lock_bh(xas) xa_lock_bh((xas)->xa)
901+#define xas_unlock_bh(xas) xa_unlock_bh((xas)->xa)
902+#define xas_lock_irq(xas) xa_lock_irq((xas)->xa)
903+#define xas_unlock_irq(xas) xa_unlock_irq((xas)->xa)
904+#define xas_lock_irqsave(xas, flags) \
905+ xa_lock_irqsave((xas)->xa, flags)
906+#define xas_unlock_irqrestore(xas, flags) \
907+ xa_unlock_irqrestore((xas)->xa, flags)
908+
909+/**
910+ * xas_error() - Return an errno stored in the xa_state.
911+ * @xas: XArray operation state.
912+ *
913+ * Return: 0 if no error has been noted. A negative errno if one has.
914+ */
915+static inline int xas_error(const struct xa_state *xas)
916+{
917+ return xa_err(xas->xa_node);
918+}
919+
920+/**
921+ * xas_set_err() - Note an error in the xa_state.
922+ * @xas: XArray operation state.
923+ * @err: Negative error number.
924+ *
925+ * Only call this function with a negative @err; zero or positive errors
926+ * will probably not behave the way you think they should. If you want
927+ * to clear the error from an xa_state, use xas_reset().
928+ */
929+static inline void xas_set_err(struct xa_state *xas, long err)
930+{
931+ xas->xa_node = XA_ERROR(err);
932+}
933+
934+/**
935+ * xas_invalid() - Is the xas in a retry or error state?
936+ * @xas: XArray operation state.
937+ *
938+ * Return: %true if the xas cannot be used for operations.
939+ */
940+static inline bool xas_invalid(const struct xa_state *xas)
941+{
942+ return (unsigned long)xas->xa_node & 3;
943+}
944+
945+/**
946+ * xas_valid() - Is the xas a valid cursor into the array?
947+ * @xas: XArray operation state.
948+ *
949+ * Return: %true if the xas can be used for operations.
950+ */
951+static inline bool xas_valid(const struct xa_state *xas)
952+{
953+ return !xas_invalid(xas);
954+}
955+
956+/**
957+ * xas_is_node() - Does the xas point to a node?
958+ * @xas: XArray operation state.
959+ *
960+ * Return: %true if the xas currently references a node.
961+ */
962+static inline bool xas_is_node(const struct xa_state *xas)
963+{
964+ return xas_valid(xas) && xas->xa_node;
965+}
966+
967+/* True if the pointer is something other than a node */
968+static inline bool xas_not_node(struct xa_node *node)
969+{
970+ return ((unsigned long)node & 3) || !node;
971+}
972+
973+/* True if the node represents RESTART or an error */
974+static inline bool xas_frozen(struct xa_node *node)
975+{
976+ return (unsigned long)node & 2;
977+}
978+
979+/* True if the node represents head-of-tree, RESTART or BOUNDS */
980+static inline bool xas_top(struct xa_node *node)
981+{
982+ return node <= XAS_RESTART;
983+}
984+
985+/**
986+ * xas_reset() - Reset an XArray operation state.
987+ * @xas: XArray operation state.
988+ *
989+ * Resets the error or walk state of the @xas so future walks of the
990+ * array will start from the root. Use this if you have dropped the
991+ * xarray lock and want to reuse the xa_state.
992+ *
993+ * Context: Any context.
994+ */
995+static inline void xas_reset(struct xa_state *xas)
996+{
997+ xas->xa_node = XAS_RESTART;
998+}
999+
1000+/**
1001+ * xas_retry() - Retry the operation if appropriate.
1002+ * @xas: XArray operation state.
1003+ * @entry: Entry from xarray.
1004+ *
1005+ * The advanced functions may sometimes return an internal entry, such as
1006+ * a retry entry or a zero entry. This function sets up the @xas to restart
1007+ * the walk from the head of the array if needed.
1008+ *
1009+ * Context: Any context.
1010+ * Return: true if the operation needs to be retried.
1011+ */
1012+static inline bool xas_retry(struct xa_state *xas, const void *entry)
1013+{
1014+ if (xa_is_zero(entry))
1015+ return true;
1016+ if (!xa_is_retry(entry))
1017+ return false;
1018+ xas_reset(xas);
1019+ return true;
1020+}
1021+
1022+void *xas_load(struct xa_state *);
1023+void *xas_store(struct xa_state *, void *entry);
1024+void *xas_find(struct xa_state *, unsigned long max);
1025+void *xas_find_conflict(struct xa_state *);
1026+
1027+bool xas_get_mark(const struct xa_state *, xa_mark_t);
1028+void xas_set_mark(const struct xa_state *, xa_mark_t);
1029+void xas_clear_mark(const struct xa_state *, xa_mark_t);
1030+void *xas_find_marked(struct xa_state *, unsigned long max, xa_mark_t);
1031+void xas_init_marks(const struct xa_state *);
1032+
1033+bool xas_nomem(struct xa_state *, gfp_t);
1034+void xas_pause(struct xa_state *);
1035+
1036+void xas_create_range(struct xa_state *);
1037+
1038+/**
1039+ * xas_reload() - Refetch an entry from the xarray.
1040+ * @xas: XArray operation state.
1041+ *
1042+ * Use this function to check that a previously loaded entry still has
1043+ * the same value. This is useful for the lockless pagecache lookup where
1044+ * we walk the array with only the RCU lock to protect us, lock the page,
1045+ * then check that the page hasn't moved since we looked it up.
1046+ *
1047+ * The caller guarantees that @xas is still valid. If it may be in an
1048+ * error or restart state, call xas_load() instead.
1049+ *
1050+ * Return: The entry at this location in the xarray.
1051+ */
1052+static inline void *xas_reload(struct xa_state *xas)
1053+{
1054+ struct xa_node *node = xas->xa_node;
1055+
1056+ if (node)
1057+ return xa_entry(xas->xa, node, xas->xa_offset);
1058+ return xa_head(xas->xa);
1059+}
1060+
1061+/**
1062+ * xas_set() - Set up XArray operation state for a different index.
1063+ * @xas: XArray operation state.
1064+ * @index: New index into the XArray.
1065+ *
1066+ * Move the operation state to refer to a different index. This will
1067+ * have the effect of starting a walk from the top; see xas_next()
1068+ * to move to an adjacent index.
1069+ */
1070+static inline void xas_set(struct xa_state *xas, unsigned long index)
1071+{
1072+ xas->xa_index = index;
1073+ xas->xa_node = XAS_RESTART;
1074+}
1075+
1076+/**
1077+ * xas_set_order() - Set up XArray operation state for a multislot entry.
1078+ * @xas: XArray operation state.
1079+ * @index: Target of the operation.
1080+ * @order: Entry occupies 2^@order indices.
1081+ */
1082+static inline void xas_set_order(struct xa_state *xas, unsigned long index,
1083+ unsigned int order)
1084+{
1085+#ifdef CONFIG_XARRAY_MULTI
1086+ xas->xa_index = order < BITS_PER_LONG ? (index >> order) << order : 0;
1087+ xas->xa_shift = order - (order % XA_CHUNK_SHIFT);
1088+ xas->xa_sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1;
1089+ xas->xa_node = XAS_RESTART;
1090+#else
1091+ BUG_ON(order > 0);
1092+ xas_set(xas, index);
1093+#endif
1094+}
1095+
1096+/**
1097+ * xas_set_update() - Set up XArray operation state for a callback.
1098+ * @xas: XArray operation state.
1099+ * @update: Function to call when updating a node.
1100+ *
1101+ * The XArray can notify a caller after it has updated an xa_node.
1102+ * This is advanced functionality and is only needed by the page cache.
1103+ */
1104+static inline void xas_set_update(struct xa_state *xas, xa_update_node_t update)
1105+{
1106+ xas->xa_update = update;
1107+}
1108+
1109+/**
1110+ * xas_next_entry() - Advance iterator to next present entry.
1111+ * @xas: XArray operation state.
1112+ * @max: Highest index to return.
1113+ *
1114+ * xas_next_entry() is an inline function to optimise xarray traversal for
1115+ * speed. It is equivalent to calling xas_find(), and will call xas_find()
1116+ * for all the hard cases.
1117+ *
1118+ * Return: The next present entry after the one currently referred to by @xas.
1119+ */
1120+static inline void *xas_next_entry(struct xa_state *xas, unsigned long max)
1121+{
1122+ struct xa_node *node = xas->xa_node;
1123+ void *entry;
1124+
1125+ if (unlikely(xas_not_node(node) || node->shift ||
1126+ xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)))
1127+ return xas_find(xas, max);
1128+
1129+ do {
1130+ if (unlikely(xas->xa_index >= max))
1131+ return xas_find(xas, max);
1132+ if (unlikely(xas->xa_offset == XA_CHUNK_MASK))
1133+ return xas_find(xas, max);
1134+ entry = xa_entry(xas->xa, node, xas->xa_offset + 1);
1135+ if (unlikely(xa_is_internal(entry)))
1136+ return xas_find(xas, max);
1137+ xas->xa_offset++;
1138+ xas->xa_index++;
1139+ } while (!entry);
1140+
1141+ return entry;
1142+}
1143+
1144+/* Private */
1145+static inline unsigned int xas_find_chunk(struct xa_state *xas, bool advance,
1146+ xa_mark_t mark)
1147+{
1148+ unsigned long *addr = xas->xa_node->marks[(__force unsigned)mark];
1149+ unsigned int offset = xas->xa_offset;
1150+
1151+ if (advance)
1152+ offset++;
1153+ if (XA_CHUNK_SIZE == BITS_PER_LONG) {
1154+ if (offset < XA_CHUNK_SIZE) {
1155+ unsigned long data = *addr & (~0UL << offset);
1156+ if (data)
1157+ return __ffs(data);
1158+ }
1159+ return XA_CHUNK_SIZE;
1160+ }
1161+
1162+ return find_next_bit(addr, XA_CHUNK_SIZE, offset);
1163+}
1164+
1165+/**
1166+ * xas_next_marked() - Advance iterator to next marked entry.
1167+ * @xas: XArray operation state.
1168+ * @max: Highest index to return.
1169+ * @mark: Mark to search for.
1170+ *
1171+ * xas_next_marked() is an inline function to optimise xarray traversal for
1172+ * speed. It is equivalent to calling xas_find_marked(), and will call
1173+ * xas_find_marked() for all the hard cases.
1174+ *
1175+ * Return: The next marked entry after the one currently referred to by @xas.
1176+ */
1177+static inline void *xas_next_marked(struct xa_state *xas, unsigned long max,
1178+ xa_mark_t mark)
1179+{
1180+ struct xa_node *node = xas->xa_node;
1181+ unsigned int offset;
1182+
1183+ if (unlikely(xas_not_node(node) || node->shift))
1184+ return xas_find_marked(xas, max, mark);
1185+ offset = xas_find_chunk(xas, true, mark);
1186+ xas->xa_offset = offset;
1187+ xas->xa_index = (xas->xa_index & ~XA_CHUNK_MASK) + offset;
1188+ if (xas->xa_index > max)
1189+ return NULL;
1190+ if (offset == XA_CHUNK_SIZE)
1191+ return xas_find_marked(xas, max, mark);
1192+ return xa_entry(xas->xa, node, offset);
1193+}
1194+
1195+/*
1196+ * If iterating while holding a lock, drop the lock and reschedule
1197+ * every %XA_CHECK_SCHED loops.
1198+ */
1199+enum {
1200+ XA_CHECK_SCHED = 4096,
1201+};
1202+
1203+/**
1204+ * xas_for_each() - Iterate over a range of an XArray.
1205+ * @xas: XArray operation state.
1206+ * @entry: Entry retrieved from the array.
1207+ * @max: Maximum index to retrieve from array.
1208+ *
1209+ * The loop body will be executed for each entry present in the xarray
1210+ * between the current xas position and @max. @entry will be set to
1211+ * the entry retrieved from the xarray. It is safe to delete entries
1212+ * from the array in the loop body. You should hold either the RCU lock
1213+ * or the xa_lock while iterating. If you need to drop the lock, call
1214+ * xas_pause() first.
1215+ */
1216+#define xas_for_each(xas, entry, max) \
1217+ for (entry = xas_find(xas, max); entry; \
1218+ entry = xas_next_entry(xas, max))
1219+
1220+/**
1221+ * xas_for_each_marked() - Iterate over a range of an XArray.
1222+ * @xas: XArray operation state.
1223+ * @entry: Entry retrieved from the array.
1224+ * @max: Maximum index to retrieve from array.
1225+ * @mark: Mark to search for.
1226+ *
1227+ * The loop body will be executed for each marked entry in the xarray
1228+ * between the current xas position and @max. @entry will be set to
1229+ * the entry retrieved from the xarray. It is safe to delete entries
1230+ * from the array in the loop body. You should hold either the RCU lock
1231+ * or the xa_lock while iterating. If you need to drop the lock, call
1232+ * xas_pause() first.
1233+ */
1234+#define xas_for_each_marked(xas, entry, max, mark) \
1235+ for (entry = xas_find_marked(xas, max, mark); entry; \
1236+ entry = xas_next_marked(xas, max, mark))
1237+
1238+/**
1239+ * xas_for_each_conflict() - Iterate over a range of an XArray.
1240+ * @xas: XArray operation state.
1241+ * @entry: Entry retrieved from the array.
1242+ *
1243+ * The loop body will be executed for each entry in the XArray that lies
1244+ * within the range specified by @xas. If the loop completes successfully,
1245+ * any entries that lie in this range will be replaced by @entry. The caller
1246+ * may break out of the loop; if they do so, the contents of the XArray will
1247+ * be unchanged. The operation may fail due to an out of memory condition.
1248+ * The caller may also call xa_set_err() to exit the loop while setting an
1249+ * error to record the reason.
1250+ */
1251+#define xas_for_each_conflict(xas, entry) \
1252+ while ((entry = xas_find_conflict(xas)))
1253+
1254+void *__xas_next(struct xa_state *);
1255+void *__xas_prev(struct xa_state *);
1256+
1257+/**
1258+ * xas_prev() - Move iterator to previous index.
1259+ * @xas: XArray operation state.
1260+ *
1261+ * If the @xas was in an error state, it will remain in an error state
1262+ * and this function will return %NULL. If the @xas has never been walked,
1263+ * it will have the effect of calling xas_load(). Otherwise one will be
1264+ * subtracted from the index and the state will be walked to the correct
1265+ * location in the array for the next operation.
1266+ *
1267+ * If the iterator was referencing index 0, this function wraps
1268+ * around to %ULONG_MAX.
1269+ *
1270+ * Return: The entry at the new index. This may be %NULL or an internal
1271+ * entry.
1272+ */
1273+static inline void *xas_prev(struct xa_state *xas)
1274+{
1275+ struct xa_node *node = xas->xa_node;
1276+
1277+ if (unlikely(xas_not_node(node) || node->shift ||
1278+ xas->xa_offset == 0))
1279+ return __xas_prev(xas);
1280+
1281+ xas->xa_index--;
1282+ xas->xa_offset--;
1283+ return xa_entry(xas->xa, node, xas->xa_offset);
1284+}
1285+
1286+/**
1287+ * xas_next() - Move state to next index.
1288+ * @xas: XArray operation state.
1289+ *
1290+ * If the @xas was in an error state, it will remain in an error state
1291+ * and this function will return %NULL. If the @xas has never been walked,
1292+ * it will have the effect of calling xas_load(). Otherwise one will be
1293+ * added to the index and the state will be walked to the correct
1294+ * location in the array for the next operation.
1295+ *
1296+ * If the iterator was referencing index %ULONG_MAX, this function wraps
1297+ * around to 0.
1298+ *
1299+ * Return: The entry at the new index. This may be %NULL or an internal
1300+ * entry.
1301+ */
1302+static inline void *xas_next(struct xa_state *xas)
1303+{
1304+ struct xa_node *node = xas->xa_node;
1305+
1306+ if (unlikely(xas_not_node(node) || node->shift ||
1307+ xas->xa_offset == XA_CHUNK_MASK))
1308+ return __xas_next(xas);
1309+
1310+ xas->xa_index++;
1311+ xas->xa_offset++;
1312+ return xa_entry(xas->xa, node, xas->xa_offset);
1313+}
1314+
241315 #endif /* _LINUX_XARRAY_H */
kernel/memremap.c+15−60
@@ -1,47 +1,21 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
22 /* Copyright(c) 2015 Intel Corporation. All rights reserved. */
3−#include <linux/radix-tree.h>
43 #include <linux/device.h>
5−#include <linux/types.h>
6−#include <linux/pfn_t.h>
74 #include <linux/io.h>
85 #include <linux/kasan.h>
9−#include <linux/mm.h>
106 #include <linux/memory_hotplug.h>
7+#include <linux/mm.h>
8+#include <linux/pfn_t.h>
119 #include <linux/swap.h>
1210 #include <linux/swapops.h>
11+#include <linux/types.h>
1312 #include <linux/wait_bit.h>
13+#include <linux/xarray.h>
1414
15−static DEFINE_MUTEX(pgmap_lock);
16−static RADIX_TREE(pgmap_radix, GFP_KERNEL);
15+static DEFINE_XARRAY(pgmap_array);
1716 #define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
1817 #define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
1918
20−static unsigned long order_at(struct resource *res, unsigned long pgoff)
21−{
22− unsigned long phys_pgoff = PHYS_PFN(res->start) + pgoff;
23− unsigned long nr_pages, mask;
24−
25− nr_pages = PHYS_PFN(resource_size(res));
26− if (nr_pages == pgoff)
27− return ULONG_MAX;
28−
29− /*
30− * What is the largest aligned power-of-2 range available from
31− * this resource pgoff to the end of the resource range,
32− * considering the alignment of the current pgoff?
33− */
34− mask = phys_pgoff | rounddown_pow_of_two(nr_pages - pgoff);
35− if (!mask)
36− return ULONG_MAX;
37−
38− return find_first_bit(&mask, BITS_PER_LONG);
39−}
40−
41−#define foreach_order_pgoff(res, order, pgoff) \
42− for (pgoff = 0, order = order_at((res), pgoff); order < ULONG_MAX; \
43− pgoff += 1UL << order, order = order_at((res), pgoff))
44−
4519 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
4620 vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
4721 unsigned long addr,
@@ -70,18 +44,10 @@ vm_fault_t device_private_entry_fault(struct vm_area_struct *vma, · 22 unmodified lines
7044 EXPORT_SYMBOL(device_private_entry_fault);
7145 #endif /* CONFIG_DEVICE_PRIVATE */
7246
73−static void pgmap_radix_release(struct resource *res, unsigned long end_pgoff)
47+static void pgmap_array_delete(struct resource *res)
7448 {
75− unsigned long pgoff, order;
76−
77− mutex_lock(&pgmap_lock);
78− foreach_order_pgoff(res, order, pgoff) {
79− if (pgoff >= end_pgoff)
80− break;
81− radix_tree_delete(&pgmap_radix, PHYS_PFN(res->start) + pgoff);
82− }
83− mutex_unlock(&pgmap_lock);
84−
49+ xa_store_range(&pgmap_array, PHYS_PFN(res->start), PHYS_PFN(res->end),
50+ NULL, GFP_KERNEL);
8551 synchronize_rcu();
8652 }
8753
@@ -142,7 +108,7 @@ static void devm_memremap_pages_release(void *data) · 54 unmodified lines
142108 mem_hotplug_done();
143109
144110 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
145− pgmap_radix_release(res, -1);
111+ pgmap_array_delete(res);
146112 dev_WARN_ONCE(dev, pgmap->altmap.alloc,
147113 "%s: failed to free all reserved pages\n", __func__);
148114 }
@@ -177,7 +143,6 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap) · 28 unmodified lines
177143 struct resource *res = &pgmap->res;
178144 struct dev_pagemap *conflict_pgmap;
179145 pgprot_t pgprot = PAGE_KERNEL;
180− unsigned long pgoff, order;
181146 int error, nid, is_ram;
182147
183148 align_start = res->start & ~(SECTION_SIZE - 1);
@@ -216,20 +181,10 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap) · 32 unmodified lines
216181
217182 pgmap->dev = dev;
218183
219− mutex_lock(&pgmap_lock);
220− error = 0;
221−
222− foreach_order_pgoff(res, order, pgoff) {
223− error = __radix_tree_insert(&pgmap_radix,
224− PHYS_PFN(res->start) + pgoff, order, pgmap);
225− if (error) {
226− dev_err(dev, "%s: failed: %d\n", __func__, error);
227− break;
228− }
229− }
230− mutex_unlock(&pgmap_lock);
184+ error = xa_err(xa_store_range(&pgmap_array, PHYS_PFN(res->start),
185+ PHYS_PFN(res->end), pgmap, GFP_KERNEL));
231186 if (error)
232− goto err_radix;
187+ goto err_array;
233188
234189 nid = dev_to_node(dev);
235190 if (nid < 0)
@@ -274,8 +229,8 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap) · 38 unmodified lines
274229 err_kasan:
275230 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
276231 err_pfn_remap:
277− err_radix:
278− pgmap_radix_release(res, pgoff);
232+ pgmap_array_delete(res);
233+ err_array:
279234 return ERR_PTR(error);
280235 }
281236 EXPORT_SYMBOL(devm_memremap_pages);
@@ -315,7 +270,7 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, · 33 unmodified lines
315270
316271 /* fall back to slow path lookup */
317272 rcu_read_lock();
318− pgmap = radix_tree_lookup(&pgmap_radix, PHYS_PFN(phys));
273+ pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));
319274 if (pgmap && !percpu_ref_tryget_live(pgmap->ref))
320275 pgmap = NULL;
321276 rcu_read_unlock();
lib/Kconfig+4−1
@@ -399,8 +399,11 @@ config INTERVAL_TREE · 398 unmodified lines
399399
400400 for more information.
401401
402−config RADIX_TREE_MULTIORDER
402+config XARRAY_MULTI
403403 bool
404+ help
405+ Support entries which occupy multiple consecutive indices in the
406+ XArray.
404407
405408 config ASSOCIATIVE_ARRAY
406409 bool
lib/Kconfig.debug+3−0
@@ -1813,6 +1813,9 @@ config TEST_BITFIELD · 1812 unmodified lines
18131813 config TEST_UUID
18141814 tristate "Test functions located in the uuid module at runtime"
18151815
1816+config TEST_XARRAY
1817+ tristate "Test the XArray code at runtime"
1818+
18161819 config TEST_OVERFLOW
18171820 tristate "Test check_*_overflow() functions at runtime"
18181821
lib/Makefile+2−1
@@ -18,7 +18,7 @@ KCOV_INSTRUMENT_debugobjects.o := n · 17 unmodified lines
1818 KCOV_INSTRUMENT_dynamic_debug.o := n
1919
2020 lib-y := ctype.o string.o vsprintf.o cmdline.o \
21− rbtree.o radix-tree.o timerqueue.o\
21+ rbtree.o radix-tree.o timerqueue.o xarray.o \
2222 idr.o int_sqrt.o extable.o \
2323 sha1.o chacha20.o irq_regs.o argv_split.o \
2424 flex_proportions.o ratelimit.o show_mem.o \
@@ -68,6 +68,7 @@ obj-$(CONFIG_TEST_PRINTF) += test_printf.o · 43 unmodified lines
6868 obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o
6969 obj-$(CONFIG_TEST_BITFIELD) += test_bitfield.o
7070 obj-$(CONFIG_TEST_UUID) += test_uuid.o
71+obj-$(CONFIG_TEST_XARRAY) += test_xarray.o
7172 obj-$(CONFIG_TEST_PARMAN) += test_parman.o
7273 obj-$(CONFIG_TEST_KMOD) += test_kmod.o
7374 obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o
lib/idr.c+207−194
@@ -6,8 +6,6 @@ · 5 unmodified lines
66 #include <linux/spinlock.h>
77 #include <linux/xarray.h>
88
9−DEFINE_PER_CPU(struct ida_bitmap *, ida_bitmap);
10−
119 /**
1210 * idr_alloc_u32() - Allocate an ID.
1311 * @idr: IDR handle.
@@ -39,10 +37,8 @@ int idr_alloc_u32(struct idr *idr, void *ptr, u32 *nextid, · 25 unmodified lines
3937 unsigned int base = idr->idr_base;
4038 unsigned int id = *nextid;
4139
42− if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))
43− return -EINVAL;
44− if (WARN_ON_ONCE(!(idr->idr_rt.gfp_mask & ROOT_IS_IDR)))
45− idr->idr_rt.gfp_mask |= IDR_RT_MARKER;
40+ if (WARN_ON_ONCE(!(idr->idr_rt.xa_flags & ROOT_IS_IDR)))
41+ idr->idr_rt.xa_flags |= IDR_RT_MARKER;
4642
4743 id = (id < base) ? 0 : id - base;
4844 radix_tree_iter_init(&iter, id);
@@ -295,15 +291,13 @@ void *idr_replace(struct idr *idr, void *ptr, unsigned long id) · 246 unmodified lines
295291 void __rcu **slot = NULL;
296292 void *entry;
297293
298− if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr)))
299− return ERR_PTR(-EINVAL);
300294 id -= idr->idr_base;
301295
302296 entry = __radix_tree_lookup(&idr->idr_rt, id, &node, &slot);
303297 if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE))
304298 return ERR_PTR(-ENOENT);
305299
306− __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL);
300+ __radix_tree_replace(&idr->idr_rt, node, slot, ptr);
307301
308302 return entry;
309303 }
@@ -324,6 +318,9 @@ EXPORT_SYMBOL(idr_replace); · 14 unmodified lines
324318 * free the individual IDs in it. You can use ida_is_empty() to find
325319 * out whether the IDA has any IDs currently allocated.
326320 *
321+ * The IDA handles its own locking. It is safe to call any of the IDA
322+ * functions without synchronisation in your code.
323+ *
327324 * IDs are currently limited to the range [0-INT_MAX]. If this is an awkward
328325 * limitation, it should be quite straightforward to raise the maximum.
329326 */
@@ -331,161 +328,197 @@ EXPORT_SYMBOL(idr_replace); · 1 unmodified line
331328 /*
332329 * Developer's notes:
333330 *
334− * The IDA uses the functionality provided by the IDR & radix tree to store
335− * bitmaps in each entry. The IDR_FREE tag means there is at least one bit
336− * free, unlike the IDR where it means at least one entry is free.
331+ * The IDA uses the functionality provided by the XArray to store bitmaps in
332+ * each entry. The XA_FREE_MARK is only cleared when all bits in the bitmap
333+ * have been set.
337334 *
338− * I considered telling the radix tree that each slot is an order-10 node
339− * and storing the bit numbers in the radix tree, but the radix tree can't
340− * allow a single multiorder entry at index 0, which would significantly
341− * increase memory consumption for the IDA. So instead we divide the index
342− * by the number of bits in the leaf bitmap before doing a radix tree lookup.
335+ * I considered telling the XArray that each slot is an order-10 node
336+ * and indexing by bit number, but the XArray can't allow a single multi-index
337+ * entry in the head, which would significantly increase memory consumption
338+ * for the IDA. So instead we divide the index by the number of bits in the
339+ * leaf bitmap before doing a radix tree lookup.
343340 *
344341 * As an optimisation, if there are only a few low bits set in any given
345− * leaf, instead of allocating a 128-byte bitmap, we use the 'exceptional
346− * entry' functionality of the radix tree to store BITS_PER_LONG - 2 bits
347− * directly in the entry. By being really tricksy, we could store
348− * BITS_PER_LONG - 1 bits, but there're diminishing returns after optimising
349− * for 0-3 allocated IDs.
350− *
351− * We allow the radix tree 'exceptional' count to get out of date. Nothing
352− * in the IDA nor the radix tree code checks it. If it becomes important
353− * to maintain an accurate exceptional count, switch the rcu_assign_pointer()
354− * calls to radix_tree_iter_replace() which will correct the exceptional
355− * count.
356− *
357− * The IDA always requires a lock to alloc/free. If we add a 'test_bit'
342+ * leaf, instead of allocating a 128-byte bitmap, we store the bits
343+ * as a value entry. Value entries never have the XA_FREE_MARK cleared
344+ * because we can always convert them into a bitmap entry.
345+ *
346+ * It would be possible to optimise further; once we've run out of a
347+ * single 128-byte bitmap, we currently switch to a 576-byte node, put
348+ * the 128-byte bitmap in the first entry and then start allocating extra
349+ * 128-byte entries. We could instead use the 512 bytes of the node's
350+ * data as a bitmap before moving to that scheme. I do not believe this
351+ * is a worthwhile optimisation; Rasmus Villemoes surveyed the current
352+ * users of the IDA and almost none of them use more than 1024 entries.
353+ * Those that do use more than the 8192 IDs that the 512 bytes would
354+ * provide.
355+ *
356+ * The IDA always uses a lock to alloc/free. If we add a 'test_bit'
358357 * equivalent, it will still need locking. Going to RCU lookup would require
359358 * using RCU to free bitmaps, and that's not trivial without embedding an
360359 * RCU head in the bitmap, which adds a 2-pointer overhead to each 128-byte
361360 * bitmap, which is excessive.
362361 */
363362
364−#define IDA_MAX (0x80000000U / IDA_BITMAP_BITS - 1)
365−
366−static int ida_get_new_above(struct ida *ida, int start)
363+/**
364+ * ida_alloc_range() - Allocate an unused ID.
365+ * @ida: IDA handle.
366+ * @min: Lowest ID to allocate.
367+ * @max: Highest ID to allocate.
368+ * @gfp: Memory allocation flags.
369+ *
370+ * Allocate an ID between @min and @max, inclusive. The allocated ID will
371+ * not exceed %INT_MAX, even if @max is larger.
372+ *
373+ * Context: Any context.
374+ * Return: The allocated ID, or %-ENOMEM if memory could not be allocated,
375+ * or %-ENOSPC if there are no free IDs.
376+ */
377+int ida_alloc_range(struct ida *ida, unsigned int min, unsigned int max,
378+ gfp_t gfp)
367379 {
368− struct radix_tree_root *root = &ida->ida_rt;
369− void __rcu **slot;
370− struct radix_tree_iter iter;
371− struct ida_bitmap *bitmap;
372− unsigned long index;
373− unsigned bit, ebit;
374− int new;
375−
376− index = start / IDA_BITMAP_BITS;
377− bit = start % IDA_BITMAP_BITS;
378− ebit = bit + RADIX_TREE_EXCEPTIONAL_SHIFT;
379−
380− slot = radix_tree_iter_init(&iter, index);
381− for (;;) {
382− if (slot)
383− slot = radix_tree_next_slot(slot, &iter,
384− RADIX_TREE_ITER_TAGGED);
385− if (!slot) {
386− slot = idr_get_free(root, &iter, GFP_NOWAIT, IDA_MAX);
387− if (IS_ERR(slot)) {
388− if (slot == ERR_PTR(-ENOMEM))
389− return -EAGAIN;
390− return PTR_ERR(slot);
380+ XA_STATE(xas, &ida->xa, min / IDA_BITMAP_BITS);
381+ unsigned bit = min % IDA_BITMAP_BITS;
382+ unsigned long flags;
383+ struct ida_bitmap *bitmap, *alloc = NULL;
384+
385+ if ((int)min < 0)
386+ return -ENOSPC;
387+
388+ if ((int)max < 0)
389+ max = INT_MAX;
390+
391+retry:
392+ xas_lock_irqsave(&xas, flags);
393+next:
394+ bitmap = xas_find_marked(&xas, max / IDA_BITMAP_BITS, XA_FREE_MARK);
395+ if (xas.xa_index > min / IDA_BITMAP_BITS)
396+ bit = 0;
397+ if (xas.xa_index * IDA_BITMAP_BITS + bit > max)
398+ goto nospc;
399+
400+ if (xa_is_value(bitmap)) {
401+ unsigned long tmp = xa_to_value(bitmap);
402+
403+ if (bit < BITS_PER_XA_VALUE) {
404+ bit = find_next_zero_bit(&tmp, BITS_PER_XA_VALUE, bit);
405+ if (xas.xa_index * IDA_BITMAP_BITS + bit > max)
406+ goto nospc;
407+ if (bit < BITS_PER_XA_VALUE) {
408+ tmp |= 1UL << bit;
409+ xas_store(&xas, xa_mk_value(tmp));
410+ goto out;
391411 }
392412 }
393− if (iter.index > index) {
394− bit = 0;
395− ebit = RADIX_TREE_EXCEPTIONAL_SHIFT;
396− }
397− new = iter.index * IDA_BITMAP_BITS;
398− bitmap = rcu_dereference_raw(*slot);
399− if (radix_tree_exception(bitmap)) {
400− unsigned long tmp = (unsigned long)bitmap;
401− ebit = find_next_zero_bit(&tmp, BITS_PER_LONG, ebit);
402− if (ebit < BITS_PER_LONG) {
403− tmp |= 1UL << ebit;
404− rcu_assign_pointer(*slot, (void *)tmp);
405− return new + ebit -
406− RADIX_TREE_EXCEPTIONAL_SHIFT;
407− }
408− bitmap = this_cpu_xchg(ida_bitmap, NULL);
409− if (!bitmap)
410− return -EAGAIN;
411− bitmap->bitmap[0] = tmp >> RADIX_TREE_EXCEPTIONAL_SHIFT;
412− rcu_assign_pointer(*slot, bitmap);
413+ bitmap = alloc;
414+ if (!bitmap)
415+ bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT);
416+ if (!bitmap)
417+ goto alloc;
418+ bitmap->bitmap[0] = tmp;
419+ xas_store(&xas, bitmap);
420+ if (xas_error(&xas)) {
421+ bitmap->bitmap[0] = 0;
422+ goto out;
413423 }
424+ }
414425
415− if (bitmap) {
416− bit = find_next_zero_bit(bitmap->bitmap,
417− IDA_BITMAP_BITS, bit);
418− new += bit;
419− if (new < 0)
420− return -ENOSPC;
421− if (bit == IDA_BITMAP_BITS)
422− continue;
426+ if (bitmap) {
427+ bit = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, bit);
428+ if (xas.xa_index * IDA_BITMAP_BITS + bit > max)
429+ goto nospc;
430+ if (bit == IDA_BITMAP_BITS)
431+ goto next;
423432
424− __set_bit(bit, bitmap->bitmap);
425− if (bitmap_full(bitmap->bitmap, IDA_BITMAP_BITS))
426− radix_tree_iter_tag_clear(root, &iter,
427− IDR_FREE);
433+ __set_bit(bit, bitmap->bitmap);
434+ if (bitmap_full(bitmap->bitmap, IDA_BITMAP_BITS))
435+ xas_clear_mark(&xas, XA_FREE_MARK);
436+ } else {
437+ if (bit < BITS_PER_XA_VALUE) {
438+ bitmap = xa_mk_value(1UL << bit);
428439 } else {
429− new += bit;
430− if (new < 0)
431− return -ENOSPC;
432− if (ebit < BITS_PER_LONG) {
433− bitmap = (void *)((1UL << ebit) |
434− RADIX_TREE_EXCEPTIONAL_ENTRY);
435− radix_tree_iter_replace(root, &iter, slot,
436− bitmap);
437− return new;
438− }
439− bitmap = this_cpu_xchg(ida_bitmap, NULL);
440+ bitmap = alloc;
440441 if (!bitmap)
441− return -EAGAIN;
442+ bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT);
443+ if (!bitmap)
444+ goto alloc;
442445 __set_bit(bit, bitmap->bitmap);
443− radix_tree_iter_replace(root, &iter, slot, bitmap);
444446 }
445−
446− return new;
447+ xas_store(&xas, bitmap);
448+ }
449+out:
450+ xas_unlock_irqrestore(&xas, flags);
451+ if (xas_nomem(&xas, gfp)) {
452+ xas.xa_index = min / IDA_BITMAP_BITS;
453+ bit = min % IDA_BITMAP_BITS;
454+ goto retry;
447455 }
456+ if (bitmap != alloc)
457+ kfree(alloc);
458+ if (xas_error(&xas))
459+ return xas_error(&xas);
460+ return xas.xa_index * IDA_BITMAP_BITS + bit;
461+alloc:
462+ xas_unlock_irqrestore(&xas, flags);
463+ alloc = kzalloc(sizeof(*bitmap), gfp);
464+ if (!alloc)
465+ return -ENOMEM;
466+ xas_set(&xas, min / IDA_BITMAP_BITS);
467+ bit = min % IDA_BITMAP_BITS;
468+ goto retry;
469+nospc:
470+ xas_unlock_irqrestore(&xas, flags);
471+ return -ENOSPC;
448472 }
473+EXPORT_SYMBOL(ida_alloc_range);
449474
450−static void ida_remove(struct ida *ida, int id)
475+/**
476+ * ida_free() - Release an allocated ID.
477+ * @ida: IDA handle.
478+ * @id: Previously allocated ID.
479+ *
480+ * Context: Any context.
481+ */
482+void ida_free(struct ida *ida, unsigned int id)
451483 {
452− unsigned long index = id / IDA_BITMAP_BITS;
453− unsigned offset = id % IDA_BITMAP_BITS;
484+ XA_STATE(xas, &ida->xa, id / IDA_BITMAP_BITS);
485+ unsigned bit = id % IDA_BITMAP_BITS;
454486 struct ida_bitmap *bitmap;
455− unsigned long *btmp;
456− struct radix_tree_iter iter;
457− void __rcu **slot;
487+ unsigned long flags;
458488
459− slot = radix_tree_iter_lookup(&ida->ida_rt, &iter, index);
460− if (!slot)
461− goto err;
489+ BUG_ON((int)id < 0);
490+
491+ xas_lock_irqsave(&xas, flags);
492+ bitmap = xas_load(&xas);
462493
463− bitmap = rcu_dereference_raw(*slot);
464− if (radix_tree_exception(bitmap)) {
465− btmp = (unsigned long *)slot;
466− offset += RADIX_TREE_EXCEPTIONAL_SHIFT;
467− if (offset >= BITS_PER_LONG)
494+ if (xa_is_value(bitmap)) {
495+ unsigned long v = xa_to_value(bitmap);
496+ if (bit >= BITS_PER_XA_VALUE)
468497 goto err;
498+ if (!(v & (1UL << bit)))
499+ goto err;
500+ v &= ~(1UL << bit);
501+ if (!v)
502+ goto delete;
503+ xas_store(&xas, xa_mk_value(v));
469504 } else {
470− btmp = bitmap->bitmap;
471− }
472− if (!test_bit(offset, btmp))
473− goto err;
474−
475− __clear_bit(offset, btmp);
476− radix_tree_iter_tag_set(&ida->ida_rt, &iter, IDR_FREE);
477− if (radix_tree_exception(bitmap)) {
478− if (rcu_dereference_raw(*slot) ==
479− (void *)RADIX_TREE_EXCEPTIONAL_ENTRY)
480− radix_tree_iter_delete(&ida->ida_rt, &iter, slot);
481− } else if (bitmap_empty(btmp, IDA_BITMAP_BITS)) {
482− kfree(bitmap);
483− radix_tree_iter_delete(&ida->ida_rt, &iter, slot);
505+ if (!test_bit(bit, bitmap->bitmap))
506+ goto err;
507+ __clear_bit(bit, bitmap->bitmap);
508+ xas_set_mark(&xas, XA_FREE_MARK);
509+ if (bitmap_empty(bitmap->bitmap, IDA_BITMAP_BITS)) {
510+ kfree(bitmap);
511+delete:
512+ xas_store(&xas, NULL);
513+ }
484514 }
515+ xas_unlock_irqrestore(&xas, flags);
485516 return;
486517 err:
518+ xas_unlock_irqrestore(&xas, flags);
487519 WARN(1, "ida_free called for id=%d which is not allocated.\n", id);
488520 }
521+EXPORT_SYMBOL(ida_free);
489522
490523 /**
491524 * ida_destroy() - Free all IDs.
@@ -500,80 +533,60 @@ static void ida_remove(struct ida *ida, int id) · 8 unmodified lines
500533 */
501534 void ida_destroy(struct ida *ida)
502535 {
536+ XA_STATE(xas, &ida->xa, 0);
537+ struct ida_bitmap *bitmap;
503538 unsigned long flags;
504− struct radix_tree_iter iter;
505− void __rcu **slot;
506539
507− xa_lock_irqsave(&ida->ida_rt, flags);
508− radix_tree_for_each_slot(slot, &ida->ida_rt, &iter, 0) {
509− struct ida_bitmap *bitmap = rcu_dereference_raw(*slot);
510− if (!radix_tree_exception(bitmap))
540+ xas_lock_irqsave(&xas, flags);
541+ xas_for_each(&xas, bitmap, ULONG_MAX) {
542+ if (!xa_is_value(bitmap))
511543 kfree(bitmap);
512− radix_tree_iter_delete(&ida->ida_rt, &iter, slot);
544+ xas_store(&xas, NULL);
513545 }
514− xa_unlock_irqrestore(&ida->ida_rt, flags);
546+ xas_unlock_irqrestore(&xas, flags);
515547 }
516548 EXPORT_SYMBOL(ida_destroy);
517549
518−/**
519− * ida_alloc_range() - Allocate an unused ID.
520− * @ida: IDA handle.
521− * @min: Lowest ID to allocate.
522− * @max: Highest ID to allocate.
523− * @gfp: Memory allocation flags.
524− *
525− * Allocate an ID between @min and @max, inclusive. The allocated ID will
526− * not exceed %INT_MAX, even if @max is larger.
527− *
528− * Context: Any context.
529− * Return: The allocated ID, or %-ENOMEM if memory could not be allocated,
530− * or %-ENOSPC if there are no free IDs.
531− */
532−int ida_alloc_range(struct ida *ida, unsigned int min, unsigned int max,
533− gfp_t gfp)
534−{
535− int id = 0;
536− unsigned long flags;
550+#ifndef __KERNEL__
551+extern void xa_dump_index(unsigned long index, unsigned int shift);
552+#define IDA_CHUNK_SHIFT ilog2(IDA_BITMAP_BITS)
537553
538− if ((int)min < 0)
539− return -ENOSPC;
540−
541− if ((int)max < 0)
542− max = INT_MAX;
543−
544−again:
545− xa_lock_irqsave(&ida->ida_rt, flags);
546− id = ida_get_new_above(ida, min);
547− if (id > (int)max) {
548− ida_remove(ida, id);
549− id = -ENOSPC;
550− }
551− xa_unlock_irqrestore(&ida->ida_rt, flags);
554+static void ida_dump_entry(void *entry, unsigned long index)
555+{
556+ unsigned long i;
557+
558+ if (!entry)
559+ return;
560+
561+ if (xa_is_node(entry)) {
562+ struct xa_node *node = xa_to_node(entry);
563+ unsigned int shift = node->shift + IDA_CHUNK_SHIFT +
564+ XA_CHUNK_SHIFT;
565+
566+ xa_dump_index(index * IDA_BITMAP_BITS, shift);
567+ xa_dump_node(node);
568+ for (i = 0; i < XA_CHUNK_SIZE; i++)
569+ ida_dump_entry(node->slots[i],
570+ index | (i << node->shift));
571+ } else if (xa_is_value(entry)) {
572+ xa_dump_index(index * IDA_BITMAP_BITS, ilog2(BITS_PER_LONG));
573+ pr_cont("value: data %lx [%px]\n", xa_to_value(entry), entry);
574+ } else {
575+ struct ida_bitmap *bitmap = entry;
552576
553− if (unlikely(id == -EAGAIN)) {
554− if (!ida_pre_get(ida, gfp))
555− return -ENOMEM;
556− goto again;
577+ xa_dump_index(index * IDA_BITMAP_BITS, IDA_CHUNK_SHIFT);
578+ pr_cont("bitmap: %p data", bitmap);
579+ for (i = 0; i < IDA_BITMAP_LONGS; i++)
580+ pr_cont(" %lx", bitmap->bitmap[i]);
581+ pr_cont("\n");
557582 }
558−
559− return id;
560583 }
561−EXPORT_SYMBOL(ida_alloc_range);
562584
563−/**
564− * ida_free() - Release an allocated ID.
565− * @ida: IDA handle.
566− * @id: Previously allocated ID.
567− *
568− * Context: Any context.
569− */
570−void ida_free(struct ida *ida, unsigned int id)
585+static void ida_dump(struct ida *ida)
571586 {
572− unsigned long flags;
573−
574− BUG_ON((int)id < 0);
575− xa_lock_irqsave(&ida->ida_rt, flags);
576− ida_remove(ida, id);
577− xa_unlock_irqrestore(&ida->ida_rt, flags);
587+ struct xarray *xa = &ida->xa;
588+ pr_debug("ida: %p node %p free %d\n", ida, xa->xa_head,
589+ xa->xa_flags >> ROOT_TAG_SHIFT);
590+ ida_dump_entry(xa->xa_head, 0);
578591 }
579−EXPORT_SYMBOL(ida_free);
592+#endif
lib/radix-tree.c+88−746
@@ -38,15 +38,13 @@ · 37 unmodified lines
3838 #include <linux/rcupdate.h>
3939 #include <linux/slab.h>
4040 #include <linux/string.h>
41+#include <linux/xarray.h>
4142
4243
43−/* Number of nodes in fully populated tree of given height */
44−static unsigned long height_to_maxnodes[RADIX_TREE_MAX_PATH + 1] __read_mostly;
45−
4644 /*
4745 * Radix tree node cache.
4846 */
49−static struct kmem_cache *radix_tree_node_cachep;
47+struct kmem_cache *radix_tree_node_cachep;
5048
5149 /*
5250 * The radix tree is variable-height, so an insert operation not only has
@@ -98,24 +96,7 @@ static inline void *node_to_entry(void *ptr) · 45 unmodified lines
9896 return (void *)((unsigned long)ptr | RADIX_TREE_INTERNAL_NODE);
9997 }
10098
101−#define RADIX_TREE_RETRY node_to_entry(NULL)
102−
103−#ifdef CONFIG_RADIX_TREE_MULTIORDER
104−/* Sibling slots point directly to another slot in the same node */
105−static inline
106−bool is_sibling_entry(const struct radix_tree_node *parent, void *node)
107−{
108− void __rcu **ptr = node;
109− return (parent->slots <= ptr) &&
110− (ptr < parent->slots + RADIX_TREE_MAP_SIZE);
111−}
112−#else
113−static inline
114−bool is_sibling_entry(const struct radix_tree_node *parent, void *node)
115−{
116− return false;
117−}
118−#endif
99+#define RADIX_TREE_RETRY XA_RETRY_ENTRY
119100
120101 static inline unsigned long
121102 get_slot_offset(const struct radix_tree_node *parent, void __rcu **slot)
@@ -129,24 +110,13 @@ static unsigned int radix_tree_descend(const struct radix_tree_node *parent, · 7 unmodified lines
129110 unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK;
130111 void __rcu **entry = rcu_dereference_raw(parent->slots[offset]);
131112
132−#ifdef CONFIG_RADIX_TREE_MULTIORDER
133− if (radix_tree_is_internal_node(entry)) {
134− if (is_sibling_entry(parent, entry)) {
135− void __rcu **sibentry;
136− sibentry = (void __rcu **) entry_to_node(entry);
137− offset = get_slot_offset(parent, sibentry);
138− entry = rcu_dereference_raw(*sibentry);
139− }
140− }
141−#endif
142−
143113 *nodep = (void *)entry;
144114 return offset;
145115 }
146116
147117 static inline gfp_t root_gfp_mask(const struct radix_tree_root *root)
148118 {
149− return root->gfp_mask & (__GFP_BITS_MASK & ~GFP_ZONEMASK);
119+ return root->xa_flags & (__GFP_BITS_MASK & ~GFP_ZONEMASK);
150120 }
151121
152122 static inline void tag_set(struct radix_tree_node *node, unsigned int tag,
@@ -169,32 +139,32 @@ static inline int tag_get(const struct radix_tree_node *node, unsigned int tag, · 16 unmodified lines
169139
170140 static inline void root_tag_set(struct radix_tree_root *root, unsigned tag)
171141 {
172− root->gfp_mask |= (__force gfp_t)(1 << (tag + ROOT_TAG_SHIFT));
142+ root->xa_flags |= (__force gfp_t)(1 << (tag + ROOT_TAG_SHIFT));
173143 }
174144
175145 static inline void root_tag_clear(struct radix_tree_root *root, unsigned tag)
176146 {
177− root->gfp_mask &= (__force gfp_t)~(1 << (tag + ROOT_TAG_SHIFT));
147+ root->xa_flags &= (__force gfp_t)~(1 << (tag + ROOT_TAG_SHIFT));
178148 }
179149
180150 static inline void root_tag_clear_all(struct radix_tree_root *root)
181151 {
182− root->gfp_mask &= (1 << ROOT_TAG_SHIFT) - 1;
152+ root->xa_flags &= (__force gfp_t)((1 << ROOT_TAG_SHIFT) - 1);
183153 }
184154
185155 static inline int root_tag_get(const struct radix_tree_root *root, unsigned tag)
186156 {
187− return (__force int)root->gfp_mask & (1 << (tag + ROOT_TAG_SHIFT));
157+ return (__force int)root->xa_flags & (1 << (tag + ROOT_TAG_SHIFT));
188158 }
189159
190160 static inline unsigned root_tags_get(const struct radix_tree_root *root)
191161 {
192− return (__force unsigned)root->gfp_mask >> ROOT_TAG_SHIFT;
162+ return (__force unsigned)root->xa_flags >> ROOT_TAG_SHIFT;
193163 }
194164
195165 static inline bool is_idr(const struct radix_tree_root *root)
196166 {
197− return !!(root->gfp_mask & ROOT_IS_IDR);
167+ return !!(root->xa_flags & ROOT_IS_IDR);
198168 }
199169
200170 /*
@@ -254,7 +224,7 @@ radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag, · 53 unmodified lines
254224
255225 static unsigned int iter_offset(const struct radix_tree_iter *iter)
256226 {
257− return (iter->index >> iter_shift(iter)) & RADIX_TREE_MAP_MASK;
227+ return iter->index & RADIX_TREE_MAP_MASK;
258228 }
259229
260230 /*
@@ -277,99 +247,6 @@ static unsigned long next_index(unsigned long index, · 16 unmodified lines
277247 return (index & ~node_maxindex(node)) + (offset << node->shift);
278248 }
279249
280−#ifndef __KERNEL__
281−static void dump_node(struct radix_tree_node *node, unsigned long index)
282−{
283− unsigned long i;
284−
285− pr_debug("radix node: %p offset %d indices %lu-%lu parent %p tags %lx %lx %lx shift %d count %d exceptional %d\n",
286− node, node->offset, index, index | node_maxindex(node),
287− node->parent,
288− node->tags[0][0], node->tags[1][0], node->tags[2][0],
289− node->shift, node->count, node->exceptional);
290−
291− for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) {
292− unsigned long first = index | (i << node->shift);
293− unsigned long last = first | ((1UL << node->shift) - 1);
294− void *entry = node->slots[i];
295− if (!entry)
296− continue;
297− if (entry == RADIX_TREE_RETRY) {
298− pr_debug("radix retry offset %ld indices %lu-%lu parent %p\n",
299− i, first, last, node);
300− } else if (!radix_tree_is_internal_node(entry)) {
301− pr_debug("radix entry %p offset %ld indices %lu-%lu parent %p\n",
302− entry, i, first, last, node);
303− } else if (is_sibling_entry(node, entry)) {
304− pr_debug("radix sblng %p offset %ld indices %lu-%lu parent %p val %p\n",
305− entry, i, first, last, node,
306− *(void **)entry_to_node(entry));
307− } else {
308− dump_node(entry_to_node(entry), first);
309− }
310− }
311−}
312−
313−/* For debug */
314−static void radix_tree_dump(struct radix_tree_root *root)
315−{
316− pr_debug("radix root: %p rnode %p tags %x\n",
317− root, root->rnode,
318− root->gfp_mask >> ROOT_TAG_SHIFT);
319− if (!radix_tree_is_internal_node(root->rnode))
320− return;
321− dump_node(entry_to_node(root->rnode), 0);
322−}
323−
324−static void dump_ida_node(void *entry, unsigned long index)
325−{
326− unsigned long i;
327−
328− if (!entry)
329− return;
330−
331− if (radix_tree_is_internal_node(entry)) {
332− struct radix_tree_node *node = entry_to_node(entry);
333−
334− pr_debug("ida node: %p offset %d indices %lu-%lu parent %p free %lx shift %d count %d\n",
335− node, node->offset, index * IDA_BITMAP_BITS,
336− ((index | node_maxindex(node)) + 1) *
337− IDA_BITMAP_BITS - 1,
338− node->parent, node->tags[0][0], node->shift,
339− node->count);
340− for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
341− dump_ida_node(node->slots[i],
342− index | (i << node->shift));
343− } else if (radix_tree_exceptional_entry(entry)) {
344− pr_debug("ida excp: %p offset %d indices %lu-%lu data %lx\n",
345− entry, (int)(index & RADIX_TREE_MAP_MASK),
346− index * IDA_BITMAP_BITS,
347− index * IDA_BITMAP_BITS + BITS_PER_LONG -
348− RADIX_TREE_EXCEPTIONAL_SHIFT,
349− (unsigned long)entry >>
350− RADIX_TREE_EXCEPTIONAL_SHIFT);
351− } else {
352− struct ida_bitmap *bitmap = entry;
353−
354− pr_debug("ida btmp: %p offset %d indices %lu-%lu data", bitmap,
355− (int)(index & RADIX_TREE_MAP_MASK),
356− index * IDA_BITMAP_BITS,
357− (index + 1) * IDA_BITMAP_BITS - 1);
358− for (i = 0; i < IDA_BITMAP_LONGS; i++)
359− pr_cont(" %lx", bitmap->bitmap[i]);
360− pr_cont("\n");
361− }
362−}
363−
364−static void ida_dump(struct ida *ida)
365−{
366− struct radix_tree_root *root = &ida->ida_rt;
367− pr_debug("ida: %p node %p free %d\n", ida, root->rnode,
368− root->gfp_mask >> ROOT_TAG_SHIFT);
369− dump_ida_node(root->rnode, 0);
370−}
371−#endif
372−
373250 /*
374251 * This assumes that the caller has performed appropriate preallocation, and
375252 * that the caller has pinned this thread of control to the current CPU.
@@ -378,7 +255,7 @@ static struct radix_tree_node * · 2 unmodified lines
378255 radix_tree_node_alloc(gfp_t gfp_mask, struct radix_tree_node *parent,
379256 struct radix_tree_root *root,
380257 unsigned int shift, unsigned int offset,
381− unsigned int count, unsigned int exceptional)
258+ unsigned int count, unsigned int nr_values)
382259 {
383260 struct radix_tree_node *ret = NULL;
384261
@@ -425,14 +302,14 @@ out: · 40 unmodified lines
425302 ret->shift = shift;
426303 ret->offset = offset;
427304 ret->count = count;
428− ret->exceptional = exceptional;
305+ ret->nr_values = nr_values;
429306 ret->parent = parent;
430− ret->root = root;
307+ ret->array = root;
431308 }
432309 return ret;
433310 }
434311
435−static void radix_tree_node_rcu_free(struct rcu_head *head)
312+void radix_tree_node_rcu_free(struct rcu_head *head)
436313 {
437314 struct radix_tree_node *node =
438315 container_of(head, struct radix_tree_node, rcu_head);
@@ -530,77 +407,10 @@ int radix_tree_maybe_preload(gfp_t gfp_mask) · 91 unmodified lines
530407 }
531408 EXPORT_SYMBOL(radix_tree_maybe_preload);
532409
533−#ifdef CONFIG_RADIX_TREE_MULTIORDER
534−/*
535− * Preload with enough objects to ensure that we can split a single entry
536− * of order @old_order into many entries of size @new_order
537− */
538−int radix_tree_split_preload(unsigned int old_order, unsigned int new_order,
539− gfp_t gfp_mask)
540−{
541− unsigned top = 1 << (old_order % RADIX_TREE_MAP_SHIFT);
542− unsigned layers = (old_order / RADIX_TREE_MAP_SHIFT) -
543− (new_order / RADIX_TREE_MAP_SHIFT);
544− unsigned nr = 0;
545−
546− WARN_ON_ONCE(!gfpflags_allow_blocking(gfp_mask));
547− BUG_ON(new_order >= old_order);
548−
549− while (layers--)
550− nr = nr * RADIX_TREE_MAP_SIZE + 1;
551− return __radix_tree_preload(gfp_mask, top * nr);
552−}
553−#endif
554−
555−/*
556− * The same as function above, but preload number of nodes required to insert
557− * (1 << order) continuous naturally-aligned elements.
558− */
559−int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order)
560−{
561− unsigned long nr_subtrees;
562− int nr_nodes, subtree_height;
563−
564− /* Preloading doesn't help anything with this gfp mask, skip it */
565− if (!gfpflags_allow_blocking(gfp_mask)) {
566− preempt_disable();
567− return 0;
568− }
569−
570− /*
571− * Calculate number and height of fully populated subtrees it takes to
572− * store (1 << order) elements.
573− */
574− nr_subtrees = 1 << order;
575− for (subtree_height = 0; nr_subtrees > RADIX_TREE_MAP_SIZE;
576− subtree_height++)
577− nr_subtrees >>= RADIX_TREE_MAP_SHIFT;
578−
579− /*
580− * The worst case is zero height tree with a single item at index 0 and
581− * then inserting items starting at ULONG_MAX - (1 << order).
582− *
583− * This requires RADIX_TREE_MAX_PATH nodes to build branch from root to
584− * 0-index item.
585− */
586− nr_nodes = RADIX_TREE_MAX_PATH;
587−
588− /* Plus branch to fully populated subtrees. */
589− nr_nodes += RADIX_TREE_MAX_PATH - subtree_height;
590−
591− /* Root node is shared. */
592− nr_nodes--;
593−
594− /* Plus nodes required to build subtrees. */
595− nr_nodes += nr_subtrees * height_to_maxnodes[subtree_height];
596−
597− return __radix_tree_preload(gfp_mask, nr_nodes);
598−}
599−
600410 static unsigned radix_tree_load_root(const struct radix_tree_root *root,
601411 struct radix_tree_node **nodep, unsigned long *maxindex)
602412 {
603− struct radix_tree_node *node = rcu_dereference_raw(root->rnode);
413+ struct radix_tree_node *node = rcu_dereference_raw(root->xa_head);
604414
605415 *nodep = node;
606416
@@ -629,7 +439,7 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, · 22 unmodified lines
629439 while (index > shift_maxindex(maxshift))
630440 maxshift += RADIX_TREE_MAP_SHIFT;
631441
632− entry = rcu_dereference_raw(root->rnode);
442+ entry = rcu_dereference_raw(root->xa_head);
633443 if (!entry && (!is_idr(root) || root_tag_get(root, IDR_FREE)))
634444 goto out;
635445
@@ -656,9 +466,9 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, · 20 unmodified lines
656466 BUG_ON(shift > BITS_PER_LONG);
657467 if (radix_tree_is_internal_node(entry)) {
658468 entry_to_node(entry)->parent = node;
659− } else if (radix_tree_exceptional_entry(entry)) {
660− /* Moving an exceptional root->rnode to a node */
661− node->exceptional = 1;
469+ } else if (xa_is_value(entry)) {
470+ /* Moving a value entry root->xa_head to a node */
471+ node->nr_values = 1;
662472 }
663473 /*
664474 * entry was already in the radix tree, so we do not need
@@ -666,7 +476,7 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, · 1 unmodified line
666476 */
667477 node->slots[0] = (void __rcu *)entry;
668478 entry = node_to_entry(node);
669− rcu_assign_pointer(root->rnode, entry);
479+ rcu_assign_pointer(root->xa_head, entry);
670480 shift += RADIX_TREE_MAP_SHIFT;
671481 } while (shift <= maxshift);
672482 out:
@@ -677,13 +487,12 @@ out: · 4 unmodified lines
677487 * radix_tree_shrink - shrink radix tree to minimum height
678488 * @root radix tree root
679489 */
680−static inline bool radix_tree_shrink(struct radix_tree_root *root,
681− radix_tree_update_node_t update_node)
490+static inline bool radix_tree_shrink(struct radix_tree_root *root)
682491 {
683492 bool shrunk = false;
684493
685494 for (;;) {
686− struct radix_tree_node *node = rcu_dereference_raw(root->rnode);
495+ struct radix_tree_node *node = rcu_dereference_raw(root->xa_head);
687496 struct radix_tree_node *child;
688497
689498 if (!radix_tree_is_internal_node(node))
@@ -692,15 +501,20 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, · 2 unmodified lines
692501
693502 /*
694503 * The candidate node has more than one child, or its child
695− * is not at the leftmost slot, or the child is a multiorder
696− * entry, we cannot shrink.
504+ * is not at the leftmost slot, we cannot shrink.
697505 */
698506 if (node->count != 1)
699507 break;
700508 child = rcu_dereference_raw(node->slots[0]);
701509 if (!child)
702510 break;
703− if (!radix_tree_is_internal_node(child) && node->shift)
511+
512+ /*
513+ * For an IDR, we must not shrink entry 0 into the root in
514+ * case somebody calls idr_replace() with a pointer that
515+ * appears to be an internal entry
516+ */
517+ if (!node->shift && is_idr(root))
704518 break;
705519
706520 if (radix_tree_is_internal_node(child))
@@ -711,9 +525,9 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, · 4 unmodified lines
711525 * moving the node from one part of the tree to another: if it
712526 * was safe to dereference the old pointer to it
713527 * (node->slots[0]), it will be safe to dereference the new
714− * one (root->rnode) as far as dependent read barriers go.
528+ * one (root->xa_head) as far as dependent read barriers go.
715529 */
716− root->rnode = (void __rcu *)child;
530+ root->xa_head = (void __rcu *)child;
717531 if (is_idr(root) && !tag_get(node, IDR_FREE, 0))
718532 root_tag_clear(root, IDR_FREE);
719533
@@ -738,8 +552,6 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, · 18 unmodified lines
738552 node->count = 0;
739553 if (!radix_tree_is_internal_node(child)) {
740554 node->slots[0] = (void __rcu *)RADIX_TREE_RETRY;
741− if (update_node)
742− update_node(node);
743555 }
744556
745557 WARN_ON_ONCE(!list_empty(&node->private_list));
@@ -751,8 +563,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, · 5 unmodified lines
751563 }
752564
753565 static bool delete_node(struct radix_tree_root *root,
754− struct radix_tree_node *node,
755− radix_tree_update_node_t update_node)
566+ struct radix_tree_node *node)
756567 {
757568 bool deleted = false;
758569
@@ -761,9 +572,8 @@ static bool delete_node(struct radix_tree_root *root, · 2 unmodified lines
761572
762573 if (node->count) {
763574 if (node_to_entry(node) ==
764− rcu_dereference_raw(root->rnode))
765− deleted |= radix_tree_shrink(root,
766− update_node);
575+ rcu_dereference_raw(root->xa_head))
576+ deleted |= radix_tree_shrink(root);
767577 return deleted;
768578 }
769579
@@ -778,7 +588,7 @@ static bool delete_node(struct radix_tree_root *root, · 8 unmodified lines
778588 */
779589 if (!is_idr(root))
780590 root_tag_clear_all(root);
781− root->rnode = NULL;
591+ root->xa_head = NULL;
782592 }
783593
784594 WARN_ON_ONCE(!list_empty(&node->private_list));
@@ -795,7 +605,6 @@ static bool delete_node(struct radix_tree_root *root, · 10 unmodified lines
795605 * __radix_tree_create - create a slot in a radix tree
796606 * @root: radix tree root
797607 * @index: index key
798− * @order: index occupies 2^order aligned slots
799608 * @nodep: returns node
800609 * @slotp: returns slot
801610 *
@@ -803,36 +612,34 @@ static bool delete_node(struct radix_tree_root *root, · 1 unmodified line
803612 * at position @index in the radix tree @root.
804613 *
805614 * Until there is more than one item in the tree, no nodes are
806− * allocated and @root->rnode is used as a direct slot instead of
615+ * allocated and @root->xa_head is used as a direct slot instead of
807616 * pointing to a node, in which case *@nodep will be NULL.
808617 *
809618 * Returns -ENOMEM, or 0 for success.
810619 */
811−int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
812− unsigned order, struct radix_tree_node **nodep,
813− void __rcu ***slotp)
620+static int __radix_tree_create(struct radix_tree_root *root,
621+ unsigned long index, struct radix_tree_node **nodep,
622+ void __rcu ***slotp)
814623 {
815624 struct radix_tree_node *node = NULL, *child;
816− void __rcu **slot = (void __rcu **)&root->rnode;
625+ void __rcu **slot = (void __rcu **)&root->xa_head;
817626 unsigned long maxindex;
818627 unsigned int shift, offset = 0;
819− unsigned long max = index | ((1UL << order) - 1);
628+ unsigned long max = index;
820629 gfp_t gfp = root_gfp_mask(root);
821630
822631 shift = radix_tree_load_root(root, &child, &maxindex);
823632
824633 /* Make sure the tree is high enough. */
825− if (order > 0 && max == ((1UL << order) - 1))
826− max++;
827634 if (max > maxindex) {
828635 int error = radix_tree_extend(root, gfp, max, shift);
829636 if (error < 0)
830637 return error;
831638 shift = error;
832− child = rcu_dereference_raw(root->rnode);
639+ child = rcu_dereference_raw(root->xa_head);
833640 }
834641
835− while (shift > order) {
642+ while (shift > 0) {
836643 shift -= RADIX_TREE_MAP_SHIFT;
837644 if (child == NULL) {
838645 /* Have to add a child node. */
@@ -875,8 +682,7 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) · 36 unmodified lines
875682
876683 for (;;) {
877684 void *entry = rcu_dereference_raw(child->slots[offset]);
878− if (radix_tree_is_internal_node(entry) &&
879− !is_sibling_entry(child, entry)) {
685+ if (xa_is_node(entry) && child->shift) {
880686 child = entry_to_node(entry);
881687 offset = 0;
882688 continue;
@@ -894,96 +700,30 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) · 11 unmodified lines
894700 }
895701 }
896702
897−#ifdef CONFIG_RADIX_TREE_MULTIORDER
898703 static inline int insert_entries(struct radix_tree_node *node,
899− void __rcu **slot, void *item, unsigned order, bool replace)
900−{
901− struct radix_tree_node *child;
902− unsigned i, n, tag, offset, tags = 0;
903−
904− if (node) {
905− if (order > node->shift)
906− n = 1 << (order - node->shift);
907− else
908− n = 1;
909− offset = get_slot_offset(node, slot);
910− } else {
911− n = 1;
912− offset = 0;
913− }
914−
915− if (n > 1) {
916− offset = offset & ~(n - 1);
917− slot = &node->slots[offset];
918− }
919− child = node_to_entry(slot);
920−
921− for (i = 0; i < n; i++) {
922− if (slot[i]) {
923− if (replace) {
924− node->count--;
925− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
926− if (tag_get(node, tag, offset + i))
927− tags |= 1 << tag;
928− } else
929− return -EEXIST;
930− }
931− }
932−
933− for (i = 0; i < n; i++) {
934− struct radix_tree_node *old = rcu_dereference_raw(slot[i]);
935− if (i) {
936− rcu_assign_pointer(slot[i], child);
937− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
938− if (tags & (1 << tag))
939− tag_clear(node, tag, offset + i);
940− } else {
941− rcu_assign_pointer(slot[i], item);
942− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
943− if (tags & (1 << tag))
944− tag_set(node, tag, offset);
945− }
946− if (radix_tree_is_internal_node(old) &&
947− !is_sibling_entry(node, old) &&
948− (old != RADIX_TREE_RETRY))
949− radix_tree_free_nodes(old);
950− if (radix_tree_exceptional_entry(old))
951− node->exceptional--;
952− }
953− if (node) {
954− node->count += n;
955− if (radix_tree_exceptional_entry(item))
956− node->exceptional += n;
957− }
958− return n;
959−}
960−#else
961−static inline int insert_entries(struct radix_tree_node *node,
962− void __rcu **slot, void *item, unsigned order, bool replace)
704+ void __rcu **slot, void *item, bool replace)
963705 {
964706 if (*slot)
965707 return -EEXIST;
966708 rcu_assign_pointer(*slot, item);
967709 if (node) {
968710 node->count++;
969− if (radix_tree_exceptional_entry(item))
970− node->exceptional++;
711+ if (xa_is_value(item))
712+ node->nr_values++;
971713 }
972714 return 1;
973715 }
974−#endif
975716
976717 /**
977718 * __radix_tree_insert - insert into a radix tree
978719 * @root: radix tree root
979720 * @index: index key
980− * @order: key covers the 2^order indices around index
981721 * @item: item to insert
982722 *
983723 * Insert an item into the radix tree at position @index.
984724 */
985−int __radix_tree_insert(struct radix_tree_root *root, unsigned long index,
986− unsigned order, void *item)
725+int radix_tree_insert(struct radix_tree_root *root, unsigned long index,
726+ void *item)
987727 {
988728 struct radix_tree_node *node;
989729 void __rcu **slot;
@@ -991,11 +731,11 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, · 1 unmodified line
991731
992732 BUG_ON(radix_tree_is_internal_node(item));
993733
994− error = __radix_tree_create(root, index, order, &node, &slot);
734+ error = __radix_tree_create(root, index, &node, &slot);
995735 if (error)
996736 return error;
997737
998− error = insert_entries(node, slot, item, order, false);
738+ error = insert_entries(node, slot, item, false);
999739 if (error < 0)
1000740 return error;
1001741
@@ -1010,7 +750,7 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, · 8 unmodified lines
1010750
1011751 return 0;
1012752 }
1013−EXPORT_SYMBOL(__radix_tree_insert);
753+EXPORT_SYMBOL(radix_tree_insert);
1014754
1015755 /**
1016756 * __radix_tree_lookup - lookup an item in a radix tree
@@ -1023,7 +763,7 @@ EXPORT_SYMBOL(__radix_tree_insert); · 6 unmodified lines
1023763 * tree @root.
1024764 *
1025765 * Until there is more than one item in the tree, no nodes are
1026− * allocated and @root->rnode is used as a direct slot instead of
766+ * allocated and @root->xa_head is used as a direct slot instead of
1027767 * pointing to a node, in which case *@nodep will be NULL.
1028768 */
1029769 void *__radix_tree_lookup(const struct radix_tree_root *root,
@@ -1036,7 +776,7 @@ void *__radix_tree_lookup(const struct radix_tree_root *root, · 6 unmodified lines
1036776
1037777 restart:
1038778 parent = NULL;
1039− slot = (void __rcu **)&root->rnode;
779+ slot = (void __rcu **)&root->xa_head;
1040780 radix_tree_load_root(root, &node, &maxindex);
1041781 if (index > maxindex)
1042782 return NULL;
@@ -1049,6 +789,8 @@ void *__radix_tree_lookup(const struct radix_tree_root *root, · 6 unmodified lines
1049789 parent = entry_to_node(node);
1050790 offset = radix_tree_descend(parent, &node, index);
1051791 slot = parent->slots + offset;
792+ if (parent->shift == 0)
793+ break;
1052794 }
1053795
1054796 if (nodep)
@@ -1100,36 +842,12 @@ void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index) · 45 unmodified lines
1100842 }
1101843 EXPORT_SYMBOL(radix_tree_lookup);
1102844
1103−static inline void replace_sibling_entries(struct radix_tree_node *node,
1104− void __rcu **slot, int count, int exceptional)
1105−{
1106−#ifdef CONFIG_RADIX_TREE_MULTIORDER
1107− void *ptr = node_to_entry(slot);
1108− unsigned offset = get_slot_offset(node, slot) + 1;
1109−
1110− while (offset < RADIX_TREE_MAP_SIZE) {
1111− if (rcu_dereference_raw(node->slots[offset]) != ptr)
1112− break;
1113− if (count < 0) {
1114− node->slots[offset] = NULL;
1115− node->count--;
1116− }
1117− node->exceptional += exceptional;
1118− offset++;
1119− }
1120−#endif
1121−}
1122−
1123845 static void replace_slot(void __rcu **slot, void *item,
1124− struct radix_tree_node *node, int count, int exceptional)
846+ struct radix_tree_node *node, int count, int values)
1125847 {
1126− if (WARN_ON_ONCE(radix_tree_is_internal_node(item)))
1127− return;
1128−
1129− if (node && (count || exceptional)) {
848+ if (node && (count || values)) {
1130849 node->count += count;
1131− node->exceptional += exceptional;
1132− replace_sibling_entries(node, slot, count, exceptional);
850+ node->nr_values += values;
1133851 }
1134852
1135853 rcu_assign_pointer(*slot, item);
@@ -1172,37 +890,31 @@ static int calculate_count(struct radix_tree_root *root, · 36 unmodified lines
1172890 * @node: pointer to tree node
1173891 * @slot: pointer to slot in @node
1174892 * @item: new item to store in the slot.
1175− * @update_node: callback for changing leaf nodes
1176893 *
1177894 * For use with __radix_tree_lookup(). Caller must hold tree write locked
1178895 * across slot lookup and replacement.
1179896 */
1180897 void __radix_tree_replace(struct radix_tree_root *root,
1181898 struct radix_tree_node *node,
1182− void __rcu **slot, void *item,
1183− radix_tree_update_node_t update_node)
899+ void __rcu **slot, void *item)
1184900 {
1185901 void *old = rcu_dereference_raw(*slot);
1186− int exceptional = !!radix_tree_exceptional_entry(item) -
1187− !!radix_tree_exceptional_entry(old);
902+ int values = !!xa_is_value(item) - !!xa_is_value(old);
1188903 int count = calculate_count(root, node, slot, item, old);
1189904
1190905 /*
1191− * This function supports replacing exceptional entries and
906+ * This function supports replacing value entries and
1192907 * deleting entries, but that needs accounting against the
1193− * node unless the slot is root->rnode.
908+ * node unless the slot is root->xa_head.
1194909 */
1195− WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->rnode) &&
1196− (count || exceptional));
1197− replace_slot(slot, item, node, count, exceptional);
910+ WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->xa_head) &&
911+ (count || values));
912+ replace_slot(slot, item, node, count, values);
1198913
1199914 if (!node)
1200915 return;
1201916
1202− if (update_node)
1203− update_node(node);
1204−
1205− delete_node(root, node, update_node);
917+ delete_node(root, node);
1206918 }
1207919
1208920 /**
@@ -1211,12 +923,12 @@ void __radix_tree_replace(struct radix_tree_root *root, · 2 unmodified lines
1211923 * @slot: pointer to slot
1212924 * @item: new item to store in the slot.
1213925 *
1214− * For use with radix_tree_lookup_slot(), radix_tree_gang_lookup_slot(),
926+ * For use with radix_tree_lookup_slot() and
1215927 * radix_tree_gang_lookup_tag_slot(). Caller must hold tree write locked
1216928 * across slot lookup and replacement.
1217929 *
1218930 * NOTE: This cannot be used to switch between non-entries (empty slots),
1219− * regular entries, and exceptional entries, as that requires accounting
931+ * regular entries, and value entries, as that requires accounting
1220932 * inside the radix tree node. When switching from one type of entry or
1221933 * deleting, use __radix_tree_lookup() and __radix_tree_replace() or
1222934 * radix_tree_iter_replace().
@@ -1224,7 +936,7 @@ void __radix_tree_replace(struct radix_tree_root *root, · 1 unmodified line
1224936 void radix_tree_replace_slot(struct radix_tree_root *root,
1225937 void __rcu **slot, void *item)
1226938 {
1227− __radix_tree_replace(root, NULL, slot, item, NULL);
939+ __radix_tree_replace(root, NULL, slot, item);
1228940 }
1229941 EXPORT_SYMBOL(radix_tree_replace_slot);
1230942
@@ -1234,162 +946,16 @@ EXPORT_SYMBOL(radix_tree_replace_slot); · 3 unmodified lines
1234946 * @slot: pointer to slot
1235947 * @item: new item to store in the slot.
1236948 *
1237− * For use with radix_tree_split() and radix_tree_for_each_slot().
1238− * Caller must hold tree write locked across split and replacement.
949+ * For use with radix_tree_for_each_slot().
950+ * Caller must hold tree write locked.
1239951 */
1240952 void radix_tree_iter_replace(struct radix_tree_root *root,
1241953 const struct radix_tree_iter *iter,
1242954 void __rcu **slot, void *item)
1243955 {
1244− __radix_tree_replace(root, iter->node, slot, item, NULL);
956+ __radix_tree_replace(root, iter->node, slot, item);
1245957 }
1246958
1247−#ifdef CONFIG_RADIX_TREE_MULTIORDER
1248−/**
1249− * radix_tree_join - replace multiple entries with one multiorder entry
1250− * @root: radix tree root
1251− * @index: an index inside the new entry
1252− * @order: order of the new entry
1253− * @item: new entry
1254− *
1255− * Call this function to replace several entries with one larger entry.
1256− * The existing entries are presumed to not need freeing as a result of
1257− * this call.
1258− *
1259− * The replacement entry will have all the tags set on it that were set
1260− * on any of the entries it is replacing.
1261− */
1262−int radix_tree_join(struct radix_tree_root *root, unsigned long index,
1263− unsigned order, void *item)
1264−{
1265− struct radix_tree_node *node;
1266− void __rcu **slot;
1267− int error;
1268−
1269− BUG_ON(radix_tree_is_internal_node(item));
1270−
1271− error = __radix_tree_create(root, index, order, &node, &slot);
1272− if (!error)
1273− error = insert_entries(node, slot, item, order, true);
1274− if (error > 0)
1275− error = 0;
1276−
1277− return error;
1278−}
1279−
1280−/**
1281− * radix_tree_split - Split an entry into smaller entries
1282− * @root: radix tree root
1283− * @index: An index within the large entry
1284− * @order: Order of new entries
1285− *
1286− * Call this function as the first step in replacing a multiorder entry
1287− * with several entries of lower order. After this function returns,
1288− * loop over the relevant portion of the tree using radix_tree_for_each_slot()
1289− * and call radix_tree_iter_replace() to set up each new entry.
1290− *
1291− * The tags from this entry are replicated to all the new entries.
1292− *
1293− * The radix tree should be locked against modification during the entire
1294− * replacement operation. Lock-free lookups will see RADIX_TREE_RETRY which
1295− * should prompt RCU walkers to restart the lookup from the root.
1296− */
1297−int radix_tree_split(struct radix_tree_root *root, unsigned long index,
1298− unsigned order)
1299−{
1300− struct radix_tree_node *parent, *node, *child;
1301− void __rcu **slot;
1302− unsigned int offset, end;
1303− unsigned n, tag, tags = 0;
1304− gfp_t gfp = root_gfp_mask(root);
1305−
1306− if (!__radix_tree_lookup(root, index, &parent, &slot))
1307− return -ENOENT;
1308− if (!parent)
1309− return -ENOENT;
1310−
1311− offset = get_slot_offset(parent, slot);
1312−
1313− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
1314− if (tag_get(parent, tag, offset))
1315− tags |= 1 << tag;
1316−
1317− for (end = offset + 1; end < RADIX_TREE_MAP_SIZE; end++) {
1318− if (!is_sibling_entry(parent,
1319− rcu_dereference_raw(parent->slots[end])))
1320− break;
1321− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
1322− if (tags & (1 << tag))
1323− tag_set(parent, tag, end);
1324− /* rcu_assign_pointer ensures tags are set before RETRY */
1325− rcu_assign_pointer(parent->slots[end], RADIX_TREE_RETRY);
1326− }
1327− rcu_assign_pointer(parent->slots[offset], RADIX_TREE_RETRY);
1328− parent->exceptional -= (end - offset);
1329−
1330− if (order == parent->shift)
1331− return 0;
1332− if (order > parent->shift) {
1333− while (offset < end)
1334− offset += insert_entries(parent, &parent->slots[offset],
1335− RADIX_TREE_RETRY, order, true);
1336− return 0;
1337− }
1338−
1339− node = parent;
1340−
1341− for (;;) {
1342− if (node->shift > order) {
1343− child = radix_tree_node_alloc(gfp, node, root,
1344− node->shift - RADIX_TREE_MAP_SHIFT,
1345− offset, 0, 0);
1346− if (!child)
1347− goto nomem;
1348− if (node != parent) {
1349− node->count++;
1350− rcu_assign_pointer(node->slots[offset],
1351− node_to_entry(child));
1352− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
1353− if (tags & (1 << tag))
1354− tag_set(node, tag, offset);
1355− }
1356−
1357− node = child;
1358− offset = 0;
1359− continue;
1360− }
1361−
1362− n = insert_entries(node, &node->slots[offset],
1363− RADIX_TREE_RETRY, order, false);
1364− BUG_ON(n > RADIX_TREE_MAP_SIZE);
1365−
1366− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
1367− if (tags & (1 << tag))
1368− tag_set(node, tag, offset);
1369− offset += n;
1370−
1371− while (offset == RADIX_TREE_MAP_SIZE) {
1372− if (node == parent)
1373− break;
1374− offset = node->offset;
1375− child = node;
1376− node = node->parent;
1377− rcu_assign_pointer(node->slots[offset],
1378− node_to_entry(child));
1379− offset++;
1380− }
1381− if ((node == parent) && (offset == end))
1382− return 0;
1383− }
1384−
1385− nomem:
1386− /* Shouldn't happen; did user forget to preload? */
1387− /* TODO: free all the allocated nodes */
1388− WARN_ON(1);
1389− return -ENOMEM;
1390−}
1391−#endif
1392−
1393959 static void node_tag_set(struct radix_tree_root *root,
1394960 struct radix_tree_node *node,
1395961 unsigned int tag, unsigned int offset)
@@ -1447,18 +1013,6 @@ void *radix_tree_tag_set(struct radix_tree_root *root, · 51 unmodified lines
14471013 }
14481014 EXPORT_SYMBOL(radix_tree_tag_set);
14491015
1450−/**
1451− * radix_tree_iter_tag_set - set a tag on the current iterator entry
1452− * @root: radix tree root
1453− * @iter: iterator state
1454− * @tag: tag to set
1455− */
1456−void radix_tree_iter_tag_set(struct radix_tree_root *root,
1457− const struct radix_tree_iter *iter, unsigned int tag)
1458−{
1459− node_tag_set(root, iter->node, tag, iter_offset(iter));
1460−}
1461−
14621016 static void node_tag_clear(struct radix_tree_root *root,
14631017 struct radix_tree_node *node,
14641018 unsigned int tag, unsigned int offset)
@@ -1574,14 +1128,6 @@ int radix_tree_tag_get(const struct radix_tree_root *root, · 109 unmodified lines
15741128 }
15751129 EXPORT_SYMBOL(radix_tree_tag_get);
15761130
1577−static inline void __set_iter_shift(struct radix_tree_iter *iter,
1578− unsigned int shift)
1579−{
1580−#ifdef CONFIG_RADIX_TREE_MULTIORDER
1581− iter->shift = shift;
1582−#endif
1583−}
1584−
15851131 /* Construct iter->tags bit-mask from node->tags[tag] array */
15861132 static void set_iter_tags(struct radix_tree_iter *iter,
15871133 struct radix_tree_node *node, unsigned offset,
@@ -1608,92 +1154,11 @@ static void set_iter_tags(struct radix_tree_iter *iter, · 20 unmodified lines
16081154 }
16091155 }
16101156
1611−#ifdef CONFIG_RADIX_TREE_MULTIORDER
1612−static void __rcu **skip_siblings(struct radix_tree_node **nodep,
1613− void __rcu **slot, struct radix_tree_iter *iter)
1614−{
1615− while (iter->index < iter->next_index) {
1616− *nodep = rcu_dereference_raw(*slot);
1617− if (*nodep && !is_sibling_entry(iter->node, *nodep))
1618− return slot;
1619− slot++;
1620− iter->index = __radix_tree_iter_add(iter, 1);
1621− iter->tags >>= 1;
1622− }
1623−
1624− *nodep = NULL;
1625− return NULL;
1626−}
1627−
1628−void __rcu **__radix_tree_next_slot(void __rcu **slot,
1629− struct radix_tree_iter *iter, unsigned flags)
1630−{
1631− unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
1632− struct radix_tree_node *node;
1633−
1634− slot = skip_siblings(&node, slot, iter);
1635−
1636− while (radix_tree_is_internal_node(node)) {
1637− unsigned offset;
1638− unsigned long next_index;
1639−
1640− if (node == RADIX_TREE_RETRY)
1641− return slot;
1642− node = entry_to_node(node);
1643− iter->node = node;
1644− iter->shift = node->shift;
1645−
1646− if (flags & RADIX_TREE_ITER_TAGGED) {
1647− offset = radix_tree_find_next_bit(node, tag, 0);
1648− if (offset == RADIX_TREE_MAP_SIZE)
1649− return NULL;
1650− slot = &node->slots[offset];
1651− iter->index = __radix_tree_iter_add(iter, offset);
1652− set_iter_tags(iter, node, offset, tag);
1653− node = rcu_dereference_raw(*slot);
1654− } else {
1655− offset = 0;
1656− slot = &node->slots[0];
1657− for (;;) {
1658− node = rcu_dereference_raw(*slot);
1659− if (node)
1660− break;
1661− slot++;
1662− offset++;
1663− if (offset == RADIX_TREE_MAP_SIZE)
1664− return NULL;
1665− }
1666− iter->index = __radix_tree_iter_add(iter, offset);
1667− }
1668− if ((flags & RADIX_TREE_ITER_CONTIG) && (offset > 0))
1669− goto none;
1670− next_index = (iter->index | shift_maxindex(iter->shift)) + 1;
1671− if (next_index < iter->next_index)
1672− iter->next_index = next_index;
1673− }
1674−
1675− return slot;
1676− none:
1677− iter->next_index = 0;
1678− return NULL;
1679−}
1680−EXPORT_SYMBOL(__radix_tree_next_slot);
1681−#else
1682−static void __rcu **skip_siblings(struct radix_tree_node **nodep,
1683− void __rcu **slot, struct radix_tree_iter *iter)
1684−{
1685− return slot;
1686−}
1687−#endif
1688−
16891157 void __rcu **radix_tree_iter_resume(void __rcu **slot,
16901158 struct radix_tree_iter *iter)
16911159 {
1692− struct radix_tree_node *node;
1693−
16941160 slot++;
16951161 iter->index = __radix_tree_iter_add(iter, 1);
1696− skip_siblings(&node, slot, iter);
16971162 iter->next_index = iter->index;
16981163 iter->tags = 0;
16991164 return NULL;
@@ -1744,8 +1209,7 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, · 44 unmodified lines
17441209 iter->next_index = maxindex + 1;
17451210 iter->tags = 1;
17461211 iter->node = NULL;
1747− __set_iter_shift(iter, 0);
1748− return (void __rcu **)&root->rnode;
1212+ return (void __rcu **)&root->xa_head;
17491213 }
17501214
17511215 do {
@@ -1765,8 +1229,6 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, · 13 unmodified lines
17651229 while (++offset < RADIX_TREE_MAP_SIZE) {
17661230 void *slot = rcu_dereference_raw(
17671231 node->slots[offset]);
1768− if (is_sibling_entry(node, slot))
1769− continue;
17701232 if (slot)
17711233 break;
17721234 }
@@ -1784,13 +1246,12 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, · 11 unmodified lines
17841246 goto restart;
17851247 if (child == RADIX_TREE_RETRY)
17861248 break;
1787− } while (radix_tree_is_internal_node(child));
1249+ } while (node->shift && radix_tree_is_internal_node(child));
17881250
17891251 /* Update the iterator state */
1790− iter->index = (index &~ node_maxindex(node)) | (offset << node->shift);
1252+ iter->index = (index &~ node_maxindex(node)) | offset;
17911253 iter->next_index = (index | node_maxindex(node)) + 1;
17921254 iter->node = node;
1793− __set_iter_shift(iter, node->shift);
17941255
17951256 if (flags & RADIX_TREE_ITER_TAGGED)
17961257 set_iter_tags(iter, node, offset, tag);
@@ -1846,48 +1307,6 @@ radix_tree_gang_lookup(const struct radix_tree_root *root, void **results, · 49 unmodified lines
18461307 }
18471308 EXPORT_SYMBOL(radix_tree_gang_lookup);
18481309
1849−/**
1850− * radix_tree_gang_lookup_slot - perform multiple slot lookup on radix tree
1851− * @root: radix tree root
1852− * @results: where the results of the lookup are placed
1853− * @indices: where their indices should be placed (but usually NULL)
1854− * @first_index: start the lookup from this key
1855− * @max_items: place up to this many items at *results
1856− *
1857− * Performs an index-ascending scan of the tree for present items. Places
1858− * their slots at *@results and returns the number of items which were
1859− * placed at *@results.
1860− *
1861− * The implementation is naive.
1862− *
1863− * Like radix_tree_gang_lookup as far as RCU and locking goes. Slots must
1864− * be dereferenced with radix_tree_deref_slot, and if using only RCU
1865− * protection, radix_tree_deref_slot may fail requiring a retry.
1866− */
1867−unsigned int
1868−radix_tree_gang_lookup_slot(const struct radix_tree_root *root,
1869− void __rcu ***results, unsigned long *indices,
1870− unsigned long first_index, unsigned int max_items)
1871−{
1872− struct radix_tree_iter iter;
1873− void __rcu **slot;
1874− unsigned int ret = 0;
1875−
1876− if (unlikely(!max_items))
1877− return 0;
1878−
1879− radix_tree_for_each_slot(slot, root, &iter, first_index) {
1880− results[ret] = slot;
1881− if (indices)
1882− indices[ret] = iter.index;
1883− if (++ret == max_items)
1884− break;
1885− }
1886−
1887− return ret;
1888−}
1889−EXPORT_SYMBOL(radix_tree_gang_lookup_slot);
1890−
18911310 /**
18921311 * radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
18931312 * based on a tag
@@ -1964,28 +1383,11 @@ radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *root, · 70 unmodified lines
19641383 }
19651384 EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot);
19661385
1967−/**
1968− * __radix_tree_delete_node - try to free node after clearing a slot
1969− * @root: radix tree root
1970− * @node: node containing @index
1971− * @update_node: callback for changing leaf nodes
1972− *
1973− * After clearing the slot at @index in @node from radix tree
1974− * rooted at @root, call this function to attempt freeing the
1975− * node and shrinking the tree.
1976− */
1977−void __radix_tree_delete_node(struct radix_tree_root *root,
1978− struct radix_tree_node *node,
1979− radix_tree_update_node_t update_node)
1980−{
1981− delete_node(root, node, update_node);
1982−}
1983−
19841386 static bool __radix_tree_delete(struct radix_tree_root *root,
19851387 struct radix_tree_node *node, void __rcu **slot)
19861388 {
19871389 void *old = rcu_dereference_raw(*slot);
1988− int exceptional = radix_tree_exceptional_entry(old) ? -1 : 0;
1390+ int values = xa_is_value(old) ? -1 : 0;
19891391 unsigned offset = get_slot_offset(node, slot);
19901392 int tag;
19911393
@@ -1995,8 +1397,8 @@ static bool __radix_tree_delete(struct radix_tree_root *root, · 3 unmodified lines
19951397 for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
19961398 node_tag_clear(root, node, tag, offset);
19971399
1998− replace_slot(slot, NULL, node, -1, exceptional);
1999− return node && delete_node(root, node, NULL);
1400+ replace_slot(slot, NULL, node, -1, values);
1401+ return node && delete_node(root, node);
20001402 }
20011403
20021404 /**
@@ -2068,19 +1470,6 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index) · 65 unmodified lines
20681470 }
20691471 EXPORT_SYMBOL(radix_tree_delete);
20701472
2071−void radix_tree_clear_tags(struct radix_tree_root *root,
2072− struct radix_tree_node *node,
2073− void __rcu **slot)
2074−{
2075− if (node) {
2076− unsigned int tag, offset = get_slot_offset(node, slot);
2077− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
2078− node_tag_clear(root, node, tag, offset);
2079− } else {
2080− root_tag_clear_all(root);
2081− }
2082−}
2083−
20841473 /**
20851474 * radix_tree_tagged - test whether any items in the tree are tagged
20861475 * @root: radix tree root
@@ -2106,33 +1495,12 @@ void idr_preload(gfp_t gfp_mask) · 19 unmodified lines
21061495 }
21071496 EXPORT_SYMBOL(idr_preload);
21081497
2109−int ida_pre_get(struct ida *ida, gfp_t gfp)
2110−{
2111− /*
2112− * The IDA API has no preload_end() equivalent. Instead,
2113− * ida_get_new() can return -EAGAIN, prompting the caller
2114− * to return to the ida_pre_get() step.
2115− */
2116− if (!__radix_tree_preload(gfp, IDA_PRELOAD_SIZE))
2117− preempt_enable();
2118−
2119− if (!this_cpu_read(ida_bitmap)) {
2120− struct ida_bitmap *bitmap = kzalloc(sizeof(*bitmap), gfp);
2121− if (!bitmap)
2122− return 0;
2123− if (this_cpu_cmpxchg(ida_bitmap, NULL, bitmap))
2124− kfree(bitmap);
2125− }
2126−
2127− return 1;
2128−}
2129−
21301498 void __rcu **idr_get_free(struct radix_tree_root *root,
21311499 struct radix_tree_iter *iter, gfp_t gfp,
21321500 unsigned long max)
21331501 {
21341502 struct radix_tree_node *node = NULL, *child;
2135− void __rcu **slot = (void __rcu **)&root->rnode;
1503+ void __rcu **slot = (void __rcu **)&root->xa_head;
21361504 unsigned long maxindex, start = iter->next_index;
21371505 unsigned int shift, offset = 0;
21381506
@@ -2148,8 +1516,10 @@ void __rcu **idr_get_free(struct radix_tree_root *root, · 9 unmodified lines
21481516 if (error < 0)
21491517 return ERR_PTR(error);
21501518 shift = error;
2151− child = rcu_dereference_raw(root->rnode);
1519+ child = rcu_dereference_raw(root->xa_head);
21521520 }
1521+ if (start == 0 && shift == 0)
1522+ shift = RADIX_TREE_MAP_SHIFT;
21531523
21541524 while (shift) {
21551525 shift -= RADIX_TREE_MAP_SHIFT;
@@ -2192,7 +1562,6 @@ void __rcu **idr_get_free(struct radix_tree_root *root, · 36 unmodified lines
21921562 else
21931563 iter->next_index = 1;
21941564 iter->node = node;
2195− __set_iter_shift(iter, shift);
21961565 set_iter_tags(iter, node, offset, IDR_FREE);
21971566
21981567 return slot;
@@ -2211,10 +1580,10 @@ void __rcu **idr_get_free(struct radix_tree_root *root, · 12 unmodified lines
22111580 */
22121581 void idr_destroy(struct idr *idr)
22131582 {
2214− struct radix_tree_node *node = rcu_dereference_raw(idr->idr_rt.rnode);
1583+ struct radix_tree_node *node = rcu_dereference_raw(idr->idr_rt.xa_head);
22151584 if (radix_tree_is_internal_node(node))
22161585 radix_tree_free_nodes(node);
2217− idr->idr_rt.rnode = NULL;
1586+ idr->idr_rt.xa_head = NULL;
22181587 root_tag_set(&idr->idr_rt, IDR_FREE);
22191588 }
22201589 EXPORT_SYMBOL(idr_destroy);
@@ -2228,31 +1597,6 @@ radix_tree_node_ctor(void *arg) · 7 unmodified lines
22281597 INIT_LIST_HEAD(&node->private_list);
22291598 }
22301599
2231−static __init unsigned long __maxindex(unsigned int height)
2232−{
2233− unsigned int width = height * RADIX_TREE_MAP_SHIFT;
2234− int shift = RADIX_TREE_INDEX_BITS - width;
2235−
2236− if (shift < 0)
2237− return ~0UL;
2238− if (shift >= BITS_PER_LONG)
2239− return 0UL;
2240− return ~0UL >> shift;
2241−}
2242−
2243−static __init void radix_tree_init_maxnodes(void)
2244−{
2245− unsigned long height_to_maxindex[RADIX_TREE_MAX_PATH + 1];
2246− unsigned int i, j;
2247−
2248− for (i = 0; i < ARRAY_SIZE(height_to_maxindex); i++)
2249− height_to_maxindex[i] = __maxindex(i);
2250− for (i = 0; i < ARRAY_SIZE(height_to_maxnodes); i++) {
2251− for (j = i; j > 0; j--)
2252− height_to_maxnodes[i] += height_to_maxindex[j - 1] + 1;
2253− }
2254−}
2255−
22561600 static int radix_tree_cpu_dead(unsigned int cpu)
22571601 {
22581602 struct radix_tree_preload *rtp;
@@ -2266,8 +1610,6 @@ static int radix_tree_cpu_dead(unsigned int cpu) · 7 unmodified lines
22661610 kmem_cache_free(radix_tree_node_cachep, node);
22671611 rtp->nr--;
22681612 }
2269− kfree(per_cpu(ida_bitmap, cpu));
2270− per_cpu(ida_bitmap, cpu) = NULL;
22711613 return 0;
22721614 }
22731615
@@ -2277,11 +1619,11 @@ void __init radix_tree_init(void) · 3 unmodified lines
22771619
22781620 BUILD_BUG_ON(RADIX_TREE_MAX_TAGS + __GFP_BITS_SHIFT > 32);
22791621 BUILD_BUG_ON(ROOT_IS_IDR & ~GFP_ZONEMASK);
1622+ BUILD_BUG_ON(XA_CHUNK_SIZE > 255);
22801623 radix_tree_node_cachep = kmem_cache_create("radix_tree_node",
22811624 sizeof(struct radix_tree_node), 0,
22821625 SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
22831626 radix_tree_node_ctor);
2284− radix_tree_init_maxnodes();
22851627 ret = cpuhp_setup_state_nocalls(CPUHP_RADIX_DEAD, "lib/radix:dead",
22861628 NULL, radix_tree_cpu_dead);
22871629 WARN_ON(ret < 0);
lib/test_xarray.c+1238−0
@@ -0,0 +1,1238 @@
1+// SPDX-License-Identifier: GPL-2.0+
2+/*
3+ * test_xarray.c: Test the XArray API
4+ * Copyright (c) 2017-2018 Microsoft Corporation
5+ * Author: Matthew Wilcox <willy@infradead.org>
6+ */
7+
8+#include <linux/xarray.h>
9+#include <linux/module.h>
10+
11+static unsigned int tests_run;
12+static unsigned int tests_passed;
13+
14+#ifndef XA_DEBUG
15+# ifdef __KERNEL__
16+void xa_dump(const struct xarray *xa) { }
17+# endif
18+#undef XA_BUG_ON
19+#define XA_BUG_ON(xa, x) do { \
20+ tests_run++; \
21+ if (x) { \
22+ printk("BUG at %s:%d\n", __func__, __LINE__); \
23+ xa_dump(xa); \
24+ dump_stack(); \
25+ } else { \
26+ tests_passed++; \
27+ } \
28+} while (0)
29+#endif
30+
31+static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp)
32+{
33+ return xa_store(xa, index, xa_mk_value(index & LONG_MAX), gfp);
34+}
35+
36+static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp)
37+{
38+ u32 id = 0;
39+
40+ XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, xa_mk_value(index & LONG_MAX),
41+ gfp) != 0);
42+ XA_BUG_ON(xa, id != index);
43+}
44+
45+static void xa_erase_index(struct xarray *xa, unsigned long index)
46+{
47+ XA_BUG_ON(xa, xa_erase(xa, index) != xa_mk_value(index & LONG_MAX));
48+ XA_BUG_ON(xa, xa_load(xa, index) != NULL);
49+}
50+
51+/*
52+ * If anyone needs this, please move it to xarray.c. We have no current
53+ * users outside the test suite because all current multislot users want
54+ * to use the advanced API.
55+ */
56+static void *xa_store_order(struct xarray *xa, unsigned long index,
57+ unsigned order, void *entry, gfp_t gfp)
58+{
59+ XA_STATE_ORDER(xas, xa, index, order);
60+ void *curr;
61+
62+ do {
63+ xas_lock(&xas);
64+ curr = xas_store(&xas, entry);
65+ xas_unlock(&xas);
66+ } while (xas_nomem(&xas, gfp));
67+
68+ return curr;
69+}
70+
71+static noinline void check_xa_err(struct xarray *xa)
72+{
73+ XA_BUG_ON(xa, xa_err(xa_store_index(xa, 0, GFP_NOWAIT)) != 0);
74+ XA_BUG_ON(xa, xa_err(xa_erase(xa, 0)) != 0);
75+#ifndef __KERNEL__
76+ /* The kernel does not fail GFP_NOWAIT allocations */
77+ XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM);
78+ XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM);
79+#endif
80+ XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_KERNEL)) != 0);
81+ XA_BUG_ON(xa, xa_err(xa_store(xa, 1, xa_mk_value(0), GFP_KERNEL)) != 0);
82+ XA_BUG_ON(xa, xa_err(xa_erase(xa, 1)) != 0);
83+// kills the test-suite :-(
84+// XA_BUG_ON(xa, xa_err(xa_store(xa, 0, xa_mk_internal(0), 0)) != -EINVAL);
85+}
86+
87+static noinline void check_xas_retry(struct xarray *xa)
88+{
89+ XA_STATE(xas, xa, 0);
90+ void *entry;
91+
92+ xa_store_index(xa, 0, GFP_KERNEL);
93+ xa_store_index(xa, 1, GFP_KERNEL);
94+
95+ rcu_read_lock();
96+ XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_value(0));
97+ xa_erase_index(xa, 1);
98+ XA_BUG_ON(xa, !xa_is_retry(xas_reload(&xas)));
99+ XA_BUG_ON(xa, xas_retry(&xas, NULL));
100+ XA_BUG_ON(xa, xas_retry(&xas, xa_mk_value(0)));
101+ xas_reset(&xas);
102+ XA_BUG_ON(xa, xas.xa_node != XAS_RESTART);
103+ XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
104+ XA_BUG_ON(xa, xas.xa_node != NULL);
105+
106+ XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);
107+ XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas)));
108+ xas.xa_node = XAS_RESTART;
109+ XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
110+ rcu_read_unlock();
111+
112+ /* Make sure we can iterate through retry entries */
113+ xas_lock(&xas);
114+ xas_set(&xas, 0);
115+ xas_store(&xas, XA_RETRY_ENTRY);
116+ xas_set(&xas, 1);
117+ xas_store(&xas, XA_RETRY_ENTRY);
118+
119+ xas_set(&xas, 0);
120+ xas_for_each(&xas, entry, ULONG_MAX) {
121+ xas_store(&xas, xa_mk_value(xas.xa_index));
122+ }
123+ xas_unlock(&xas);
124+
125+ xa_erase_index(xa, 0);
126+ xa_erase_index(xa, 1);
127+}
128+
129+static noinline void check_xa_load(struct xarray *xa)
130+{
131+ unsigned long i, j;
132+
133+ for (i = 0; i < 1024; i++) {
134+ for (j = 0; j < 1024; j++) {
135+ void *entry = xa_load(xa, j);
136+ if (j < i)
137+ XA_BUG_ON(xa, xa_to_value(entry) != j);
138+ else
139+ XA_BUG_ON(xa, entry);
140+ }
141+ XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
142+ }
143+
144+ for (i = 0; i < 1024; i++) {
145+ for (j = 0; j < 1024; j++) {
146+ void *entry = xa_load(xa, j);
147+ if (j >= i)
148+ XA_BUG_ON(xa, xa_to_value(entry) != j);
149+ else
150+ XA_BUG_ON(xa, entry);
151+ }
152+ xa_erase_index(xa, i);
153+ }
154+ XA_BUG_ON(xa, !xa_empty(xa));
155+}
156+
157+static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index)
158+{
159+ unsigned int order;
160+ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 8 : 1;
161+
162+ /* NULL elements have no marks set */
163+ XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
164+ xa_set_mark(xa, index, XA_MARK_0);
165+ XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
166+
167+ /* Storing a pointer will not make a mark appear */
168+ XA_BUG_ON(xa, xa_store_index(xa, index, GFP_KERNEL) != NULL);
169+ XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
170+ xa_set_mark(xa, index, XA_MARK_0);
171+ XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0));
172+
173+ /* Setting one mark will not set another mark */
174+ XA_BUG_ON(xa, xa_get_mark(xa, index + 1, XA_MARK_0));
175+ XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_1));
176+
177+ /* Storing NULL clears marks, and they can't be set again */
178+ xa_erase_index(xa, index);
179+ XA_BUG_ON(xa, !xa_empty(xa));
180+ XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
181+ xa_set_mark(xa, index, XA_MARK_0);
182+ XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
183+
184+ /*
185+ * Storing a multi-index entry over entries with marks gives the
186+ * entire entry the union of the marks
187+ */
188+ BUG_ON((index % 4) != 0);
189+ for (order = 2; order < max_order; order++) {
190+ unsigned long base = round_down(index, 1UL << order);
191+ unsigned long next = base + (1UL << order);
192+ unsigned long i;
193+
194+ XA_BUG_ON(xa, xa_store_index(xa, index + 1, GFP_KERNEL));
195+ xa_set_mark(xa, index + 1, XA_MARK_0);
196+ XA_BUG_ON(xa, xa_store_index(xa, index + 2, GFP_KERNEL));
197+ xa_set_mark(xa, index + 2, XA_MARK_1);
198+ XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL));
199+ xa_store_order(xa, index, order, xa_mk_value(index),
200+ GFP_KERNEL);
201+ for (i = base; i < next; i++) {
202+ XA_STATE(xas, xa, i);
203+ unsigned int seen = 0;
204+ void *entry;
205+
206+ XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0));
207+ XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_1));
208+ XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_2));
209+
210+ /* We should see two elements in the array */
211+ xas_for_each(&xas, entry, ULONG_MAX)
212+ seen++;
213+ XA_BUG_ON(xa, seen != 2);
214+
215+ /* One of which is marked */
216+ xas_set(&xas, 0);
217+ seen = 0;
218+ xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0)
219+ seen++;
220+ XA_BUG_ON(xa, seen != 1);
221+ }
222+ XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_0));
223+ XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_1));
224+ XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_2));
225+ xa_erase_index(xa, index);
226+ xa_erase_index(xa, next);
227+ XA_BUG_ON(xa, !xa_empty(xa));
228+ }
229+ XA_BUG_ON(xa, !xa_empty(xa));
230+}
231+
232+static noinline void check_xa_mark_2(struct xarray *xa)
233+{
234+ XA_STATE(xas, xa, 0);
235+ unsigned long index;
236+ unsigned int count = 0;
237+ void *entry;
238+
239+ xa_store_index(xa, 0, GFP_KERNEL);
240+ xa_set_mark(xa, 0, XA_MARK_0);
241+ xas_lock(&xas);
242+ xas_load(&xas);
243+ xas_init_marks(&xas);
244+ xas_unlock(&xas);
245+ XA_BUG_ON(xa, !xa_get_mark(xa, 0, XA_MARK_0) == 0);
246+
247+ for (index = 3500; index < 4500; index++) {
248+ xa_store_index(xa, index, GFP_KERNEL);
249+ xa_set_mark(xa, index, XA_MARK_0);
250+ }
251+
252+ xas_reset(&xas);
253+ rcu_read_lock();
254+ xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0)
255+ count++;
256+ rcu_read_unlock();
257+ XA_BUG_ON(xa, count != 1000);
258+
259+ xas_lock(&xas);
260+ xas_for_each(&xas, entry, ULONG_MAX) {
261+ xas_init_marks(&xas);
262+ XA_BUG_ON(xa, !xa_get_mark(xa, xas.xa_index, XA_MARK_0));
263+ XA_BUG_ON(xa, !xas_get_mark(&xas, XA_MARK_0));
264+ }
265+ xas_unlock(&xas);
266+
267+ xa_destroy(xa);
268+}
269+
270+static noinline void check_xa_mark(struct xarray *xa)
271+{
272+ unsigned long index;
273+
274+ for (index = 0; index < 16384; index += 4)
275+ check_xa_mark_1(xa, index);
276+
277+ check_xa_mark_2(xa);
278+}
279+
280+static noinline void check_xa_shrink(struct xarray *xa)
281+{
282+ XA_STATE(xas, xa, 1);
283+ struct xa_node *node;
284+ unsigned int order;
285+ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 15 : 1;
286+
287+ XA_BUG_ON(xa, !xa_empty(xa));
288+ XA_BUG_ON(xa, xa_store_index(xa, 0, GFP_KERNEL) != NULL);
289+ XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);
290+
291+ /*
292+ * Check that erasing the entry at 1 shrinks the tree and properly
293+ * marks the node as being deleted.
294+ */
295+ xas_lock(&xas);
296+ XA_BUG_ON(xa, xas_load(&xas) != xa_mk_value(1));
297+ node = xas.xa_node;
298+ XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != xa_mk_value(0));
299+ XA_BUG_ON(xa, xas_store(&xas, NULL) != xa_mk_value(1));
300+ XA_BUG_ON(xa, xa_load(xa, 1) != NULL);
301+ XA_BUG_ON(xa, xas.xa_node != XAS_BOUNDS);
302+ XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != XA_RETRY_ENTRY);
303+ XA_BUG_ON(xa, xas_load(&xas) != NULL);
304+ xas_unlock(&xas);
305+ XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
306+ xa_erase_index(xa, 0);
307+ XA_BUG_ON(xa, !xa_empty(xa));
308+
309+ for (order = 0; order < max_order; order++) {
310+ unsigned long max = (1UL << order) - 1;
311+ xa_store_order(xa, 0, order, xa_mk_value(0), GFP_KERNEL);
312+ XA_BUG_ON(xa, xa_load(xa, max) != xa_mk_value(0));
313+ XA_BUG_ON(xa, xa_load(xa, max + 1) != NULL);
314+ rcu_read_lock();
315+ node = xa_head(xa);
316+ rcu_read_unlock();
317+ XA_BUG_ON(xa, xa_store_index(xa, ULONG_MAX, GFP_KERNEL) !=
318+ NULL);
319+ rcu_read_lock();
320+ XA_BUG_ON(xa, xa_head(xa) == node);
321+ rcu_read_unlock();
322+ XA_BUG_ON(xa, xa_load(xa, max + 1) != NULL);
323+ xa_erase_index(xa, ULONG_MAX);
324+ XA_BUG_ON(xa, xa->xa_head != node);
325+ xa_erase_index(xa, 0);
326+ }
327+}
328+
329+static noinline void check_cmpxchg(struct xarray *xa)
330+{
331+ void *FIVE = xa_mk_value(5);
332+ void *SIX = xa_mk_value(6);
333+ void *LOTS = xa_mk_value(12345678);
334+
335+ XA_BUG_ON(xa, !xa_empty(xa));
336+ XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL);
337+ XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST);
338+ XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS);
339+ XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS);
340+ XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE);
341+ XA_BUG_ON(xa, xa_cmpxchg(xa, 5, FIVE, NULL, GFP_KERNEL) != NULL);
342+ XA_BUG_ON(xa, xa_cmpxchg(xa, 5, NULL, FIVE, GFP_KERNEL) != NULL);
343+ xa_erase_index(xa, 12345678);
344+ xa_erase_index(xa, 5);
345+ XA_BUG_ON(xa, !xa_empty(xa));
346+}
347+
348+static noinline void check_reserve(struct xarray *xa)
349+{
350+ void *entry;
351+ unsigned long index = 0;
352+
353+ /* An array with a reserved entry is not empty */
354+ XA_BUG_ON(xa, !xa_empty(xa));
355+ xa_reserve(xa, 12345678, GFP_KERNEL);
356+ XA_BUG_ON(xa, xa_empty(xa));
357+ XA_BUG_ON(xa, xa_load(xa, 12345678));
358+ xa_release(xa, 12345678);
359+ XA_BUG_ON(xa, !xa_empty(xa));
360+
361+ /* Releasing a used entry does nothing */
362+ xa_reserve(xa, 12345678, GFP_KERNEL);
363+ XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_NOWAIT) != NULL);
364+ xa_release(xa, 12345678);
365+ xa_erase_index(xa, 12345678);
366+ XA_BUG_ON(xa, !xa_empty(xa));
367+
368+ /* cmpxchg sees a reserved entry as NULL */
369+ xa_reserve(xa, 12345678, GFP_KERNEL);
370+ XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, NULL, xa_mk_value(12345678),
371+ GFP_NOWAIT) != NULL);
372+ xa_release(xa, 12345678);
373+ xa_erase_index(xa, 12345678);
374+ XA_BUG_ON(xa, !xa_empty(xa));
375+
376+ /* Can iterate through a reserved entry */
377+ xa_store_index(xa, 5, GFP_KERNEL);
378+ xa_reserve(xa, 6, GFP_KERNEL);
379+ xa_store_index(xa, 7, GFP_KERNEL);
380+
381+ xa_for_each(xa, entry, index, ULONG_MAX, XA_PRESENT) {
382+ XA_BUG_ON(xa, index != 5 && index != 7);
383+ }
384+ xa_destroy(xa);
385+}
386+
387+static noinline void check_xas_erase(struct xarray *xa)
388+{
389+ XA_STATE(xas, xa, 0);
390+ void *entry;
391+ unsigned long i, j;
392+
393+ for (i = 0; i < 200; i++) {
394+ for (j = i; j < 2 * i + 17; j++) {
395+ xas_set(&xas, j);
396+ do {
397+ xas_lock(&xas);
398+ xas_store(&xas, xa_mk_value(j));
399+ xas_unlock(&xas);
400+ } while (xas_nomem(&xas, GFP_KERNEL));
401+ }
402+
403+ xas_set(&xas, ULONG_MAX);
404+ do {
405+ xas_lock(&xas);
406+ xas_store(&xas, xa_mk_value(0));
407+ xas_unlock(&xas);
408+ } while (xas_nomem(&xas, GFP_KERNEL));
409+
410+ xas_lock(&xas);
411+ xas_store(&xas, NULL);
412+
413+ xas_set(&xas, 0);
414+ j = i;
415+ xas_for_each(&xas, entry, ULONG_MAX) {
416+ XA_BUG_ON(xa, entry != xa_mk_value(j));
417+ xas_store(&xas, NULL);
418+ j++;
419+ }
420+ xas_unlock(&xas);
421+ XA_BUG_ON(xa, !xa_empty(xa));
422+ }
423+}
424+
425+#ifdef CONFIG_XARRAY_MULTI
426+static noinline void check_multi_store_1(struct xarray *xa, unsigned long index,
427+ unsigned int order)
428+{
429+ XA_STATE(xas, xa, index);
430+ unsigned long min = index & ~((1UL << order) - 1);
431+ unsigned long max = min + (1UL << order);
432+
433+ xa_store_order(xa, index, order, xa_mk_value(index), GFP_KERNEL);
434+ XA_BUG_ON(xa, xa_load(xa, min) != xa_mk_value(index));
435+ XA_BUG_ON(xa, xa_load(xa, max - 1) != xa_mk_value(index));
436+ XA_BUG_ON(xa, xa_load(xa, max) != NULL);
437+ XA_BUG_ON(xa, xa_load(xa, min - 1) != NULL);
438+
439+ XA_BUG_ON(xa, xas_store(&xas, xa_mk_value(min)) != xa_mk_value(index));
440+ XA_BUG_ON(xa, xa_load(xa, min) != xa_mk_value(min));
441+ XA_BUG_ON(xa, xa_load(xa, max - 1) != xa_mk_value(min));
442+ XA_BUG_ON(xa, xa_load(xa, max) != NULL);
443+ XA_BUG_ON(xa, xa_load(xa, min - 1) != NULL);
444+
445+ xa_erase_index(xa, min);
446+ XA_BUG_ON(xa, !xa_empty(xa));
447+}
448+
449+static noinline void check_multi_store_2(struct xarray *xa, unsigned long index,
450+ unsigned int order)
451+{
452+ XA_STATE(xas, xa, index);
453+ xa_store_order(xa, index, order, xa_mk_value(0), GFP_KERNEL);
454+
455+ XA_BUG_ON(xa, xas_store(&xas, xa_mk_value(1)) != xa_mk_value(0));
456+ XA_BUG_ON(xa, xas.xa_index != index);
457+ XA_BUG_ON(xa, xas_store(&xas, NULL) != xa_mk_value(1));
458+ XA_BUG_ON(xa, !xa_empty(xa));
459+}
460+#endif
461+
462+static noinline void check_multi_store(struct xarray *xa)
463+{
464+#ifdef CONFIG_XARRAY_MULTI
465+ unsigned long i, j, k;
466+ unsigned int max_order = (sizeof(long) == 4) ? 30 : 60;
467+
468+ /* Loading from any position returns the same value */
469+ xa_store_order(xa, 0, 1, xa_mk_value(0), GFP_KERNEL);
470+ XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
471+ XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0));
472+ XA_BUG_ON(xa, xa_load(xa, 2) != NULL);
473+ rcu_read_lock();
474+ XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 2);
475+ XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2);
476+ rcu_read_unlock();
477+
478+ /* Storing adjacent to the value does not alter the value */
479+ xa_store(xa, 3, xa, GFP_KERNEL);
480+ XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
481+ XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0));
482+ XA_BUG_ON(xa, xa_load(xa, 2) != NULL);
483+ rcu_read_lock();
484+ XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 3);
485+ XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2);
486+ rcu_read_unlock();
487+
488+ /* Overwriting multiple indexes works */
489+ xa_store_order(xa, 0, 2, xa_mk_value(1), GFP_KERNEL);
490+ XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(1));
491+ XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(1));
492+ XA_BUG_ON(xa, xa_load(xa, 2) != xa_mk_value(1));
493+ XA_BUG_ON(xa, xa_load(xa, 3) != xa_mk_value(1));
494+ XA_BUG_ON(xa, xa_load(xa, 4) != NULL);
495+ rcu_read_lock();
496+ XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 4);
497+ XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 4);
498+ rcu_read_unlock();
499+
500+ /* We can erase multiple values with a single store */
501+ xa_store_order(xa, 0, 63, NULL, GFP_KERNEL);
502+ XA_BUG_ON(xa, !xa_empty(xa));
503+
504+ /* Even when the first slot is empty but the others aren't */
505+ xa_store_index(xa, 1, GFP_KERNEL);
506+ xa_store_index(xa, 2, GFP_KERNEL);
507+ xa_store_order(xa, 0, 2, NULL, GFP_KERNEL);
508+ XA_BUG_ON(xa, !xa_empty(xa));
509+
510+ for (i = 0; i < max_order; i++) {
511+ for (j = 0; j < max_order; j++) {
512+ xa_store_order(xa, 0, i, xa_mk_value(i), GFP_KERNEL);
513+ xa_store_order(xa, 0, j, xa_mk_value(j), GFP_KERNEL);
514+
515+ for (k = 0; k < max_order; k++) {
516+ void *entry = xa_load(xa, (1UL << k) - 1);
517+ if ((i < k) && (j < k))
518+ XA_BUG_ON(xa, entry != NULL);
519+ else
520+ XA_BUG_ON(xa, entry != xa_mk_value(j));
521+ }
522+
523+ xa_erase(xa, 0);
524+ XA_BUG_ON(xa, !xa_empty(xa));
525+ }
526+ }
527+
528+ for (i = 0; i < 20; i++) {
529+ check_multi_store_1(xa, 200, i);
530+ check_multi_store_1(xa, 0, i);
531+ check_multi_store_1(xa, (1UL << i) + 1, i);
532+ }
533+ check_multi_store_2(xa, 4095, 9);
534+#endif
535+}
536+
537+static DEFINE_XARRAY_ALLOC(xa0);
538+
539+static noinline void check_xa_alloc(void)
540+{
541+ int i;
542+ u32 id;
543+
544+ /* An empty array should assign 0 to the first alloc */
545+ xa_alloc_index(&xa0, 0, GFP_KERNEL);
546+
547+ /* Erasing it should make the array empty again */
548+ xa_erase_index(&xa0, 0);
549+ XA_BUG_ON(&xa0, !xa_empty(&xa0));
550+
551+ /* And it should assign 0 again */
552+ xa_alloc_index(&xa0, 0, GFP_KERNEL);
553+
554+ /* The next assigned ID should be 1 */
555+ xa_alloc_index(&xa0, 1, GFP_KERNEL);
556+ xa_erase_index(&xa0, 1);
557+
558+ /* Storing a value should mark it used */
559+ xa_store_index(&xa0, 1, GFP_KERNEL);
560+ xa_alloc_index(&xa0, 2, GFP_KERNEL);
561+
562+ /* If we then erase 0, it should be free */
563+ xa_erase_index(&xa0, 0);
564+ xa_alloc_index(&xa0, 0, GFP_KERNEL);
565+
566+ xa_erase_index(&xa0, 1);
567+ xa_erase_index(&xa0, 2);
568+
569+ for (i = 1; i < 5000; i++) {
570+ xa_alloc_index(&xa0, i, GFP_KERNEL);
571+ }
572+
573+ xa_destroy(&xa0);
574+
575+ id = 0xfffffffeU;
576+ XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0),
577+ GFP_KERNEL) != 0);
578+ XA_BUG_ON(&xa0, id != 0xfffffffeU);
579+ XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0),
580+ GFP_KERNEL) != 0);
581+ XA_BUG_ON(&xa0, id != 0xffffffffU);
582+ XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0),
583+ GFP_KERNEL) != -ENOSPC);
584+ XA_BUG_ON(&xa0, id != 0xffffffffU);
585+ xa_destroy(&xa0);
586+}
587+
588+static noinline void __check_store_iter(struct xarray *xa, unsigned long start,
589+ unsigned int order, unsigned int present)
590+{
591+ XA_STATE_ORDER(xas, xa, start, order);
592+ void *entry;
593+ unsigned int count = 0;
594+
595+retry:
596+ xas_lock(&xas);
597+ xas_for_each_conflict(&xas, entry) {
598+ XA_BUG_ON(xa, !xa_is_value(entry));
599+ XA_BUG_ON(xa, entry < xa_mk_value(start));
600+ XA_BUG_ON(xa, entry > xa_mk_value(start + (1UL << order) - 1));
601+ count++;
602+ }
603+ xas_store(&xas, xa_mk_value(start));
604+ xas_unlock(&xas);
605+ if (xas_nomem(&xas, GFP_KERNEL)) {
606+ count = 0;
607+ goto retry;
608+ }
609+ XA_BUG_ON(xa, xas_error(&xas));
610+ XA_BUG_ON(xa, count != present);
611+ XA_BUG_ON(xa, xa_load(xa, start) != xa_mk_value(start));
612+ XA_BUG_ON(xa, xa_load(xa, start + (1UL << order) - 1) !=
613+ xa_mk_value(start));
614+ xa_erase_index(xa, start);
615+}
616+
617+static noinline void check_store_iter(struct xarray *xa)
618+{
619+ unsigned int i, j;
620+ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 20 : 1;
621+
622+ for (i = 0; i < max_order; i++) {
623+ unsigned int min = 1 << i;
624+ unsigned int max = (2 << i) - 1;
625+ __check_store_iter(xa, 0, i, 0);
626+ XA_BUG_ON(xa, !xa_empty(xa));
627+ __check_store_iter(xa, min, i, 0);
628+ XA_BUG_ON(xa, !xa_empty(xa));
629+
630+ xa_store_index(xa, min, GFP_KERNEL);
631+ __check_store_iter(xa, min, i, 1);
632+ XA_BUG_ON(xa, !xa_empty(xa));
633+ xa_store_index(xa, max, GFP_KERNEL);
634+ __check_store_iter(xa, min, i, 1);
635+ XA_BUG_ON(xa, !xa_empty(xa));
636+
637+ for (j = 0; j < min; j++)
638+ xa_store_index(xa, j, GFP_KERNEL);
639+ __check_store_iter(xa, 0, i, min);
640+ XA_BUG_ON(xa, !xa_empty(xa));
641+ for (j = 0; j < min; j++)
642+ xa_store_index(xa, min + j, GFP_KERNEL);
643+ __check_store_iter(xa, min, i, min);
644+ XA_BUG_ON(xa, !xa_empty(xa));
645+ }
646+#ifdef CONFIG_XARRAY_MULTI
647+ xa_store_index(xa, 63, GFP_KERNEL);
648+ xa_store_index(xa, 65, GFP_KERNEL);
649+ __check_store_iter(xa, 64, 2, 1);
650+ xa_erase_index(xa, 63);
651+#endif
652+ XA_BUG_ON(xa, !xa_empty(xa));
653+}
654+
655+static noinline void check_multi_find(struct xarray *xa)
656+{
657+#ifdef CONFIG_XARRAY_MULTI
658+ unsigned long index;
659+
660+ xa_store_order(xa, 12, 2, xa_mk_value(12), GFP_KERNEL);
661+ XA_BUG_ON(xa, xa_store_index(xa, 16, GFP_KERNEL) != NULL);
662+
663+ index = 0;
664+ XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
665+ xa_mk_value(12));
666+ XA_BUG_ON(xa, index != 12);
667+ index = 13;
668+ XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
669+ xa_mk_value(12));
670+ XA_BUG_ON(xa, (index < 12) || (index >= 16));
671+ XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) !=
672+ xa_mk_value(16));
673+ XA_BUG_ON(xa, index != 16);
674+
675+ xa_erase_index(xa, 12);
676+ xa_erase_index(xa, 16);
677+ XA_BUG_ON(xa, !xa_empty(xa));
678+#endif
679+}
680+
681+static noinline void check_multi_find_2(struct xarray *xa)
682+{
683+ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 10 : 1;
684+ unsigned int i, j;
685+ void *entry;
686+
687+ for (i = 0; i < max_order; i++) {
688+ unsigned long index = 1UL << i;
689+ for (j = 0; j < index; j++) {
690+ XA_STATE(xas, xa, j + index);
691+ xa_store_index(xa, index - 1, GFP_KERNEL);
692+ xa_store_order(xa, index, i, xa_mk_value(index),
693+ GFP_KERNEL);
694+ rcu_read_lock();
695+ xas_for_each(&xas, entry, ULONG_MAX) {
696+ xa_erase_index(xa, index);
697+ }
698+ rcu_read_unlock();
699+ xa_erase_index(xa, index - 1);
700+ XA_BUG_ON(xa, !xa_empty(xa));
701+ }
702+ }
703+}
704+
705+static noinline void check_find(struct xarray *xa)
706+{
707+ unsigned long i, j, k;
708+
709+ XA_BUG_ON(xa, !xa_empty(xa));
710+
711+ /*
712+ * Check xa_find with all pairs between 0 and 99 inclusive,
713+ * starting at every index between 0 and 99
714+ */
715+ for (i = 0; i < 100; i++) {
716+ XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
717+ xa_set_mark(xa, i, XA_MARK_0);
718+ for (j = 0; j < i; j++) {
719+ XA_BUG_ON(xa, xa_store_index(xa, j, GFP_KERNEL) !=
720+ NULL);
721+ xa_set_mark(xa, j, XA_MARK_0);
722+ for (k = 0; k < 100; k++) {
723+ unsigned long index = k;
724+ void *entry = xa_find(xa, &index, ULONG_MAX,
725+ XA_PRESENT);
726+ if (k <= j)
727+ XA_BUG_ON(xa, index != j);
728+ else if (k <= i)
729+ XA_BUG_ON(xa, index != i);
730+ else
731+ XA_BUG_ON(xa, entry != NULL);
732+
733+ index = k;
734+ entry = xa_find(xa, &index, ULONG_MAX,
735+ XA_MARK_0);
736+ if (k <= j)
737+ XA_BUG_ON(xa, index != j);
738+ else if (k <= i)
739+ XA_BUG_ON(xa, index != i);
740+ else
741+ XA_BUG_ON(xa, entry != NULL);
742+ }
743+ xa_erase_index(xa, j);
744+ XA_BUG_ON(xa, xa_get_mark(xa, j, XA_MARK_0));
745+ XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0));
746+ }
747+ xa_erase_index(xa, i);
748+ XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_0));
749+ }
750+ XA_BUG_ON(xa, !xa_empty(xa));
751+ check_multi_find(xa);
752+ check_multi_find_2(xa);
753+}
754+
755+/* See find_swap_entry() in mm/shmem.c */
756+static noinline unsigned long xa_find_entry(struct xarray *xa, void *item)
757+{
758+ XA_STATE(xas, xa, 0);
759+ unsigned int checked = 0;
760+ void *entry;
761+
762+ rcu_read_lock();
763+ xas_for_each(&xas, entry, ULONG_MAX) {
764+ if (xas_retry(&xas, entry))
765+ continue;
766+ if (entry == item)
767+ break;
768+ checked++;
769+ if ((checked % 4) != 0)
770+ continue;
771+ xas_pause(&xas);
772+ }
773+ rcu_read_unlock();
774+
775+ return entry ? xas.xa_index : -1;
776+}
777+
778+static noinline void check_find_entry(struct xarray *xa)
779+{
780+#ifdef CONFIG_XARRAY_MULTI
781+ unsigned int order;
782+ unsigned long offset, index;
783+
784+ for (order = 0; order < 20; order++) {
785+ for (offset = 0; offset < (1UL << (order + 3));
786+ offset += (1UL << order)) {
787+ for (index = 0; index < (1UL << (order + 5));
788+ index += (1UL << order)) {
789+ xa_store_order(xa, index, order,
790+ xa_mk_value(index), GFP_KERNEL);
791+ XA_BUG_ON(xa, xa_load(xa, index) !=
792+ xa_mk_value(index));
793+ XA_BUG_ON(xa, xa_find_entry(xa,
794+ xa_mk_value(index)) != index);
795+ }
796+ XA_BUG_ON(xa, xa_find_entry(xa, xa) != -1);
797+ xa_destroy(xa);
798+ }
799+ }
800+#endif
801+
802+ XA_BUG_ON(xa, xa_find_entry(xa, xa) != -1);
803+ xa_store_index(xa, ULONG_MAX, GFP_KERNEL);
804+ XA_BUG_ON(xa, xa_find_entry(xa, xa) != -1);
805+ XA_BUG_ON(xa, xa_find_entry(xa, xa_mk_value(LONG_MAX)) != -1);
806+ xa_erase_index(xa, ULONG_MAX);
807+ XA_BUG_ON(xa, !xa_empty(xa));
808+}
809+
810+static noinline void check_move_small(struct xarray *xa, unsigned long idx)
811+{
812+ XA_STATE(xas, xa, 0);
813+ unsigned long i;
814+
815+ xa_store_index(xa, 0, GFP_KERNEL);
816+ xa_store_index(xa, idx, GFP_KERNEL);
817+
818+ rcu_read_lock();
819+ for (i = 0; i < idx * 4; i++) {
820+ void *entry = xas_next(&xas);
821+ if (i <= idx)
822+ XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);
823+ XA_BUG_ON(xa, xas.xa_index != i);
824+ if (i == 0 || i == idx)
825+ XA_BUG_ON(xa, entry != xa_mk_value(i));
826+ else
827+ XA_BUG_ON(xa, entry != NULL);
828+ }
829+ xas_next(&xas);
830+ XA_BUG_ON(xa, xas.xa_index != i);
831+
832+ do {
833+ void *entry = xas_prev(&xas);
834+ i--;
835+ if (i <= idx)
836+ XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);
837+ XA_BUG_ON(xa, xas.xa_index != i);
838+ if (i == 0 || i == idx)
839+ XA_BUG_ON(xa, entry != xa_mk_value(i));
840+ else
841+ XA_BUG_ON(xa, entry != NULL);
842+ } while (i > 0);
843+
844+ xas_set(&xas, ULONG_MAX);
845+ XA_BUG_ON(xa, xas_next(&xas) != NULL);
846+ XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
847+ XA_BUG_ON(xa, xas_next(&xas) != xa_mk_value(0));
848+ XA_BUG_ON(xa, xas.xa_index != 0);
849+ XA_BUG_ON(xa, xas_prev(&xas) != NULL);
850+ XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
851+ rcu_read_unlock();
852+
853+ xa_erase_index(xa, 0);
854+ xa_erase_index(xa, idx);
855+ XA_BUG_ON(xa, !xa_empty(xa));
856+}
857+
858+static noinline void check_move(struct xarray *xa)
859+{
860+ XA_STATE(xas, xa, (1 << 16) - 1);
861+ unsigned long i;
862+
863+ for (i = 0; i < (1 << 16); i++)
864+ XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
865+
866+ rcu_read_lock();
867+ do {
868+ void *entry = xas_prev(&xas);
869+ i--;
870+ XA_BUG_ON(xa, entry != xa_mk_value(i));
871+ XA_BUG_ON(xa, i != xas.xa_index);
872+ } while (i != 0);
873+
874+ XA_BUG_ON(xa, xas_prev(&xas) != NULL);
875+ XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
876+
877+ do {
878+ void *entry = xas_next(&xas);
879+ XA_BUG_ON(xa, entry != xa_mk_value(i));
880+ XA_BUG_ON(xa, i != xas.xa_index);
881+ i++;
882+ } while (i < (1 << 16));
883+ rcu_read_unlock();
884+
885+ for (i = (1 << 8); i < (1 << 15); i++)
886+ xa_erase_index(xa, i);
887+
888+ i = xas.xa_index;
889+
890+ rcu_read_lock();
891+ do {
892+ void *entry = xas_prev(&xas);
893+ i--;
894+ if ((i < (1 << 8)) || (i >= (1 << 15)))
895+ XA_BUG_ON(xa, entry != xa_mk_value(i));
896+ else
897+ XA_BUG_ON(xa, entry != NULL);
898+ XA_BUG_ON(xa, i != xas.xa_index);
899+ } while (i != 0);
900+
901+ XA_BUG_ON(xa, xas_prev(&xas) != NULL);
902+ XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
903+
904+ do {
905+ void *entry = xas_next(&xas);
906+ if ((i < (1 << 8)) || (i >= (1 << 15)))
907+ XA_BUG_ON(xa, entry != xa_mk_value(i));
908+ else
909+ XA_BUG_ON(xa, entry != NULL);
910+ XA_BUG_ON(xa, i != xas.xa_index);
911+ i++;
912+ } while (i < (1 << 16));
913+ rcu_read_unlock();
914+
915+ xa_destroy(xa);
916+
917+ for (i = 0; i < 16; i++)
918+ check_move_small(xa, 1UL << i);
919+
920+ for (i = 2; i < 16; i++)
921+ check_move_small(xa, (1UL << i) - 1);
922+}
923+
924+static noinline void xa_store_many_order(struct xarray *xa,
925+ unsigned long index, unsigned order)
926+{
927+ XA_STATE_ORDER(xas, xa, index, order);
928+ unsigned int i = 0;
929+
930+ do {
931+ xas_lock(&xas);
932+ XA_BUG_ON(xa, xas_find_conflict(&xas));
933+ xas_create_range(&xas);
934+ if (xas_error(&xas))
935+ goto unlock;
936+ for (i = 0; i < (1U << order); i++) {
937+ XA_BUG_ON(xa, xas_store(&xas, xa_mk_value(index + i)));
938+ xas_next(&xas);
939+ }
940+unlock:
941+ xas_unlock(&xas);
942+ } while (xas_nomem(&xas, GFP_KERNEL));
943+
944+ XA_BUG_ON(xa, xas_error(&xas));
945+}
946+
947+static noinline void check_create_range_1(struct xarray *xa,
948+ unsigned long index, unsigned order)
949+{
950+ unsigned long i;
951+
952+ xa_store_many_order(xa, index, order);
953+ for (i = index; i < index + (1UL << order); i++)
954+ xa_erase_index(xa, i);
955+ XA_BUG_ON(xa, !xa_empty(xa));
956+}
957+
958+static noinline void check_create_range_2(struct xarray *xa, unsigned order)
959+{
960+ unsigned long i;
961+ unsigned long nr = 1UL << order;
962+
963+ for (i = 0; i < nr * nr; i += nr)
964+ xa_store_many_order(xa, i, order);
965+ for (i = 0; i < nr * nr; i++)
966+ xa_erase_index(xa, i);
967+ XA_BUG_ON(xa, !xa_empty(xa));
968+}
969+
970+static noinline void check_create_range_3(void)
971+{
972+ XA_STATE(xas, NULL, 0);
973+ xas_set_err(&xas, -EEXIST);
974+ xas_create_range(&xas);
975+ XA_BUG_ON(NULL, xas_error(&xas) != -EEXIST);
976+}
977+
978+static noinline void check_create_range_4(struct xarray *xa,
979+ unsigned long index, unsigned order)
980+{
981+ XA_STATE_ORDER(xas, xa, index, order);
982+ unsigned long base = xas.xa_index;
983+ unsigned long i = 0;
984+
985+ xa_store_index(xa, index, GFP_KERNEL);
986+ do {
987+ xas_lock(&xas);
988+ xas_create_range(&xas);
989+ if (xas_error(&xas))
990+ goto unlock;
991+ for (i = 0; i < (1UL << order); i++) {
992+ void *old = xas_store(&xas, xa_mk_value(base + i));
993+ if (xas.xa_index == index)
994+ XA_BUG_ON(xa, old != xa_mk_value(base + i));
995+ else
996+ XA_BUG_ON(xa, old != NULL);
997+ xas_next(&xas);
998+ }
999+unlock:
1000+ xas_unlock(&xas);
1001+ } while (xas_nomem(&xas, GFP_KERNEL));
1002+
1003+ XA_BUG_ON(xa, xas_error(&xas));
1004+
1005+ for (i = base; i < base + (1UL << order); i++)
1006+ xa_erase_index(xa, i);
1007+ XA_BUG_ON(xa, !xa_empty(xa));
1008+}
1009+
1010+static noinline void check_create_range(struct xarray *xa)
1011+{
1012+ unsigned int order;
1013+ unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 12 : 1;
1014+
1015+ for (order = 0; order < max_order; order++) {
1016+ check_create_range_1(xa, 0, order);
1017+ check_create_range_1(xa, 1U << order, order);
1018+ check_create_range_1(xa, 2U << order, order);
1019+ check_create_range_1(xa, 3U << order, order);
1020+ check_create_range_1(xa, 1U << 24, order);
1021+ if (order < 10)
1022+ check_create_range_2(xa, order);
1023+
1024+ check_create_range_4(xa, 0, order);
1025+ check_create_range_4(xa, 1U << order, order);
1026+ check_create_range_4(xa, 2U << order, order);
1027+ check_create_range_4(xa, 3U << order, order);
1028+ check_create_range_4(xa, 1U << 24, order);
1029+
1030+ check_create_range_4(xa, 1, order);
1031+ check_create_range_4(xa, (1U << order) + 1, order);
1032+ check_create_range_4(xa, (2U << order) + 1, order);
1033+ check_create_range_4(xa, (2U << order) - 1, order);
1034+ check_create_range_4(xa, (3U << order) + 1, order);
1035+ check_create_range_4(xa, (3U << order) - 1, order);
1036+ check_create_range_4(xa, (1U << 24) + 1, order);
1037+ }
1038+
1039+ check_create_range_3();
1040+}
1041+
1042+static noinline void __check_store_range(struct xarray *xa, unsigned long first,
1043+ unsigned long last)
1044+{
1045+#ifdef CONFIG_XARRAY_MULTI
1046+ xa_store_range(xa, first, last, xa_mk_value(first), GFP_KERNEL);
1047+
1048+ XA_BUG_ON(xa, xa_load(xa, first) != xa_mk_value(first));
1049+ XA_BUG_ON(xa, xa_load(xa, last) != xa_mk_value(first));
1050+ XA_BUG_ON(xa, xa_load(xa, first - 1) != NULL);
1051+ XA_BUG_ON(xa, xa_load(xa, last + 1) != NULL);
1052+
1053+ xa_store_range(xa, first, last, NULL, GFP_KERNEL);
1054+#endif
1055+
1056+ XA_BUG_ON(xa, !xa_empty(xa));
1057+}
1058+
1059+static noinline void check_store_range(struct xarray *xa)
1060+{
1061+ unsigned long i, j;
1062+
1063+ for (i = 0; i < 128; i++) {
1064+ for (j = i; j < 128; j++) {
1065+ __check_store_range(xa, i, j);
1066+ __check_store_range(xa, 128 + i, 128 + j);
1067+ __check_store_range(xa, 4095 + i, 4095 + j);
1068+ __check_store_range(xa, 4096 + i, 4096 + j);
1069+ __check_store_range(xa, 123456 + i, 123456 + j);
1070+ __check_store_range(xa, UINT_MAX + i, UINT_MAX + j);
1071+ }
1072+ }
1073+}
1074+
1075+static LIST_HEAD(shadow_nodes);
1076+
1077+static void test_update_node(struct xa_node *node)
1078+{
1079+ if (node->count && node->count == node->nr_values) {
1080+ if (list_empty(&node->private_list))
1081+ list_add(&shadow_nodes, &node->private_list);
1082+ } else {
1083+ if (!list_empty(&node->private_list))
1084+ list_del_init(&node->private_list);
1085+ }
1086+}
1087+
1088+static noinline void shadow_remove(struct xarray *xa)
1089+{
1090+ struct xa_node *node;
1091+
1092+ xa_lock(xa);
1093+ while ((node = list_first_entry_or_null(&shadow_nodes,
1094+ struct xa_node, private_list))) {
1095+ XA_STATE(xas, node->array, 0);
1096+ XA_BUG_ON(xa, node->array != xa);
1097+ list_del_init(&node->private_list);
1098+ xas.xa_node = xa_parent_locked(node->array, node);
1099+ xas.xa_offset = node->offset;
1100+ xas.xa_shift = node->shift + XA_CHUNK_SHIFT;
1101+ xas_set_update(&xas, test_update_node);
1102+ xas_store(&xas, NULL);
1103+ }
1104+ xa_unlock(xa);
1105+}
1106+
1107+static noinline void check_workingset(struct xarray *xa, unsigned long index)
1108+{
1109+ XA_STATE(xas, xa, index);
1110+ xas_set_update(&xas, test_update_node);
1111+
1112+ do {
1113+ xas_lock(&xas);
1114+ xas_store(&xas, xa_mk_value(0));
1115+ xas_next(&xas);
1116+ xas_store(&xas, xa_mk_value(1));
1117+ xas_unlock(&xas);
1118+ } while (xas_nomem(&xas, GFP_KERNEL));
1119+
1120+ XA_BUG_ON(xa, list_empty(&shadow_nodes));
1121+
1122+ xas_lock(&xas);
1123+ xas_next(&xas);
1124+ xas_store(&xas, &xas);
1125+ XA_BUG_ON(xa, !list_empty(&shadow_nodes));
1126+
1127+ xas_store(&xas, xa_mk_value(2));
1128+ xas_unlock(&xas);
1129+ XA_BUG_ON(xa, list_empty(&shadow_nodes));
1130+
1131+ shadow_remove(xa);
1132+ XA_BUG_ON(xa, !list_empty(&shadow_nodes));
1133+ XA_BUG_ON(xa, !xa_empty(xa));
1134+}
1135+
1136+/*
1137+ * Check that the pointer / value / sibling entries are accounted the
1138+ * way we expect them to be.
1139+ */
1140+static noinline void check_account(struct xarray *xa)
1141+{
1142+#ifdef CONFIG_XARRAY_MULTI
1143+ unsigned int order;
1144+
1145+ for (order = 1; order < 12; order++) {
1146+ XA_STATE(xas, xa, 1 << order);
1147+
1148+ xa_store_order(xa, 0, order, xa, GFP_KERNEL);
1149+ xas_load(&xas);
1150+ XA_BUG_ON(xa, xas.xa_node->count == 0);
1151+ XA_BUG_ON(xa, xas.xa_node->count > (1 << order));
1152+ XA_BUG_ON(xa, xas.xa_node->nr_values != 0);
1153+
1154+ xa_store_order(xa, 1 << order, order, xa_mk_value(1 << order),
1155+ GFP_KERNEL);
1156+ XA_BUG_ON(xa, xas.xa_node->count != xas.xa_node->nr_values * 2);
1157+
1158+ xa_erase(xa, 1 << order);
1159+ XA_BUG_ON(xa, xas.xa_node->nr_values != 0);
1160+
1161+ xa_erase(xa, 0);
1162+ XA_BUG_ON(xa, !xa_empty(xa));
1163+ }
1164+#endif
1165+}
1166+
1167+static noinline void check_destroy(struct xarray *xa)
1168+{
1169+ unsigned long index;
1170+
1171+ XA_BUG_ON(xa, !xa_empty(xa));
1172+
1173+ /* Destroying an empty array is a no-op */
1174+ xa_destroy(xa);
1175+ XA_BUG_ON(xa, !xa_empty(xa));
1176+
1177+ /* Destroying an array with a single entry */
1178+ for (index = 0; index < 1000; index++) {
1179+ xa_store_index(xa, index, GFP_KERNEL);
1180+ XA_BUG_ON(xa, xa_empty(xa));
1181+ xa_destroy(xa);
1182+ XA_BUG_ON(xa, !xa_empty(xa));
1183+ }
1184+
1185+ /* Destroying an array with a single entry at ULONG_MAX */
1186+ xa_store(xa, ULONG_MAX, xa, GFP_KERNEL);
1187+ XA_BUG_ON(xa, xa_empty(xa));
1188+ xa_destroy(xa);
1189+ XA_BUG_ON(xa, !xa_empty(xa));
1190+
1191+#ifdef CONFIG_XARRAY_MULTI
1192+ /* Destroying an array with a multi-index entry */
1193+ xa_store_order(xa, 1 << 11, 11, xa, GFP_KERNEL);
1194+ XA_BUG_ON(xa, xa_empty(xa));
1195+ xa_destroy(xa);
1196+ XA_BUG_ON(xa, !xa_empty(xa));
1197+#endif
1198+}
1199+
1200+static DEFINE_XARRAY(array);
1201+
1202+static int xarray_checks(void)
1203+{
1204+ check_xa_err(&array);
1205+ check_xas_retry(&array);
1206+ check_xa_load(&array);
1207+ check_xa_mark(&array);
1208+ check_xa_shrink(&array);
1209+ check_xas_erase(&array);
1210+ check_cmpxchg(&array);
1211+ check_reserve(&array);
1212+ check_multi_store(&array);
1213+ check_xa_alloc();
1214+ check_find(&array);
1215+ check_find_entry(&array);
1216+ check_account(&array);
1217+ check_destroy(&array);
1218+ check_move(&array);
1219+ check_create_range(&array);
1220+ check_store_range(&array);
1221+ check_store_iter(&array);
1222+
1223+ check_workingset(&array, 0);
1224+ check_workingset(&array, 64);
1225+ check_workingset(&array, 4096);
1226+
1227+ printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);
1228+ return (tests_run == tests_passed) ? 0 : -EINVAL;
1229+}
1230+
1231+static void xarray_exit(void)
1232+{
1233+}
1234+
1235+module_init(xarray_checks);
1236+module_exit(xarray_exit);
1237+MODULE_AUTHOR("Matthew Wilcox <willy@infradead.org>");
1238+MODULE_LICENSE("GPL");
lib/xarray.c+2036−0
@@ -0,0 +1,2036 @@
1+// SPDX-License-Identifier: GPL-2.0+
2+/*
3+ * XArray implementation
4+ * Copyright (c) 2017 Microsoft Corporation
5+ * Author: Matthew Wilcox <willy@infradead.org>
6+ */
7+
8+#include <linux/bitmap.h>
9+#include <linux/export.h>
10+#include <linux/list.h>
11+#include <linux/slab.h>
12+#include <linux/xarray.h>
13+
14+/*
15+ * Coding conventions in this file:
16+ *
17+ * @xa is used to refer to the entire xarray.
18+ * @xas is the 'xarray operation state'. It may be either a pointer to
19+ * an xa_state, or an xa_state stored on the stack. This is an unfortunate
20+ * ambiguity.
21+ * @index is the index of the entry being operated on
22+ * @mark is an xa_mark_t; a small number indicating one of the mark bits.
23+ * @node refers to an xa_node; usually the primary one being operated on by
24+ * this function.
25+ * @offset is the index into the slots array inside an xa_node.
26+ * @parent refers to the @xa_node closer to the head than @node.
27+ * @entry refers to something stored in a slot in the xarray
28+ */
29+
30+static inline unsigned int xa_lock_type(const struct xarray *xa)
31+{
32+ return (__force unsigned int)xa->xa_flags & 3;
33+}
34+
35+static inline void xas_lock_type(struct xa_state *xas, unsigned int lock_type)
36+{
37+ if (lock_type == XA_LOCK_IRQ)
38+ xas_lock_irq(xas);
39+ else if (lock_type == XA_LOCK_BH)
40+ xas_lock_bh(xas);
41+ else
42+ xas_lock(xas);
43+}
44+
45+static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type)
46+{
47+ if (lock_type == XA_LOCK_IRQ)
48+ xas_unlock_irq(xas);
49+ else if (lock_type == XA_LOCK_BH)
50+ xas_unlock_bh(xas);
51+ else
52+ xas_unlock(xas);
53+}
54+
55+static inline bool xa_track_free(const struct xarray *xa)
56+{
57+ return xa->xa_flags & XA_FLAGS_TRACK_FREE;
58+}
59+
60+static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)
61+{
62+ if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))
63+ xa->xa_flags |= XA_FLAGS_MARK(mark);
64+}
65+
66+static inline void xa_mark_clear(struct xarray *xa, xa_mark_t mark)
67+{
68+ if (xa->xa_flags & XA_FLAGS_MARK(mark))
69+ xa->xa_flags &= ~(XA_FLAGS_MARK(mark));
70+}
71+
72+static inline unsigned long *node_marks(struct xa_node *node, xa_mark_t mark)
73+{
74+ return node->marks[(__force unsigned)mark];
75+}
76+
77+static inline bool node_get_mark(struct xa_node *node,
78+ unsigned int offset, xa_mark_t mark)
79+{
80+ return test_bit(offset, node_marks(node, mark));
81+}
82+
83+/* returns true if the bit was set */
84+static inline bool node_set_mark(struct xa_node *node, unsigned int offset,
85+ xa_mark_t mark)
86+{
87+ return __test_and_set_bit(offset, node_marks(node, mark));
88+}
89+
90+/* returns true if the bit was set */
91+static inline bool node_clear_mark(struct xa_node *node, unsigned int offset,
92+ xa_mark_t mark)
93+{
94+ return __test_and_clear_bit(offset, node_marks(node, mark));
95+}
96+
97+static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark)
98+{
99+ return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE);
100+}
101+
102+static inline void node_mark_all(struct xa_node *node, xa_mark_t mark)
103+{
104+ bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE);
105+}
106+
107+#define mark_inc(mark) do { \
108+ mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \
109+} while (0)
110+
111+/*
112+ * xas_squash_marks() - Merge all marks to the first entry
113+ * @xas: Array operation state.
114+ *
115+ * Set a mark on the first entry if any entry has it set. Clear marks on
116+ * all sibling entries.
117+ */
118+static void xas_squash_marks(const struct xa_state *xas)
119+{
120+ unsigned int mark = 0;
121+ unsigned int limit = xas->xa_offset + xas->xa_sibs + 1;
122+
123+ if (!xas->xa_sibs)
124+ return;
125+
126+ do {
127+ unsigned long *marks = xas->xa_node->marks[mark];
128+ if (find_next_bit(marks, limit, xas->xa_offset + 1) == limit)
129+ continue;
130+ __set_bit(xas->xa_offset, marks);
131+ bitmap_clear(marks, xas->xa_offset + 1, xas->xa_sibs);
132+ } while (mark++ != (__force unsigned)XA_MARK_MAX);
133+}
134+
135+/* extracts the offset within this node from the index */
136+static unsigned int get_offset(unsigned long index, struct xa_node *node)
137+{
138+ return (index >> node->shift) & XA_CHUNK_MASK;
139+}
140+
141+static void xas_set_offset(struct xa_state *xas)
142+{
143+ xas->xa_offset = get_offset(xas->xa_index, xas->xa_node);
144+}
145+
146+/* move the index either forwards (find) or backwards (sibling slot) */
147+static void xas_move_index(struct xa_state *xas, unsigned long offset)
148+{
149+ unsigned int shift = xas->xa_node->shift;
150+ xas->xa_index &= ~XA_CHUNK_MASK << shift;
151+ xas->xa_index += offset << shift;
152+}
153+
154+static void xas_advance(struct xa_state *xas)
155+{
156+ xas->xa_offset++;
157+ xas_move_index(xas, xas->xa_offset);
158+}
159+
160+static void *set_bounds(struct xa_state *xas)
161+{
162+ xas->xa_node = XAS_BOUNDS;
163+ return NULL;
164+}
165+
166+/*
167+ * Starts a walk. If the @xas is already valid, we assume that it's on
168+ * the right path and just return where we've got to. If we're in an
169+ * error state, return NULL. If the index is outside the current scope
170+ * of the xarray, return NULL without changing @xas->xa_node. Otherwise
171+ * set @xas->xa_node to NULL and return the current head of the array.
172+ */
173+static void *xas_start(struct xa_state *xas)
174+{
175+ void *entry;
176+
177+ if (xas_valid(xas))
178+ return xas_reload(xas);
179+ if (xas_error(xas))
180+ return NULL;
181+
182+ entry = xa_head(xas->xa);
183+ if (!xa_is_node(entry)) {
184+ if (xas->xa_index)
185+ return set_bounds(xas);
186+ } else {
187+ if ((xas->xa_index >> xa_to_node(entry)->shift) > XA_CHUNK_MASK)
188+ return set_bounds(xas);
189+ }
190+
191+ xas->xa_node = NULL;
192+ return entry;
193+}
194+
195+static void *xas_descend(struct xa_state *xas, struct xa_node *node)
196+{
197+ unsigned int offset = get_offset(xas->xa_index, node);
198+ void *entry = xa_entry(xas->xa, node, offset);
199+
200+ xas->xa_node = node;
201+ if (xa_is_sibling(entry)) {
202+ offset = xa_to_sibling(entry);
203+ entry = xa_entry(xas->xa, node, offset);
204+ }
205+
206+ xas->xa_offset = offset;
207+ return entry;
208+}
209+
210+/**
211+ * xas_load() - Load an entry from the XArray (advanced).
212+ * @xas: XArray operation state.
213+ *
214+ * Usually walks the @xas to the appropriate state to load the entry
215+ * stored at xa_index. However, it will do nothing and return %NULL if
216+ * @xas is in an error state. xas_load() will never expand the tree.
217+ *
218+ * If the xa_state is set up to operate on a multi-index entry, xas_load()
219+ * may return %NULL or an internal entry, even if there are entries
220+ * present within the range specified by @xas.
221+ *
222+ * Context: Any context. The caller should hold the xa_lock or the RCU lock.
223+ * Return: Usually an entry in the XArray, but see description for exceptions.
224+ */
225+void *xas_load(struct xa_state *xas)
226+{
227+ void *entry = xas_start(xas);
228+
229+ while (xa_is_node(entry)) {
230+ struct xa_node *node = xa_to_node(entry);
231+
232+ if (xas->xa_shift > node->shift)
233+ break;
234+ entry = xas_descend(xas, node);
235+ }
236+ return entry;
237+}
238+EXPORT_SYMBOL_GPL(xas_load);
239+
240+/* Move the radix tree node cache here */
241+extern struct kmem_cache *radix_tree_node_cachep;
242+extern void radix_tree_node_rcu_free(struct rcu_head *head);
243+
244+#define XA_RCU_FREE ((struct xarray *)1)
245+
246+static void xa_node_free(struct xa_node *node)
247+{
248+ XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
249+ node->array = XA_RCU_FREE;
250+ call_rcu(&node->rcu_head, radix_tree_node_rcu_free);
251+}
252+
253+/*
254+ * xas_destroy() - Free any resources allocated during the XArray operation.
255+ * @xas: XArray operation state.
256+ *
257+ * This function is now internal-only.
258+ */
259+static void xas_destroy(struct xa_state *xas)
260+{
261+ struct xa_node *node = xas->xa_alloc;
262+
263+ if (!node)
264+ return;
265+ XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
266+ kmem_cache_free(radix_tree_node_cachep, node);
267+ xas->xa_alloc = NULL;
268+}
269+
270+/**
271+ * xas_nomem() - Allocate memory if needed.
272+ * @xas: XArray operation state.
273+ * @gfp: Memory allocation flags.
274+ *
275+ * If we need to add new nodes to the XArray, we try to allocate memory
276+ * with GFP_NOWAIT while holding the lock, which will usually succeed.
277+ * If it fails, @xas is flagged as needing memory to continue. The caller
278+ * should drop the lock and call xas_nomem(). If xas_nomem() succeeds,
279+ * the caller should retry the operation.
280+ *
281+ * Forward progress is guaranteed as one node is allocated here and
282+ * stored in the xa_state where it will be found by xas_alloc(). More
283+ * nodes will likely be found in the slab allocator, but we do not tie
284+ * them up here.
285+ *
286+ * Return: true if memory was needed, and was successfully allocated.
287+ */
288+bool xas_nomem(struct xa_state *xas, gfp_t gfp)
289+{
290+ if (xas->xa_node != XA_ERROR(-ENOMEM)) {
291+ xas_destroy(xas);
292+ return false;
293+ }
294+ xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp);
295+ if (!xas->xa_alloc)
296+ return false;
297+ XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list));
298+ xas->xa_node = XAS_RESTART;
299+ return true;
300+}
301+EXPORT_SYMBOL_GPL(xas_nomem);
302+
303+/*
304+ * __xas_nomem() - Drop locks and allocate memory if needed.
305+ * @xas: XArray operation state.
306+ * @gfp: Memory allocation flags.
307+ *
308+ * Internal variant of xas_nomem().
309+ *
310+ * Return: true if memory was needed, and was successfully allocated.
311+ */
312+static bool __xas_nomem(struct xa_state *xas, gfp_t gfp)
313+ __must_hold(xas->xa->xa_lock)
314+{
315+ unsigned int lock_type = xa_lock_type(xas->xa);
316+
317+ if (xas->xa_node != XA_ERROR(-ENOMEM)) {
318+ xas_destroy(xas);
319+ return false;
320+ }
321+ if (gfpflags_allow_blocking(gfp)) {
322+ xas_unlock_type(xas, lock_type);
323+ xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp);
324+ xas_lock_type(xas, lock_type);
325+ } else {
326+ xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp);
327+ }
328+ if (!xas->xa_alloc)
329+ return false;
330+ XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list));
331+ xas->xa_node = XAS_RESTART;
332+ return true;
333+}
334+
335+static void xas_update(struct xa_state *xas, struct xa_node *node)
336+{
337+ if (xas->xa_update)
338+ xas->xa_update(node);
339+ else
340+ XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
341+}
342+
343+static void *xas_alloc(struct xa_state *xas, unsigned int shift)
344+{
345+ struct xa_node *parent = xas->xa_node;
346+ struct xa_node *node = xas->xa_alloc;
347+
348+ if (xas_invalid(xas))
349+ return NULL;
350+
351+ if (node) {
352+ xas->xa_alloc = NULL;
353+ } else {
354+ node = kmem_cache_alloc(radix_tree_node_cachep,
355+ GFP_NOWAIT | __GFP_NOWARN);
356+ if (!node) {
357+ xas_set_err(xas, -ENOMEM);
358+ return NULL;
359+ }
360+ }
361+
362+ if (parent) {
363+ node->offset = xas->xa_offset;
364+ parent->count++;
365+ XA_NODE_BUG_ON(node, parent->count > XA_CHUNK_SIZE);
366+ xas_update(xas, parent);
367+ }
368+ XA_NODE_BUG_ON(node, shift > BITS_PER_LONG);
369+ XA_NODE_BUG_ON(node, !list_empty(&node->private_list));
370+ node->shift = shift;
371+ node->count = 0;
372+ node->nr_values = 0;
373+ RCU_INIT_POINTER(node->parent, xas->xa_node);
374+ node->array = xas->xa;
375+
376+ return node;
377+}
378+
379+#ifdef CONFIG_XARRAY_MULTI
380+/* Returns the number of indices covered by a given xa_state */
381+static unsigned long xas_size(const struct xa_state *xas)
382+{
383+ return (xas->xa_sibs + 1UL) << xas->xa_shift;
384+}
385+#endif
386+
387+/*
388+ * Use this to calculate the maximum index that will need to be created
389+ * in order to add the entry described by @xas. Because we cannot store a
390+ * multiple-index entry at index 0, the calculation is a little more complex
391+ * than you might expect.
392+ */
393+static unsigned long xas_max(struct xa_state *xas)
394+{
395+ unsigned long max = xas->xa_index;
396+
397+#ifdef CONFIG_XARRAY_MULTI
398+ if (xas->xa_shift || xas->xa_sibs) {
399+ unsigned long mask = xas_size(xas) - 1;
400+ max |= mask;
401+ if (mask == max)
402+ max++;
403+ }
404+#endif
405+
406+ return max;
407+}
408+
409+/* The maximum index that can be contained in the array without expanding it */
410+static unsigned long max_index(void *entry)
411+{
412+ if (!xa_is_node(entry))
413+ return 0;
414+ return (XA_CHUNK_SIZE << xa_to_node(entry)->shift) - 1;
415+}
416+
417+static void xas_shrink(struct xa_state *xas)
418+{
419+ struct xarray *xa = xas->xa;
420+ struct xa_node *node = xas->xa_node;
421+
422+ for (;;) {
423+ void *entry;
424+
425+ XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE);
426+ if (node->count != 1)
427+ break;
428+ entry = xa_entry_locked(xa, node, 0);
429+ if (!entry)
430+ break;
431+ if (!xa_is_node(entry) && node->shift)
432+ break;
433+ xas->xa_node = XAS_BOUNDS;
434+
435+ RCU_INIT_POINTER(xa->xa_head, entry);
436+ if (xa_track_free(xa) && !node_get_mark(node, 0, XA_FREE_MARK))
437+ xa_mark_clear(xa, XA_FREE_MARK);
438+
439+ node->count = 0;
440+ node->nr_values = 0;
441+ if (!xa_is_node(entry))
442+ RCU_INIT_POINTER(node->slots[0], XA_RETRY_ENTRY);
443+ xas_update(xas, node);
444+ xa_node_free(node);
445+ if (!xa_is_node(entry))
446+ break;
447+ node = xa_to_node(entry);
448+ node->parent = NULL;
449+ }
450+}
451+
452+/*
453+ * xas_delete_node() - Attempt to delete an xa_node
454+ * @xas: Array operation state.
455+ *
456+ * Attempts to delete the @xas->xa_node. This will fail if xa->node has
457+ * a non-zero reference count.
458+ */
459+static void xas_delete_node(struct xa_state *xas)
460+{
461+ struct xa_node *node = xas->xa_node;
462+
463+ for (;;) {
464+ struct xa_node *parent;
465+
466+ XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE);
467+ if (node->count)
468+ break;
469+
470+ parent = xa_parent_locked(xas->xa, node);
471+ xas->xa_node = parent;
472+ xas->xa_offset = node->offset;
473+ xa_node_free(node);
474+
475+ if (!parent) {
476+ xas->xa->xa_head = NULL;
477+ xas->xa_node = XAS_BOUNDS;
478+ return;
479+ }
480+
481+ parent->slots[xas->xa_offset] = NULL;
482+ parent->count--;
483+ XA_NODE_BUG_ON(parent, parent->count > XA_CHUNK_SIZE);
484+ node = parent;
485+ xas_update(xas, node);
486+ }
487+
488+ if (!node->parent)
489+ xas_shrink(xas);
490+}
491+
492+/**
493+ * xas_free_nodes() - Free this node and all nodes that it references
494+ * @xas: Array operation state.
495+ * @top: Node to free
496+ *
497+ * This node has been removed from the tree. We must now free it and all
498+ * of its subnodes. There may be RCU walkers with references into the tree,
499+ * so we must replace all entries with retry markers.
500+ */
501+static void xas_free_nodes(struct xa_state *xas, struct xa_node *top)
502+{
503+ unsigned int offset = 0;
504+ struct xa_node *node = top;
505+
506+ for (;;) {
507+ void *entry = xa_entry_locked(xas->xa, node, offset);
508+
509+ if (xa_is_node(entry)) {
510+ node = xa_to_node(entry);
511+ offset = 0;
512+ continue;
513+ }
514+ if (entry)
515+ RCU_INIT_POINTER(node->slots[offset], XA_RETRY_ENTRY);
516+ offset++;
517+ while (offset == XA_CHUNK_SIZE) {
518+ struct xa_node *parent;
519+
520+ parent = xa_parent_locked(xas->xa, node);
521+ offset = node->offset + 1;
522+ node->count = 0;
523+ node->nr_values = 0;
524+ xas_update(xas, node);
525+ xa_node_free(node);
526+ if (node == top)
527+ return;
528+ node = parent;
529+ }
530+ }
531+}
532+
533+/*
534+ * xas_expand adds nodes to the head of the tree until it has reached
535+ * sufficient height to be able to contain @xas->xa_index
536+ */
537+static int xas_expand(struct xa_state *xas, void *head)
538+{
539+ struct xarray *xa = xas->xa;
540+ struct xa_node *node = NULL;
541+ unsigned int shift = 0;
542+ unsigned long max = xas_max(xas);
543+
544+ if (!head) {
545+ if (max == 0)
546+ return 0;
547+ while ((max >> shift) >= XA_CHUNK_SIZE)
548+ shift += XA_CHUNK_SHIFT;
549+ return shift + XA_CHUNK_SHIFT;
550+ } else if (xa_is_node(head)) {
551+ node = xa_to_node(head);
552+ shift = node->shift + XA_CHUNK_SHIFT;
553+ }
554+ xas->xa_node = NULL;
555+
556+ while (max > max_index(head)) {
557+ xa_mark_t mark = 0;
558+
559+ XA_NODE_BUG_ON(node, shift > BITS_PER_LONG);
560+ node = xas_alloc(xas, shift);
561+ if (!node)
562+ return -ENOMEM;
563+
564+ node->count = 1;
565+ if (xa_is_value(head))
566+ node->nr_values = 1;
567+ RCU_INIT_POINTER(node->slots[0], head);
568+
569+ /* Propagate the aggregated mark info to the new child */
570+ for (;;) {
571+ if (xa_track_free(xa) && mark == XA_FREE_MARK) {
572+ node_mark_all(node, XA_FREE_MARK);
573+ if (!xa_marked(xa, XA_FREE_MARK)) {
574+ node_clear_mark(node, 0, XA_FREE_MARK);
575+ xa_mark_set(xa, XA_FREE_MARK);
576+ }
577+ } else if (xa_marked(xa, mark)) {
578+ node_set_mark(node, 0, mark);
579+ }
580+ if (mark == XA_MARK_MAX)
581+ break;
582+ mark_inc(mark);
583+ }
584+
585+ /*
586+ * Now that the new node is fully initialised, we can add
587+ * it to the tree
588+ */
589+ if (xa_is_node(head)) {
590+ xa_to_node(head)->offset = 0;
591+ rcu_assign_pointer(xa_to_node(head)->parent, node);
592+ }
593+ head = xa_mk_node(node);
594+ rcu_assign_pointer(xa->xa_head, head);
595+ xas_update(xas, node);
596+
597+ shift += XA_CHUNK_SHIFT;
598+ }
599+
600+ xas->xa_node = node;
601+ return shift;
602+}
603+
604+/*
605+ * xas_create() - Create a slot to store an entry in.
606+ * @xas: XArray operation state.
607+ *
608+ * Most users will not need to call this function directly, as it is called
609+ * by xas_store(). It is useful for doing conditional store operations
610+ * (see the xa_cmpxchg() implementation for an example).
611+ *
612+ * Return: If the slot already existed, returns the contents of this slot.
613+ * If the slot was newly created, returns NULL. If it failed to create the
614+ * slot, returns NULL and indicates the error in @xas.
615+ */
616+static void *xas_create(struct xa_state *xas)
617+{
618+ struct xarray *xa = xas->xa;
619+ void *entry;
620+ void __rcu **slot;
621+ struct xa_node *node = xas->xa_node;
622+ int shift;
623+ unsigned int order = xas->xa_shift;
624+
625+ if (xas_top(node)) {
626+ entry = xa_head_locked(xa);
627+ xas->xa_node = NULL;
628+ shift = xas_expand(xas, entry);
629+ if (shift < 0)
630+ return NULL;
631+ entry = xa_head_locked(xa);
632+ slot = &xa->xa_head;
633+ } else if (xas_error(xas)) {
634+ return NULL;
635+ } else if (node) {
636+ unsigned int offset = xas->xa_offset;
637+
638+ shift = node->shift;
639+ entry = xa_entry_locked(xa, node, offset);
640+ slot = &node->slots[offset];
641+ } else {
642+ shift = 0;
643+ entry = xa_head_locked(xa);
644+ slot = &xa->xa_head;
645+ }
646+
647+ while (shift > order) {
648+ shift -= XA_CHUNK_SHIFT;
649+ if (!entry) {
650+ node = xas_alloc(xas, shift);
651+ if (!node)
652+ break;
653+ if (xa_track_free(xa))
654+ node_mark_all(node, XA_FREE_MARK);
655+ rcu_assign_pointer(*slot, xa_mk_node(node));
656+ } else if (xa_is_node(entry)) {
657+ node = xa_to_node(entry);
658+ } else {
659+ break;
660+ }
661+ entry = xas_descend(xas, node);
662+ slot = &node->slots[xas->xa_offset];
663+ }
664+
665+ return entry;
666+}
667+
668+/**
669+ * xas_create_range() - Ensure that stores to this range will succeed
670+ * @xas: XArray operation state.
671+ *
672+ * Creates all of the slots in the range covered by @xas. Sets @xas to
673+ * create single-index entries and positions it at the beginning of the
674+ * range. This is for the benefit of users which have not yet been
675+ * converted to use multi-index entries.
676+ */
677+void xas_create_range(struct xa_state *xas)
678+{
679+ unsigned long index = xas->xa_index;
680+ unsigned char shift = xas->xa_shift;
681+ unsigned char sibs = xas->xa_sibs;
682+
683+ xas->xa_index |= ((sibs + 1) << shift) - 1;
684+ if (xas_is_node(xas) && xas->xa_node->shift == xas->xa_shift)
685+ xas->xa_offset |= sibs;
686+ xas->xa_shift = 0;
687+ xas->xa_sibs = 0;
688+
689+ for (;;) {
690+ xas_create(xas);
691+ if (xas_error(xas))
692+ goto restore;
693+ if (xas->xa_index <= (index | XA_CHUNK_MASK))
694+ goto success;
695+ xas->xa_index -= XA_CHUNK_SIZE;
696+
697+ for (;;) {
698+ struct xa_node *node = xas->xa_node;
699+ xas->xa_node = xa_parent_locked(xas->xa, node);
700+ xas->xa_offset = node->offset - 1;
701+ if (node->offset != 0)
702+ break;
703+ }
704+ }
705+
706+restore:
707+ xas->xa_shift = shift;
708+ xas->xa_sibs = sibs;
709+ xas->xa_index = index;
710+ return;
711+success:
712+ xas->xa_index = index;
713+ if (xas->xa_node)
714+ xas_set_offset(xas);
715+}
716+EXPORT_SYMBOL_GPL(xas_create_range);
717+
718+static void update_node(struct xa_state *xas, struct xa_node *node,
719+ int count, int values)
720+{
721+ if (!node || (!count && !values))
722+ return;
723+
724+ node->count += count;
725+ node->nr_values += values;
726+ XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE);
727+ XA_NODE_BUG_ON(node, node->nr_values > XA_CHUNK_SIZE);
728+ xas_update(xas, node);
729+ if (count < 0)
730+ xas_delete_node(xas);
731+}
732+
733+/**
734+ * xas_store() - Store this entry in the XArray.
735+ * @xas: XArray operation state.
736+ * @entry: New entry.
737+ *
738+ * If @xas is operating on a multi-index entry, the entry returned by this
739+ * function is essentially meaningless (it may be an internal entry or it
740+ * may be %NULL, even if there are non-NULL entries at some of the indices
741+ * covered by the range). This is not a problem for any current users,
742+ * and can be changed if needed.
743+ *
744+ * Return: The old entry at this index.
745+ */
746+void *xas_store(struct xa_state *xas, void *entry)
747+{
748+ struct xa_node *node;
749+ void __rcu **slot = &xas->xa->xa_head;
750+ unsigned int offset, max;
751+ int count = 0;
752+ int values = 0;
753+ void *first, *next;
754+ bool value = xa_is_value(entry);
755+
756+ if (entry)
757+ first = xas_create(xas);
758+ else
759+ first = xas_load(xas);
760+
761+ if (xas_invalid(xas))
762+ return first;
763+ node = xas->xa_node;
764+ if (node && (xas->xa_shift < node->shift))
765+ xas->xa_sibs = 0;
766+ if ((first == entry) && !xas->xa_sibs)
767+ return first;
768+
769+ next = first;
770+ offset = xas->xa_offset;
771+ max = xas->xa_offset + xas->xa_sibs;
772+ if (node) {
773+ slot = &node->slots[offset];
774+ if (xas->xa_sibs)
775+ xas_squash_marks(xas);
776+ }
777+ if (!entry)
778+ xas_init_marks(xas);
779+
780+ for (;;) {
781+ /*
782+ * Must clear the marks before setting the entry to NULL,
783+ * otherwise xas_for_each_marked may find a NULL entry and
784+ * stop early. rcu_assign_pointer contains a release barrier
785+ * so the mark clearing will appear to happen before the
786+ * entry is set to NULL.
787+ */
788+ rcu_assign_pointer(*slot, entry);
789+ if (xa_is_node(next))
790+ xas_free_nodes(xas, xa_to_node(next));
791+ if (!node)
792+ break;
793+ count += !next - !entry;
794+ values += !xa_is_value(first) - !value;
795+ if (entry) {
796+ if (offset == max)
797+ break;
798+ if (!xa_is_sibling(entry))
799+ entry = xa_mk_sibling(xas->xa_offset);
800+ } else {
801+ if (offset == XA_CHUNK_MASK)
802+ break;
803+ }
804+ next = xa_entry_locked(xas->xa, node, ++offset);
805+ if (!xa_is_sibling(next)) {
806+ if (!entry && (offset > max))
807+ break;
808+ first = next;
809+ }
810+ slot++;
811+ }
812+
813+ update_node(xas, node, count, values);
814+ return first;
815+}
816+EXPORT_SYMBOL_GPL(xas_store);
817+
818+/**
819+ * xas_get_mark() - Returns the state of this mark.
820+ * @xas: XArray operation state.
821+ * @mark: Mark number.
822+ *
823+ * Return: true if the mark is set, false if the mark is clear or @xas
824+ * is in an error state.
825+ */
826+bool xas_get_mark(const struct xa_state *xas, xa_mark_t mark)
827+{
828+ if (xas_invalid(xas))
829+ return false;
830+ if (!xas->xa_node)
831+ return xa_marked(xas->xa, mark);
832+ return node_get_mark(xas->xa_node, xas->xa_offset, mark);
833+}
834+EXPORT_SYMBOL_GPL(xas_get_mark);
835+
836+/**
837+ * xas_set_mark() - Sets the mark on this entry and its parents.
838+ * @xas: XArray operation state.
839+ * @mark: Mark number.
840+ *
841+ * Sets the specified mark on this entry, and walks up the tree setting it
842+ * on all the ancestor entries. Does nothing if @xas has not been walked to
843+ * an entry, or is in an error state.
844+ */
845+void xas_set_mark(const struct xa_state *xas, xa_mark_t mark)
846+{
847+ struct xa_node *node = xas->xa_node;
848+ unsigned int offset = xas->xa_offset;
849+
850+ if (xas_invalid(xas))
851+ return;
852+
853+ while (node) {
854+ if (node_set_mark(node, offset, mark))
855+ return;
856+ offset = node->offset;
857+ node = xa_parent_locked(xas->xa, node);
858+ }
859+
860+ if (!xa_marked(xas->xa, mark))
861+ xa_mark_set(xas->xa, mark);
862+}
863+EXPORT_SYMBOL_GPL(xas_set_mark);
864+
865+/**
866+ * xas_clear_mark() - Clears the mark on this entry and its parents.
867+ * @xas: XArray operation state.
868+ * @mark: Mark number.
869+ *
870+ * Clears the specified mark on this entry, and walks back to the head
871+ * attempting to clear it on all the ancestor entries. Does nothing if
872+ * @xas has not been walked to an entry, or is in an error state.
873+ */
874+void xas_clear_mark(const struct xa_state *xas, xa_mark_t mark)
875+{
876+ struct xa_node *node = xas->xa_node;
877+ unsigned int offset = xas->xa_offset;
878+
879+ if (xas_invalid(xas))
880+ return;
881+
882+ while (node) {
883+ if (!node_clear_mark(node, offset, mark))
884+ return;
885+ if (node_any_mark(node, mark))
886+ return;
887+
888+ offset = node->offset;
889+ node = xa_parent_locked(xas->xa, node);
890+ }
891+
892+ if (xa_marked(xas->xa, mark))
893+ xa_mark_clear(xas->xa, mark);
894+}
895+EXPORT_SYMBOL_GPL(xas_clear_mark);
896+
897+/**
898+ * xas_init_marks() - Initialise all marks for the entry
899+ * @xas: Array operations state.
900+ *
901+ * Initialise all marks for the entry specified by @xas. If we're tracking
902+ * free entries with a mark, we need to set it on all entries. All other
903+ * marks are cleared.
904+ *
905+ * This implementation is not as efficient as it could be; we may walk
906+ * up the tree multiple times.
907+ */
908+void xas_init_marks(const struct xa_state *xas)
909+{
910+ xa_mark_t mark = 0;
911+
912+ for (;;) {
913+ if (xa_track_free(xas->xa) && mark == XA_FREE_MARK)
914+ xas_set_mark(xas, mark);
915+ else
916+ xas_clear_mark(xas, mark);
917+ if (mark == XA_MARK_MAX)
918+ break;
919+ mark_inc(mark);
920+ }
921+}
922+EXPORT_SYMBOL_GPL(xas_init_marks);
923+
924+/**
925+ * xas_pause() - Pause a walk to drop a lock.
926+ * @xas: XArray operation state.
927+ *
928+ * Some users need to pause a walk and drop the lock they're holding in
929+ * order to yield to a higher priority thread or carry out an operation
930+ * on an entry. Those users should call this function before they drop
931+ * the lock. It resets the @xas to be suitable for the next iteration
932+ * of the loop after the user has reacquired the lock. If most entries
933+ * found during a walk require you to call xas_pause(), the xa_for_each()
934+ * iterator may be more appropriate.
935+ *
936+ * Note that xas_pause() only works for forward iteration. If a user needs
937+ * to pause a reverse iteration, we will need a xas_pause_rev().
938+ */
939+void xas_pause(struct xa_state *xas)
940+{
941+ struct xa_node *node = xas->xa_node;
942+
943+ if (xas_invalid(xas))
944+ return;
945+
946+ if (node) {
947+ unsigned int offset = xas->xa_offset;
948+ while (++offset < XA_CHUNK_SIZE) {
949+ if (!xa_is_sibling(xa_entry(xas->xa, node, offset)))
950+ break;
951+ }
952+ xas->xa_index += (offset - xas->xa_offset) << node->shift;
953+ } else {
954+ xas->xa_index++;
955+ }
956+ xas->xa_node = XAS_RESTART;
957+}
958+EXPORT_SYMBOL_GPL(xas_pause);
959+
960+/*
961+ * __xas_prev() - Find the previous entry in the XArray.
962+ * @xas: XArray operation state.
963+ *
964+ * Helper function for xas_prev() which handles all the complex cases
965+ * out of line.
966+ */
967+void *__xas_prev(struct xa_state *xas)
968+{
969+ void *entry;
970+
971+ if (!xas_frozen(xas->xa_node))
972+ xas->xa_index--;
973+ if (xas_not_node(xas->xa_node))
974+ return xas_load(xas);
975+
976+ if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
977+ xas->xa_offset--;
978+
979+ while (xas->xa_offset == 255) {
980+ xas->xa_offset = xas->xa_node->offset - 1;
981+ xas->xa_node = xa_parent(xas->xa, xas->xa_node);
982+ if (!xas->xa_node)
983+ return set_bounds(xas);
984+ }
985+
986+ for (;;) {
987+ entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
988+ if (!xa_is_node(entry))
989+ return entry;
990+
991+ xas->xa_node = xa_to_node(entry);
992+ xas_set_offset(xas);
993+ }
994+}
995+EXPORT_SYMBOL_GPL(__xas_prev);
996+
997+/*
998+ * __xas_next() - Find the next entry in the XArray.
999+ * @xas: XArray operation state.
1000+ *
1001+ * Helper function for xas_next() which handles all the complex cases
1002+ * out of line.
1003+ */
1004+void *__xas_next(struct xa_state *xas)
1005+{
1006+ void *entry;
1007+
1008+ if (!xas_frozen(xas->xa_node))
1009+ xas->xa_index++;
1010+ if (xas_not_node(xas->xa_node))
1011+ return xas_load(xas);
1012+
1013+ if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
1014+ xas->xa_offset++;
1015+
1016+ while (xas->xa_offset == XA_CHUNK_SIZE) {
1017+ xas->xa_offset = xas->xa_node->offset + 1;
1018+ xas->xa_node = xa_parent(xas->xa, xas->xa_node);
1019+ if (!xas->xa_node)
1020+ return set_bounds(xas);
1021+ }
1022+
1023+ for (;;) {
1024+ entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
1025+ if (!xa_is_node(entry))
1026+ return entry;
1027+
1028+ xas->xa_node = xa_to_node(entry);
1029+ xas_set_offset(xas);
1030+ }
1031+}
1032+EXPORT_SYMBOL_GPL(__xas_next);
1033+
1034+/**
1035+ * xas_find() - Find the next present entry in the XArray.
1036+ * @xas: XArray operation state.
1037+ * @max: Highest index to return.
1038+ *
1039+ * If the @xas has not yet been walked to an entry, return the entry
1040+ * which has an index >= xas.xa_index. If it has been walked, the entry
1041+ * currently being pointed at has been processed, and so we move to the
1042+ * next entry.
1043+ *
1044+ * If no entry is found and the array is smaller than @max, the iterator
1045+ * is set to the smallest index not yet in the array. This allows @xas
1046+ * to be immediately passed to xas_store().
1047+ *
1048+ * Return: The entry, if found, otherwise %NULL.
1049+ */
1050+void *xas_find(struct xa_state *xas, unsigned long max)
1051+{
1052+ void *entry;
1053+
1054+ if (xas_error(xas))
1055+ return NULL;
1056+
1057+ if (!xas->xa_node) {
1058+ xas->xa_index = 1;
1059+ return set_bounds(xas);
1060+ } else if (xas_top(xas->xa_node)) {
1061+ entry = xas_load(xas);
1062+ if (entry || xas_not_node(xas->xa_node))
1063+ return entry;
1064+ } else if (!xas->xa_node->shift &&
1065+ xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)) {
1066+ xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
1067+ }
1068+
1069+ xas_advance(xas);
1070+
1071+ while (xas->xa_node && (xas->xa_index <= max)) {
1072+ if (unlikely(xas->xa_offset == XA_CHUNK_SIZE)) {
1073+ xas->xa_offset = xas->xa_node->offset + 1;
1074+ xas->xa_node = xa_parent(xas->xa, xas->xa_node);
1075+ continue;
1076+ }
1077+
1078+ entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
1079+ if (xa_is_node(entry)) {
1080+ xas->xa_node = xa_to_node(entry);
1081+ xas->xa_offset = 0;
1082+ continue;
1083+ }
1084+ if (entry && !xa_is_sibling(entry))
1085+ return entry;
1086+
1087+ xas_advance(xas);
1088+ }
1089+
1090+ if (!xas->xa_node)
1091+ xas->xa_node = XAS_BOUNDS;
1092+ return NULL;
1093+}
1094+EXPORT_SYMBOL_GPL(xas_find);
1095+
1096+/**
1097+ * xas_find_marked() - Find the next marked entry in the XArray.
1098+ * @xas: XArray operation state.
1099+ * @max: Highest index to return.
1100+ * @mark: Mark number to search for.
1101+ *
1102+ * If the @xas has not yet been walked to an entry, return the marked entry
1103+ * which has an index >= xas.xa_index. If it has been walked, the entry
1104+ * currently being pointed at has been processed, and so we return the
1105+ * first marked entry with an index > xas.xa_index.
1106+ *
1107+ * If no marked entry is found and the array is smaller than @max, @xas is
1108+ * set to the bounds state and xas->xa_index is set to the smallest index
1109+ * not yet in the array. This allows @xas to be immediately passed to
1110+ * xas_store().
1111+ *
1112+ * If no entry is found before @max is reached, @xas is set to the restart
1113+ * state.
1114+ *
1115+ * Return: The entry, if found, otherwise %NULL.
1116+ */
1117+void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark)
1118+{
1119+ bool advance = true;
1120+ unsigned int offset;
1121+ void *entry;
1122+
1123+ if (xas_error(xas))
1124+ return NULL;
1125+
1126+ if (!xas->xa_node) {
1127+ xas->xa_index = 1;
1128+ goto out;
1129+ } else if (xas_top(xas->xa_node)) {
1130+ advance = false;
1131+ entry = xa_head(xas->xa);
1132+ xas->xa_node = NULL;
1133+ if (xas->xa_index > max_index(entry))
1134+ goto bounds;
1135+ if (!xa_is_node(entry)) {
1136+ if (xa_marked(xas->xa, mark))
1137+ return entry;
1138+ xas->xa_index = 1;
1139+ goto out;
1140+ }
1141+ xas->xa_node = xa_to_node(entry);
1142+ xas->xa_offset = xas->xa_index >> xas->xa_node->shift;
1143+ }
1144+
1145+ while (xas->xa_index <= max) {
1146+ if (unlikely(xas->xa_offset == XA_CHUNK_SIZE)) {
1147+ xas->xa_offset = xas->xa_node->offset + 1;
1148+ xas->xa_node = xa_parent(xas->xa, xas->xa_node);
1149+ if (!xas->xa_node)
1150+ break;
1151+ advance = false;
1152+ continue;
1153+ }
1154+
1155+ if (!advance) {
1156+ entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
1157+ if (xa_is_sibling(entry)) {
1158+ xas->xa_offset = xa_to_sibling(entry);
1159+ xas_move_index(xas, xas->xa_offset);
1160+ }
1161+ }
1162+
1163+ offset = xas_find_chunk(xas, advance, mark);
1164+ if (offset > xas->xa_offset) {
1165+ advance = false;
1166+ xas_move_index(xas, offset);
1167+ /* Mind the wrap */
1168+ if ((xas->xa_index - 1) >= max)
1169+ goto max;
1170+ xas->xa_offset = offset;
1171+ if (offset == XA_CHUNK_SIZE)
1172+ continue;
1173+ }
1174+
1175+ entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
1176+ if (!xa_is_node(entry))
1177+ return entry;
1178+ xas->xa_node = xa_to_node(entry);
1179+ xas_set_offset(xas);
1180+ }
1181+
1182+out:
1183+ if (!max)
1184+ goto max;
1185+bounds:
1186+ xas->xa_node = XAS_BOUNDS;
1187+ return NULL;
1188+max:
1189+ xas->xa_node = XAS_RESTART;
1190+ return NULL;
1191+}
1192+EXPORT_SYMBOL_GPL(xas_find_marked);
1193+
1194+/**
1195+ * xas_find_conflict() - Find the next present entry in a range.
1196+ * @xas: XArray operation state.
1197+ *
1198+ * The @xas describes both a range and a position within that range.
1199+ *
1200+ * Context: Any context. Expects xa_lock to be held.
1201+ * Return: The next entry in the range covered by @xas or %NULL.
1202+ */
1203+void *xas_find_conflict(struct xa_state *xas)
1204+{
1205+ void *curr;
1206+
1207+ if (xas_error(xas))
1208+ return NULL;
1209+
1210+ if (!xas->xa_node)
1211+ return NULL;
1212+
1213+ if (xas_top(xas->xa_node)) {
1214+ curr = xas_start(xas);
1215+ if (!curr)
1216+ return NULL;
1217+ while (xa_is_node(curr)) {
1218+ struct xa_node *node = xa_to_node(curr);
1219+ curr = xas_descend(xas, node);
1220+ }
1221+ if (curr)
1222+ return curr;
1223+ }
1224+
1225+ if (xas->xa_node->shift > xas->xa_shift)
1226+ return NULL;
1227+
1228+ for (;;) {
1229+ if (xas->xa_node->shift == xas->xa_shift) {
1230+ if ((xas->xa_offset & xas->xa_sibs) == xas->xa_sibs)
1231+ break;
1232+ } else if (xas->xa_offset == XA_CHUNK_MASK) {
1233+ xas->xa_offset = xas->xa_node->offset;
1234+ xas->xa_node = xa_parent_locked(xas->xa, xas->xa_node);
1235+ if (!xas->xa_node)
1236+ break;
1237+ continue;
1238+ }
1239+ curr = xa_entry_locked(xas->xa, xas->xa_node, ++xas->xa_offset);
1240+ if (xa_is_sibling(curr))
1241+ continue;
1242+ while (xa_is_node(curr)) {
1243+ xas->xa_node = xa_to_node(curr);
1244+ xas->xa_offset = 0;
1245+ curr = xa_entry_locked(xas->xa, xas->xa_node, 0);
1246+ }
1247+ if (curr)
1248+ return curr;
1249+ }
1250+ xas->xa_offset -= xas->xa_sibs;
1251+ return NULL;
1252+}
1253+EXPORT_SYMBOL_GPL(xas_find_conflict);
1254+
1255+/**
1256+ * xa_init_flags() - Initialise an empty XArray with flags.
1257+ * @xa: XArray.
1258+ * @flags: XA_FLAG values.
1259+ *
1260+ * If you need to initialise an XArray with special flags (eg you need
1261+ * to take the lock from interrupt context), use this function instead
1262+ * of xa_init().
1263+ *
1264+ * Context: Any context.
1265+ */
1266+void xa_init_flags(struct xarray *xa, gfp_t flags)
1267+{
1268+ unsigned int lock_type;
1269+ static struct lock_class_key xa_lock_irq;
1270+ static struct lock_class_key xa_lock_bh;
1271+
1272+ spin_lock_init(&xa->xa_lock);
1273+ xa->xa_flags = flags;
1274+ xa->xa_head = NULL;
1275+
1276+ lock_type = xa_lock_type(xa);
1277+ if (lock_type == XA_LOCK_IRQ)
1278+ lockdep_set_class(&xa->xa_lock, &xa_lock_irq);
1279+ else if (lock_type == XA_LOCK_BH)
1280+ lockdep_set_class(&xa->xa_lock, &xa_lock_bh);
1281+}
1282+EXPORT_SYMBOL(xa_init_flags);
1283+
1284+/**
1285+ * xa_load() - Load an entry from an XArray.
1286+ * @xa: XArray.
1287+ * @index: index into array.
1288+ *
1289+ * Context: Any context. Takes and releases the RCU lock.
1290+ * Return: The entry at @index in @xa.
1291+ */
1292+void *xa_load(struct xarray *xa, unsigned long index)
1293+{
1294+ XA_STATE(xas, xa, index);
1295+ void *entry;
1296+
1297+ rcu_read_lock();
1298+ do {
1299+ entry = xas_load(&xas);
1300+ if (xa_is_zero(entry))
1301+ entry = NULL;
1302+ } while (xas_retry(&xas, entry));
1303+ rcu_read_unlock();
1304+
1305+ return entry;
1306+}
1307+EXPORT_SYMBOL(xa_load);
1308+
1309+static void *xas_result(struct xa_state *xas, void *curr)
1310+{
1311+ if (xa_is_zero(curr))
1312+ return NULL;
1313+ XA_NODE_BUG_ON(xas->xa_node, xa_is_internal(curr));
1314+ if (xas_error(xas))
1315+ curr = xas->xa_node;
1316+ return curr;
1317+}
1318+
1319+/**
1320+ * __xa_erase() - Erase this entry from the XArray while locked.
1321+ * @xa: XArray.
1322+ * @index: Index into array.
1323+ *
1324+ * If the entry at this index is a multi-index entry then all indices will
1325+ * be erased, and the entry will no longer be a multi-index entry.
1326+ * This function expects the xa_lock to be held on entry.
1327+ *
1328+ * Context: Any context. Expects xa_lock to be held on entry. May
1329+ * release and reacquire xa_lock if @gfp flags permit.
1330+ * Return: The old entry at this index.
1331+ */
1332+void *__xa_erase(struct xarray *xa, unsigned long index)
1333+{
1334+ XA_STATE(xas, xa, index);
1335+ return xas_result(&xas, xas_store(&xas, NULL));
1336+}
1337+EXPORT_SYMBOL_GPL(__xa_erase);
1338+
1339+/**
1340+ * xa_store() - Store this entry in the XArray.
1341+ * @xa: XArray.
1342+ * @index: Index into array.
1343+ * @entry: New entry.
1344+ * @gfp: Memory allocation flags.
1345+ *
1346+ * After this function returns, loads from this index will return @entry.
1347+ * Storing into an existing multislot entry updates the entry of every index.
1348+ * The marks associated with @index are unaffected unless @entry is %NULL.
1349+ *
1350+ * Context: Process context. Takes and releases the xa_lock. May sleep
1351+ * if the @gfp flags permit.
1352+ * Return: The old entry at this index on success, xa_err(-EINVAL) if @entry
1353+ * cannot be stored in an XArray, or xa_err(-ENOMEM) if memory allocation
1354+ * failed.
1355+ */
1356+void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
1357+{
1358+ XA_STATE(xas, xa, index);
1359+ void *curr;
1360+
1361+ if (WARN_ON_ONCE(xa_is_internal(entry)))
1362+ return XA_ERROR(-EINVAL);
1363+
1364+ do {
1365+ xas_lock(&xas);
1366+ curr = xas_store(&xas, entry);
1367+ if (xa_track_free(xa) && entry)
1368+ xas_clear_mark(&xas, XA_FREE_MARK);
1369+ xas_unlock(&xas);
1370+ } while (xas_nomem(&xas, gfp));
1371+
1372+ return xas_result(&xas, curr);
1373+}
1374+EXPORT_SYMBOL(xa_store);
1375+
1376+/**
1377+ * __xa_store() - Store this entry in the XArray.
1378+ * @xa: XArray.
1379+ * @index: Index into array.
1380+ * @entry: New entry.
1381+ * @gfp: Memory allocation flags.
1382+ *
1383+ * You must already be holding the xa_lock when calling this function.
1384+ * It will drop the lock if needed to allocate memory, and then reacquire
1385+ * it afterwards.
1386+ *
1387+ * Context: Any context. Expects xa_lock to be held on entry. May
1388+ * release and reacquire xa_lock if @gfp flags permit.
1389+ * Return: The old entry at this index or xa_err() if an error happened.
1390+ */
1391+void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
1392+{
1393+ XA_STATE(xas, xa, index);
1394+ void *curr;
1395+
1396+ if (WARN_ON_ONCE(xa_is_internal(entry)))
1397+ return XA_ERROR(-EINVAL);
1398+
1399+ do {
1400+ curr = xas_store(&xas, entry);
1401+ if (xa_track_free(xa) && entry)
1402+ xas_clear_mark(&xas, XA_FREE_MARK);
1403+ } while (__xas_nomem(&xas, gfp));
1404+
1405+ return xas_result(&xas, curr);
1406+}
1407+EXPORT_SYMBOL(__xa_store);
1408+
1409+/**
1410+ * xa_cmpxchg() - Conditionally replace an entry in the XArray.
1411+ * @xa: XArray.
1412+ * @index: Index into array.
1413+ * @old: Old value to test against.
1414+ * @entry: New value to place in array.
1415+ * @gfp: Memory allocation flags.
1416+ *
1417+ * If the entry at @index is the same as @old, replace it with @entry.
1418+ * If the return value is equal to @old, then the exchange was successful.
1419+ *
1420+ * Context: Process context. Takes and releases the xa_lock. May sleep
1421+ * if the @gfp flags permit.
1422+ * Return: The old value at this index or xa_err() if an error happened.
1423+ */
1424+void *xa_cmpxchg(struct xarray *xa, unsigned long index,
1425+ void *old, void *entry, gfp_t gfp)
1426+{
1427+ XA_STATE(xas, xa, index);
1428+ void *curr;
1429+
1430+ if (WARN_ON_ONCE(xa_is_internal(entry)))
1431+ return XA_ERROR(-EINVAL);
1432+
1433+ do {
1434+ xas_lock(&xas);
1435+ curr = xas_load(&xas);
1436+ if (curr == XA_ZERO_ENTRY)
1437+ curr = NULL;
1438+ if (curr == old) {
1439+ xas_store(&xas, entry);
1440+ if (xa_track_free(xa) && entry)
1441+ xas_clear_mark(&xas, XA_FREE_MARK);
1442+ }
1443+ xas_unlock(&xas);
1444+ } while (xas_nomem(&xas, gfp));
1445+
1446+ return xas_result(&xas, curr);
1447+}
1448+EXPORT_SYMBOL(xa_cmpxchg);
1449+
1450+/**
1451+ * __xa_cmpxchg() - Store this entry in the XArray.
1452+ * @xa: XArray.
1453+ * @index: Index into array.
1454+ * @old: Old value to test against.
1455+ * @entry: New entry.
1456+ * @gfp: Memory allocation flags.
1457+ *
1458+ * You must already be holding the xa_lock when calling this function.
1459+ * It will drop the lock if needed to allocate memory, and then reacquire
1460+ * it afterwards.
1461+ *
1462+ * Context: Any context. Expects xa_lock to be held on entry. May
1463+ * release and reacquire xa_lock if @gfp flags permit.
1464+ * Return: The old entry at this index or xa_err() if an error happened.
1465+ */
1466+void *__xa_cmpxchg(struct xarray *xa, unsigned long index,
1467+ void *old, void *entry, gfp_t gfp)
1468+{
1469+ XA_STATE(xas, xa, index);
1470+ void *curr;
1471+
1472+ if (WARN_ON_ONCE(xa_is_internal(entry)))
1473+ return XA_ERROR(-EINVAL);
1474+
1475+ do {
1476+ curr = xas_load(&xas);
1477+ if (curr == XA_ZERO_ENTRY)
1478+ curr = NULL;
1479+ if (curr == old) {
1480+ xas_store(&xas, entry);
1481+ if (xa_track_free(xa) && entry)
1482+ xas_clear_mark(&xas, XA_FREE_MARK);
1483+ }
1484+ } while (__xas_nomem(&xas, gfp));
1485+
1486+ return xas_result(&xas, curr);
1487+}
1488+EXPORT_SYMBOL(__xa_cmpxchg);
1489+
1490+/**
1491+ * xa_reserve() - Reserve this index in the XArray.
1492+ * @xa: XArray.
1493+ * @index: Index into array.
1494+ * @gfp: Memory allocation flags.
1495+ *
1496+ * Ensures there is somewhere to store an entry at @index in the array.
1497+ * If there is already something stored at @index, this function does
1498+ * nothing. If there was nothing there, the entry is marked as reserved.
1499+ * Loads from @index will continue to see a %NULL pointer until a
1500+ * subsequent store to @index.
1501+ *
1502+ * If you do not use the entry that you have reserved, call xa_release()
1503+ * or xa_erase() to free any unnecessary memory.
1504+ *
1505+ * Context: Process context. Takes and releases the xa_lock, IRQ or BH safe
1506+ * if specified in XArray flags. May sleep if the @gfp flags permit.
1507+ * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
1508+ */
1509+int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)
1510+{
1511+ XA_STATE(xas, xa, index);
1512+ unsigned int lock_type = xa_lock_type(xa);
1513+ void *curr;
1514+
1515+ do {
1516+ xas_lock_type(&xas, lock_type);
1517+ curr = xas_load(&xas);
1518+ if (!curr)
1519+ xas_store(&xas, XA_ZERO_ENTRY);
1520+ xas_unlock_type(&xas, lock_type);
1521+ } while (xas_nomem(&xas, gfp));
1522+
1523+ return xas_error(&xas);
1524+}
1525+EXPORT_SYMBOL(xa_reserve);
1526+
1527+#ifdef CONFIG_XARRAY_MULTI
1528+static void xas_set_range(struct xa_state *xas, unsigned long first,
1529+ unsigned long last)
1530+{
1531+ unsigned int shift = 0;
1532+ unsigned long sibs = last - first;
1533+ unsigned int offset = XA_CHUNK_MASK;
1534+
1535+ xas_set(xas, first);
1536+
1537+ while ((first & XA_CHUNK_MASK) == 0) {
1538+ if (sibs < XA_CHUNK_MASK)
1539+ break;
1540+ if ((sibs == XA_CHUNK_MASK) && (offset < XA_CHUNK_MASK))
1541+ break;
1542+ shift += XA_CHUNK_SHIFT;
1543+ if (offset == XA_CHUNK_MASK)
1544+ offset = sibs & XA_CHUNK_MASK;
1545+ sibs >>= XA_CHUNK_SHIFT;
1546+ first >>= XA_CHUNK_SHIFT;
1547+ }
1548+
1549+ offset = first & XA_CHUNK_MASK;
1550+ if (offset + sibs > XA_CHUNK_MASK)
1551+ sibs = XA_CHUNK_MASK - offset;
1552+ if ((((first + sibs + 1) << shift) - 1) > last)
1553+ sibs -= 1;
1554+
1555+ xas->xa_shift = shift;
1556+ xas->xa_sibs = sibs;
1557+}
1558+
1559+/**
1560+ * xa_store_range() - Store this entry at a range of indices in the XArray.
1561+ * @xa: XArray.
1562+ * @first: First index to affect.
1563+ * @last: Last index to affect.
1564+ * @entry: New entry.
1565+ * @gfp: Memory allocation flags.
1566+ *
1567+ * After this function returns, loads from any index between @first and @last,
1568+ * inclusive will return @entry.
1569+ * Storing into an existing multislot entry updates the entry of every index.
1570+ * The marks associated with @index are unaffected unless @entry is %NULL.
1571+ *
1572+ * Context: Process context. Takes and releases the xa_lock. May sleep
1573+ * if the @gfp flags permit.
1574+ * Return: %NULL on success, xa_err(-EINVAL) if @entry cannot be stored in
1575+ * an XArray, or xa_err(-ENOMEM) if memory allocation failed.
1576+ */
1577+void *xa_store_range(struct xarray *xa, unsigned long first,
1578+ unsigned long last, void *entry, gfp_t gfp)
1579+{
1580+ XA_STATE(xas, xa, 0);
1581+
1582+ if (WARN_ON_ONCE(xa_is_internal(entry)))
1583+ return XA_ERROR(-EINVAL);
1584+ if (last < first)
1585+ return XA_ERROR(-EINVAL);
1586+
1587+ do {
1588+ xas_lock(&xas);
1589+ if (entry) {
1590+ unsigned int order = (last == ~0UL) ? 64 :
1591+ ilog2(last + 1);
1592+ xas_set_order(&xas, last, order);
1593+ xas_create(&xas);
1594+ if (xas_error(&xas))
1595+ goto unlock;
1596+ }
1597+ do {
1598+ xas_set_range(&xas, first, last);
1599+ xas_store(&xas, entry);
1600+ if (xas_error(&xas))
1601+ goto unlock;
1602+ first += xas_size(&xas);
1603+ } while (first <= last);
1604+unlock:
1605+ xas_unlock(&xas);
1606+ } while (xas_nomem(&xas, gfp));
1607+
1608+ return xas_result(&xas, NULL);
1609+}
1610+EXPORT_SYMBOL(xa_store_range);
1611+#endif /* CONFIG_XARRAY_MULTI */
1612+
1613+/**
1614+ * __xa_alloc() - Find somewhere to store this entry in the XArray.
1615+ * @xa: XArray.
1616+ * @id: Pointer to ID.
1617+ * @max: Maximum ID to allocate (inclusive).
1618+ * @entry: New entry.
1619+ * @gfp: Memory allocation flags.
1620+ *
1621+ * Allocates an unused ID in the range specified by @id and @max.
1622+ * Updates the @id pointer with the index, then stores the entry at that
1623+ * index. A concurrent lookup will not see an uninitialised @id.
1624+ *
1625+ * Context: Any context. Expects xa_lock to be held on entry. May
1626+ * release and reacquire xa_lock if @gfp flags permit.
1627+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
1628+ * there is no more space in the XArray.
1629+ */
1630+int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp)
1631+{
1632+ XA_STATE(xas, xa, 0);
1633+ int err;
1634+
1635+ if (WARN_ON_ONCE(xa_is_internal(entry)))
1636+ return -EINVAL;
1637+ if (WARN_ON_ONCE(!xa_track_free(xa)))
1638+ return -EINVAL;
1639+
1640+ if (!entry)
1641+ entry = XA_ZERO_ENTRY;
1642+
1643+ do {
1644+ xas.xa_index = *id;
1645+ xas_find_marked(&xas, max, XA_FREE_MARK);
1646+ if (xas.xa_node == XAS_RESTART)
1647+ xas_set_err(&xas, -ENOSPC);
1648+ xas_store(&xas, entry);
1649+ xas_clear_mark(&xas, XA_FREE_MARK);
1650+ } while (__xas_nomem(&xas, gfp));
1651+
1652+ err = xas_error(&xas);
1653+ if (!err)
1654+ *id = xas.xa_index;
1655+ return err;
1656+}
1657+EXPORT_SYMBOL(__xa_alloc);
1658+
1659+/**
1660+ * __xa_set_mark() - Set this mark on this entry while locked.
1661+ * @xa: XArray.
1662+ * @index: Index of entry.
1663+ * @mark: Mark number.
1664+ *
1665+ * Attempting to set a mark on a NULL entry does not succeed.
1666+ *
1667+ * Context: Any context. Expects xa_lock to be held on entry.
1668+ */
1669+void __xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
1670+{
1671+ XA_STATE(xas, xa, index);
1672+ void *entry = xas_load(&xas);
1673+
1674+ if (entry)
1675+ xas_set_mark(&xas, mark);
1676+}
1677+EXPORT_SYMBOL_GPL(__xa_set_mark);
1678+
1679+/**
1680+ * __xa_clear_mark() - Clear this mark on this entry while locked.
1681+ * @xa: XArray.
1682+ * @index: Index of entry.
1683+ * @mark: Mark number.
1684+ *
1685+ * Context: Any context. Expects xa_lock to be held on entry.
1686+ */
1687+void __xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
1688+{
1689+ XA_STATE(xas, xa, index);
1690+ void *entry = xas_load(&xas);
1691+
1692+ if (entry)
1693+ xas_clear_mark(&xas, mark);
1694+}
1695+EXPORT_SYMBOL_GPL(__xa_clear_mark);
1696+
1697+/**
1698+ * xa_get_mark() - Inquire whether this mark is set on this entry.
1699+ * @xa: XArray.
1700+ * @index: Index of entry.
1701+ * @mark: Mark number.
1702+ *
1703+ * This function uses the RCU read lock, so the result may be out of date
1704+ * by the time it returns. If you need the result to be stable, use a lock.
1705+ *
1706+ * Context: Any context. Takes and releases the RCU lock.
1707+ * Return: True if the entry at @index has this mark set, false if it doesn't.
1708+ */
1709+bool xa_get_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
1710+{
1711+ XA_STATE(xas, xa, index);
1712+ void *entry;
1713+
1714+ rcu_read_lock();
1715+ entry = xas_start(&xas);
1716+ while (xas_get_mark(&xas, mark)) {
1717+ if (!xa_is_node(entry))
1718+ goto found;
1719+ entry = xas_descend(&xas, xa_to_node(entry));
1720+ }
1721+ rcu_read_unlock();
1722+ return false;
1723+ found:
1724+ rcu_read_unlock();
1725+ return true;
1726+}
1727+EXPORT_SYMBOL(xa_get_mark);
1728+
1729+/**
1730+ * xa_set_mark() - Set this mark on this entry.
1731+ * @xa: XArray.
1732+ * @index: Index of entry.
1733+ * @mark: Mark number.
1734+ *
1735+ * Attempting to set a mark on a NULL entry does not succeed.
1736+ *
1737+ * Context: Process context. Takes and releases the xa_lock.
1738+ */
1739+void xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
1740+{
1741+ xa_lock(xa);
1742+ __xa_set_mark(xa, index, mark);
1743+ xa_unlock(xa);
1744+}
1745+EXPORT_SYMBOL(xa_set_mark);
1746+
1747+/**
1748+ * xa_clear_mark() - Clear this mark on this entry.
1749+ * @xa: XArray.
1750+ * @index: Index of entry.
1751+ * @mark: Mark number.
1752+ *
1753+ * Clearing a mark always succeeds.
1754+ *
1755+ * Context: Process context. Takes and releases the xa_lock.
1756+ */
1757+void xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)
1758+{
1759+ xa_lock(xa);
1760+ __xa_clear_mark(xa, index, mark);
1761+ xa_unlock(xa);
1762+}
1763+EXPORT_SYMBOL(xa_clear_mark);
1764+
1765+/**
1766+ * xa_find() - Search the XArray for an entry.
1767+ * @xa: XArray.
1768+ * @indexp: Pointer to an index.
1769+ * @max: Maximum index to search to.
1770+ * @filter: Selection criterion.
1771+ *
1772+ * Finds the entry in @xa which matches the @filter, and has the lowest
1773+ * index that is at least @indexp and no more than @max.
1774+ * If an entry is found, @indexp is updated to be the index of the entry.
1775+ * This function is protected by the RCU read lock, so it may not find
1776+ * entries which are being simultaneously added. It will not return an
1777+ * %XA_RETRY_ENTRY; if you need to see retry entries, use xas_find().
1778+ *
1779+ * Context: Any context. Takes and releases the RCU lock.
1780+ * Return: The entry, if found, otherwise %NULL.
1781+ */
1782+void *xa_find(struct xarray *xa, unsigned long *indexp,
1783+ unsigned long max, xa_mark_t filter)
1784+{
1785+ XA_STATE(xas, xa, *indexp);
1786+ void *entry;
1787+
1788+ rcu_read_lock();
1789+ do {
1790+ if ((__force unsigned int)filter < XA_MAX_MARKS)
1791+ entry = xas_find_marked(&xas, max, filter);
1792+ else
1793+ entry = xas_find(&xas, max);
1794+ } while (xas_retry(&xas, entry));
1795+ rcu_read_unlock();
1796+
1797+ if (entry)
1798+ *indexp = xas.xa_index;
1799+ return entry;
1800+}
1801+EXPORT_SYMBOL(xa_find);
1802+
1803+/**
1804+ * xa_find_after() - Search the XArray for a present entry.
1805+ * @xa: XArray.
1806+ * @indexp: Pointer to an index.
1807+ * @max: Maximum index to search to.
1808+ * @filter: Selection criterion.
1809+ *
1810+ * Finds the entry in @xa which matches the @filter and has the lowest
1811+ * index that is above @indexp and no more than @max.
1812+ * If an entry is found, @indexp is updated to be the index of the entry.
1813+ * This function is protected by the RCU read lock, so it may miss entries
1814+ * which are being simultaneously added. It will not return an
1815+ * %XA_RETRY_ENTRY; if you need to see retry entries, use xas_find().
1816+ *
1817+ * Context: Any context. Takes and releases the RCU lock.
1818+ * Return: The pointer, if found, otherwise %NULL.
1819+ */
1820+void *xa_find_after(struct xarray *xa, unsigned long *indexp,
1821+ unsigned long max, xa_mark_t filter)
1822+{
1823+ XA_STATE(xas, xa, *indexp + 1);
1824+ void *entry;
1825+
1826+ rcu_read_lock();
1827+ for (;;) {
1828+ if ((__force unsigned int)filter < XA_MAX_MARKS)
1829+ entry = xas_find_marked(&xas, max, filter);
1830+ else
1831+ entry = xas_find(&xas, max);
1832+ if (xas.xa_shift) {
1833+ if (xas.xa_index & ((1UL << xas.xa_shift) - 1))
1834+ continue;
1835+ } else {
1836+ if (xas.xa_offset < (xas.xa_index & XA_CHUNK_MASK))
1837+ continue;
1838+ }
1839+ if (!xas_retry(&xas, entry))
1840+ break;
1841+ }
1842+ rcu_read_unlock();
1843+
1844+ if (entry)
1845+ *indexp = xas.xa_index;
1846+ return entry;
1847+}
1848+EXPORT_SYMBOL(xa_find_after);
1849+
1850+static unsigned int xas_extract_present(struct xa_state *xas, void **dst,
1851+ unsigned long max, unsigned int n)
1852+{
1853+ void *entry;
1854+ unsigned int i = 0;
1855+
1856+ rcu_read_lock();
1857+ xas_for_each(xas, entry, max) {
1858+ if (xas_retry(xas, entry))
1859+ continue;
1860+ dst[i++] = entry;
1861+ if (i == n)
1862+ break;
1863+ }
1864+ rcu_read_unlock();
1865+
1866+ return i;
1867+}
1868+
1869+static unsigned int xas_extract_marked(struct xa_state *xas, void **dst,
1870+ unsigned long max, unsigned int n, xa_mark_t mark)
1871+{
1872+ void *entry;
1873+ unsigned int i = 0;
1874+
1875+ rcu_read_lock();
1876+ xas_for_each_marked(xas, entry, max, mark) {
1877+ if (xas_retry(xas, entry))
1878+ continue;
1879+ dst[i++] = entry;
1880+ if (i == n)
1881+ break;
1882+ }
1883+ rcu_read_unlock();
1884+
1885+ return i;
1886+}
1887+
1888+/**
1889+ * xa_extract() - Copy selected entries from the XArray into a normal array.
1890+ * @xa: The source XArray to copy from.
1891+ * @dst: The buffer to copy entries into.
1892+ * @start: The first index in the XArray eligible to be selected.
1893+ * @max: The last index in the XArray eligible to be selected.
1894+ * @n: The maximum number of entries to copy.
1895+ * @filter: Selection criterion.
1896+ *
1897+ * Copies up to @n entries that match @filter from the XArray. The
1898+ * copied entries will have indices between @start and @max, inclusive.
1899+ *
1900+ * The @filter may be an XArray mark value, in which case entries which are
1901+ * marked with that mark will be copied. It may also be %XA_PRESENT, in
1902+ * which case all entries which are not NULL will be copied.
1903+ *
1904+ * The entries returned may not represent a snapshot of the XArray at a
1905+ * moment in time. For example, if another thread stores to index 5, then
1906+ * index 10, calling xa_extract() may return the old contents of index 5
1907+ * and the new contents of index 10. Indices not modified while this
1908+ * function is running will not be skipped.
1909+ *
1910+ * If you need stronger guarantees, holding the xa_lock across calls to this
1911+ * function will prevent concurrent modification.
1912+ *
1913+ * Context: Any context. Takes and releases the RCU lock.
1914+ * Return: The number of entries copied.
1915+ */
1916+unsigned int xa_extract(struct xarray *xa, void **dst, unsigned long start,
1917+ unsigned long max, unsigned int n, xa_mark_t filter)
1918+{
1919+ XA_STATE(xas, xa, start);
1920+
1921+ if (!n)
1922+ return 0;
1923+
1924+ if ((__force unsigned int)filter < XA_MAX_MARKS)
1925+ return xas_extract_marked(&xas, dst, max, n, filter);
1926+ return xas_extract_present(&xas, dst, max, n);
1927+}
1928+EXPORT_SYMBOL(xa_extract);
1929+
1930+/**
1931+ * xa_destroy() - Free all internal data structures.
1932+ * @xa: XArray.
1933+ *
1934+ * After calling this function, the XArray is empty and has freed all memory
1935+ * allocated for its internal data structures. You are responsible for
1936+ * freeing the objects referenced by the XArray.
1937+ *
1938+ * Context: Any context. Takes and releases the xa_lock, interrupt-safe.
1939+ */
1940+void xa_destroy(struct xarray *xa)
1941+{
1942+ XA_STATE(xas, xa, 0);
1943+ unsigned long flags;
1944+ void *entry;
1945+
1946+ xas.xa_node = NULL;
1947+ xas_lock_irqsave(&xas, flags);
1948+ entry = xa_head_locked(xa);
1949+ RCU_INIT_POINTER(xa->xa_head, NULL);
1950+ xas_init_marks(&xas);
1951+ /* lockdep checks we're still holding the lock in xas_free_nodes() */
1952+ if (xa_is_node(entry))
1953+ xas_free_nodes(&xas, xa_to_node(entry));
1954+ xas_unlock_irqrestore(&xas, flags);
1955+}
1956+EXPORT_SYMBOL(xa_destroy);
1957+
1958+#ifdef XA_DEBUG
1959+void xa_dump_node(const struct xa_node *node)
1960+{
1961+ unsigned i, j;
1962+
1963+ if (!node)
1964+ return;
1965+ if ((unsigned long)node & 3) {
1966+ pr_cont("node %px\n", node);
1967+ return;
1968+ }
1969+
1970+ pr_cont("node %px %s %d parent %px shift %d count %d values %d "
1971+ "array %px list %px %px marks",
1972+ node, node->parent ? "offset" : "max", node->offset,
1973+ node->parent, node->shift, node->count, node->nr_values,
1974+ node->array, node->private_list.prev, node->private_list.next);
1975+ for (i = 0; i < XA_MAX_MARKS; i++)
1976+ for (j = 0; j < XA_MARK_LONGS; j++)
1977+ pr_cont(" %lx", node->marks[i][j]);
1978+ pr_cont("\n");
1979+}
1980+
1981+void xa_dump_index(unsigned long index, unsigned int shift)
1982+{
1983+ if (!shift)
1984+ pr_info("%lu: ", index);
1985+ else if (shift >= BITS_PER_LONG)
1986+ pr_info("0-%lu: ", ~0UL);
1987+ else
1988+ pr_info("%lu-%lu: ", index, index | ((1UL << shift) - 1));
1989+}
1990+
1991+void xa_dump_entry(const void *entry, unsigned long index, unsigned long shift)
1992+{
1993+ if (!entry)
1994+ return;
1995+
1996+ xa_dump_index(index, shift);
1997+
1998+ if (xa_is_node(entry)) {
1999+ if (shift == 0) {
2000+ pr_cont("%px\n", entry);
2001+ } else {
2002+ unsigned long i;
2003+ struct xa_node *node = xa_to_node(entry);
2004+ xa_dump_node(node);
2005+ for (i = 0; i < XA_CHUNK_SIZE; i++)
2006+ xa_dump_entry(node->slots[i],
2007+ index + (i << node->shift), node->shift);
2008+ }
2009+ } else if (xa_is_value(entry))
2010+ pr_cont("value %ld (0x%lx) [%px]\n", xa_to_value(entry),
2011+ xa_to_value(entry), entry);
2012+ else if (!xa_is_internal(entry))
2013+ pr_cont("%px\n", entry);
2014+ else if (xa_is_retry(entry))
2015+ pr_cont("retry (%ld)\n", xa_to_internal(entry));
2016+ else if (xa_is_sibling(entry))
2017+ pr_cont("sibling (slot %ld)\n", xa_to_sibling(entry));
2018+ else if (xa_is_zero(entry))
2019+ pr_cont("zero (%ld)\n", xa_to_internal(entry));
2020+ else
2021+ pr_cont("UNKNOWN ENTRY (%px)\n", entry);
2022+}
2023+
2024+void xa_dump(const struct xarray *xa)
2025+{
2026+ void *entry = xa->xa_head;
2027+ unsigned int shift = 0;
2028+
2029+ pr_info("xarray: %px head %px flags %x marks %d %d %d\n", xa, entry,
2030+ xa->xa_flags, xa_marked(xa, XA_MARK_0),
2031+ xa_marked(xa, XA_MARK_1), xa_marked(xa, XA_MARK_2));
2032+ if (xa_is_node(entry))
2033+ shift = xa_to_node(entry)->shift + XA_CHUNK_SHIFT;
2034+ xa_dump_entry(entry, 0, shift);
2035+}
2036+#endif
mm/Kconfig+2−2
@@ -379,7 +379,7 @@ config TRANSPARENT_HUGEPAGE · 378 unmodified lines
379379 bool "Transparent Hugepage Support"
380380 depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE
381381 select COMPACTION
382− select RADIX_TREE_MULTIORDER
382+ select XARRAY_MULTI
383383 help
384384 Transparent Hugepages allows the kernel to use huge pages and
385385 huge tlb transparently to the applications whenever possible.
@@ -671,7 +671,7 @@ config ZONE_DEVICE · 285 unmodified lines
671671 depends on MEMORY_HOTREMOVE
672672 depends on SPARSEMEM_VMEMMAP
673673 depends on ARCH_HAS_ZONE_DEVICE
674− select RADIX_TREE_MULTIORDER
674+ select XARRAY_MULTI
675675
676676 help
677677 Device memory hotplug support allows for establishing pmem,
mm/filemap.c+310−414
@@ -113,60 +113,26 @@ · 112 unmodified lines
113113 * ->tasklist_lock (memory_failure, collect_procs_ao)
114114 */
115115
116−static int page_cache_tree_insert(struct address_space *mapping,
117− struct page *page, void **shadowp)
118−{
119− struct radix_tree_node *node;
120− void **slot;
121− int error;
122−
123− error = __radix_tree_create(&mapping->i_pages, page->index, 0,
124− &node, &slot);
125− if (error)
126− return error;
127− if (*slot) {
128− void *p;
129−
130− p = radix_tree_deref_slot_protected(slot,
131− &mapping->i_pages.xa_lock);
132− if (!radix_tree_exceptional_entry(p))
133− return -EEXIST;
134−
135− mapping->nrexceptional--;
136− if (shadowp)
137− *shadowp = p;
138− }
139− __radix_tree_replace(&mapping->i_pages, node, slot, page,
140− workingset_lookup_update(mapping));
141− mapping->nrpages++;
142− return 0;
143−}
144−
145−static void page_cache_tree_delete(struct address_space *mapping,
116+static void page_cache_delete(struct address_space *mapping,
146117 struct page *page, void *shadow)
147118 {
148− int i, nr;
119+ XA_STATE(xas, &mapping->i_pages, page->index);
120+ unsigned int nr = 1;
121+
122+ mapping_set_update(&xas, mapping);
149123
150− /* hugetlb pages are represented by one entry in the radix tree */
151− nr = PageHuge(page) ? 1 : hpage_nr_pages(page);
124+ /* hugetlb pages are represented by a single entry in the xarray */
125+ if (!PageHuge(page)) {
126+ xas_set_order(&xas, page->index, compound_order(page));
127+ nr = 1U << compound_order(page);
128+ }
152129
153130 VM_BUG_ON_PAGE(!PageLocked(page), page);
154131 VM_BUG_ON_PAGE(PageTail(page), page);
155132 VM_BUG_ON_PAGE(nr != 1 && shadow, page);
156133
157− for (i = 0; i < nr; i++) {
158− struct radix_tree_node *node;
159− void **slot;
160−
161− __radix_tree_lookup(&mapping->i_pages, page->index + i,
162− &node, &slot);
163−
164− VM_BUG_ON_PAGE(!node && nr != 1, page);
165−
166− radix_tree_clear_tags(&mapping->i_pages, node, slot);
167− __radix_tree_replace(&mapping->i_pages, node, slot, shadow,
168− workingset_lookup_update(mapping));
169− }
134+ xas_store(&xas, shadow);
135+ xas_init_marks(&xas);
170136
171137 page->mapping = NULL;
172138 /* Leave page->index set: truncation lookup relies upon it */
@@ -265,7 +231,7 @@ void __delete_from_page_cache(struct page *page, void *shadow) · 92 unmodified lines
265231 trace_mm_filemap_delete_from_page_cache(page);
266232
267233 unaccount_page_cache_page(mapping, page);
268− page_cache_tree_delete(mapping, page, shadow);
234+ page_cache_delete(mapping, page, shadow);
269235 }
270236
271237 static void page_cache_free_page(struct address_space *mapping,
@@ -308,7 +274,7 @@ void delete_from_page_cache(struct page *page) · 36 unmodified lines
308274 EXPORT_SYMBOL(delete_from_page_cache);
309275
310276 /*
311− * page_cache_tree_delete_batch - delete several pages from page cache
277+ * page_cache_delete_batch - delete several pages from page cache
312278 * @mapping: the mapping to which pages belong
313279 * @pvec: pagevec with pages to delete
314280 *
@@ -321,24 +287,19 @@ EXPORT_SYMBOL(delete_from_page_cache); · 6 unmodified lines
321287 *
322288 * The function expects the i_pages lock to be held.
323289 */
324−static void
325−page_cache_tree_delete_batch(struct address_space *mapping,
290+static void page_cache_delete_batch(struct address_space *mapping,
326291 struct pagevec *pvec)
327292 {
328− struct radix_tree_iter iter;
329− void **slot;
293+ XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);
330294 int total_pages = 0;
331295 int i = 0, tail_pages = 0;
332296 struct page *page;
333− pgoff_t start;
334297
335− start = pvec->pages[0]->index;
336− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
298+ mapping_set_update(&xas, mapping);
299+ xas_for_each(&xas, page, ULONG_MAX) {
337300 if (i >= pagevec_count(pvec) && !tail_pages)
338301 break;
339− page = radix_tree_deref_slot_protected(slot,
340− &mapping->i_pages.xa_lock);
341− if (radix_tree_exceptional_entry(page))
302+ if (xa_is_value(page))
342303 continue;
343304 if (!tail_pages) {
344305 /*
@@ -346,8 +307,11 @@ page_cache_tree_delete_batch(struct address_space *mapping, · 1 unmodified line
346307 * have our pages locked so they are protected from
347308 * being removed.
348309 */
349− if (page != pvec->pages[i])
310+ if (page != pvec->pages[i]) {
311+ VM_BUG_ON_PAGE(page->index >
312+ pvec->pages[i]->index, page);
350313 continue;
314+ }
351315 WARN_ON_ONCE(!PageLocked(page));
352316 if (PageTransHuge(page) && !PageHuge(page))
353317 tail_pages = HPAGE_PMD_NR - 1;
@@ -358,11 +322,11 @@ page_cache_tree_delete_batch(struct address_space *mapping, · 4 unmodified lines
358322 */
359323 i++;
360324 } else {
325+ VM_BUG_ON_PAGE(page->index + HPAGE_PMD_NR - tail_pages
326+ != pvec->pages[i]->index, page);
361327 tail_pages--;
362328 }
363− radix_tree_clear_tags(&mapping->i_pages, iter.node, slot);
364− __radix_tree_replace(&mapping->i_pages, iter.node, slot, NULL,
365− workingset_lookup_update(mapping));
329+ xas_store(&xas, NULL);
366330 total_pages++;
367331 }
368332 mapping->nrpages -= total_pages;
@@ -383,7 +347,7 @@ void delete_from_page_cache_batch(struct address_space *mapping, · 14 unmodified lines
383347
384348 unaccount_page_cache_page(mapping, pvec->pages[i]);
385349 }
386− page_cache_tree_delete_batch(mapping, pvec);
350+ page_cache_delete_batch(mapping, pvec);
387351 xa_unlock_irqrestore(&mapping->i_pages, flags);
388352
389353 for (i = 0; i < pagevec_count(pvec); i++)
@@ -493,20 +457,31 @@ EXPORT_SYMBOL(filemap_flush); · 103 unmodified lines
493457 bool filemap_range_has_page(struct address_space *mapping,
494458 loff_t start_byte, loff_t end_byte)
495459 {
496− pgoff_t index = start_byte >> PAGE_SHIFT;
497− pgoff_t end = end_byte >> PAGE_SHIFT;
498460 struct page *page;
461+ XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
462+ pgoff_t max = end_byte >> PAGE_SHIFT;
499463
500464 if (end_byte < start_byte)
501465 return false;
502466
503− if (mapping->nrpages == 0)
504− return false;
467+ rcu_read_lock();
468+ for (;;) {
469+ page = xas_find(&xas, max);
470+ if (xas_retry(&xas, page))
471+ continue;
472+ /* Shadow entries don't count */
473+ if (xa_is_value(page))
474+ continue;
475+ /*
476+ * We don't need to try to pin this page; we're about to
477+ * release the RCU lock anyway. It is enough to know that
478+ * there was a page here recently.
479+ */
480+ break;
481+ }
482+ rcu_read_unlock();
505483
506− if (!find_get_pages_range(mapping, &index, end, 1, &page))
507− return false;
508− put_page(page);
509− return true;
484+ return page != NULL;
510485 }
511486 EXPORT_SYMBOL(filemap_range_has_page);
512487
@@ -777,51 +752,44 @@ EXPORT_SYMBOL(file_write_and_wait_range); · 264 unmodified lines
777752 * locked. This function does not add the new page to the LRU, the
778753 * caller must do that.
779754 *
780− * The remove + add is atomic. The only way this function can fail is
781− * memory allocation failure.
755+ * The remove + add is atomic. This function cannot fail.
782756 */
783757 int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
784758 {
785− int error;
759+ struct address_space *mapping = old->mapping;
760+ void (*freepage)(struct page *) = mapping->a_ops->freepage;
761+ pgoff_t offset = old->index;
762+ XA_STATE(xas, &mapping->i_pages, offset);
763+ unsigned long flags;
786764
787765 VM_BUG_ON_PAGE(!PageLocked(old), old);
788766 VM_BUG_ON_PAGE(!PageLocked(new), new);
789767 VM_BUG_ON_PAGE(new->mapping, new);
790768
791− error = radix_tree_preload(gfp_mask & GFP_RECLAIM_MASK);
792− if (!error) {
793− struct address_space *mapping = old->mapping;
794− void (*freepage)(struct page *);
795− unsigned long flags;
796−
797− pgoff_t offset = old->index;
798− freepage = mapping->a_ops->freepage;
799−
800− get_page(new);
801− new->mapping = mapping;
802− new->index = offset;
769+ get_page(new);
770+ new->mapping = mapping;
771+ new->index = offset;
803772
804− xa_lock_irqsave(&mapping->i_pages, flags);
805− __delete_from_page_cache(old, NULL);
806− error = page_cache_tree_insert(mapping, new, NULL);
807− BUG_ON(error);
773+ xas_lock_irqsave(&xas, flags);
774+ xas_store(&xas, new);
808775
809− /*
810− * hugetlb pages do not participate in page cache accounting.
811− */
812− if (!PageHuge(new))
813− __inc_node_page_state(new, NR_FILE_PAGES);
814− if (PageSwapBacked(new))
815− __inc_node_page_state(new, NR_SHMEM);
816− xa_unlock_irqrestore(&mapping->i_pages, flags);
817− mem_cgroup_migrate(old, new);
818− radix_tree_preload_end();
819− if (freepage)
820− freepage(old);
821− put_page(old);
822− }
776+ old->mapping = NULL;
777+ /* hugetlb pages do not participate in page cache accounting. */
778+ if (!PageHuge(old))
779+ __dec_node_page_state(new, NR_FILE_PAGES);
780+ if (!PageHuge(new))
781+ __inc_node_page_state(new, NR_FILE_PAGES);
782+ if (PageSwapBacked(old))
783+ __dec_node_page_state(new, NR_SHMEM);
784+ if (PageSwapBacked(new))
785+ __inc_node_page_state(new, NR_SHMEM);
786+ xas_unlock_irqrestore(&xas, flags);
787+ mem_cgroup_migrate(old, new);
788+ if (freepage)
789+ freepage(old);
790+ put_page(old);
823791
824− return error;
792+ return 0;
825793 }
826794 EXPORT_SYMBOL_GPL(replace_page_cache_page);
827795
@@ -830,12 +798,15 @@ static int __add_to_page_cache_locked(struct page *page, · 2 unmodified lines
830798 pgoff_t offset, gfp_t gfp_mask,
831799 void **shadowp)
832800 {
801+ XA_STATE(xas, &mapping->i_pages, offset);
833802 int huge = PageHuge(page);
834803 struct mem_cgroup *memcg;
835804 int error;
805+ void *old;
836806
837807 VM_BUG_ON_PAGE(!PageLocked(page), page);
838808 VM_BUG_ON_PAGE(PageSwapBacked(page), page);
809+ mapping_set_update(&xas, mapping);
839810
840811 if (!huge) {
841812 error = mem_cgroup_try_charge(page, current->mm,
@@ -844,39 +815,47 @@ static int __add_to_page_cache_locked(struct page *page, · 2 unmodified lines
844815 return error;
845816 }
846817
847− error = radix_tree_maybe_preload(gfp_mask & GFP_RECLAIM_MASK);
848− if (error) {
849− if (!huge)
850− mem_cgroup_cancel_charge(page, memcg, false);
851− return error;
852− }
853−
854818 get_page(page);
855819 page->mapping = mapping;
856820 page->index = offset;
857821
858− xa_lock_irq(&mapping->i_pages);
859− error = page_cache_tree_insert(mapping, page, shadowp);
860− radix_tree_preload_end();
861− if (unlikely(error))
862− goto err_insert;
822+ do {
823+ xas_lock_irq(&xas);
824+ old = xas_load(&xas);
825+ if (old && !xa_is_value(old))
826+ xas_set_err(&xas, -EEXIST);
827+ xas_store(&xas, page);
828+ if (xas_error(&xas))
829+ goto unlock;
830+
831+ if (xa_is_value(old)) {
832+ mapping->nrexceptional--;
833+ if (shadowp)
834+ *shadowp = old;
835+ }
836+ mapping->nrpages++;
837+
838+ /* hugetlb pages do not participate in page cache accounting */
839+ if (!huge)
840+ __inc_node_page_state(page, NR_FILE_PAGES);
841+unlock:
842+ xas_unlock_irq(&xas);
843+ } while (xas_nomem(&xas, gfp_mask & GFP_RECLAIM_MASK));
844+
845+ if (xas_error(&xas))
846+ goto error;
863847
864− /* hugetlb pages do not participate in page cache accounting. */
865− if (!huge)
866− __inc_node_page_state(page, NR_FILE_PAGES);
867− xa_unlock_irq(&mapping->i_pages);
868848 if (!huge)
869849 mem_cgroup_commit_charge(page, memcg, false, false);
870850 trace_mm_filemap_add_to_page_cache(page);
871851 return 0;
872−err_insert:
852+error:
873853 page->mapping = NULL;
874854 /* Leave page->index set: truncation relies upon it */
875− xa_unlock_irq(&mapping->i_pages);
876855 if (!huge)
877856 mem_cgroup_cancel_charge(page, memcg, false);
878857 put_page(page);
879− return error;
858+ return xas_error(&xas);
880859 }
881860
882861 /**
@@ -1341,86 +1320,76 @@ int __lock_page_or_retry(struct page *page, struct mm_struct *mm, · 458 unmodified lines
13411320 }
13421321
13431322 /**
1344− * page_cache_next_hole - find the next hole (not-present entry)
1345− * @mapping: mapping
1346− * @index: index
1347− * @max_scan: maximum range to search
1348− *
1349− * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
1350− * lowest indexed hole.
1351− *
1352− * Returns: the index of the hole if found, otherwise returns an index
1353− * outside of the set specified (in which case 'return - index >=
1354− * max_scan' will be true). In rare cases of index wrap-around, 0 will
1355− * be returned.
1356− *
1357− * page_cache_next_hole may be called under rcu_read_lock. However,
1358− * like radix_tree_gang_lookup, this will not atomically search a
1359− * snapshot of the tree at a single point in time. For example, if a
1360− * hole is created at index 5, then subsequently a hole is created at
1361− * index 10, page_cache_next_hole covering both indexes may return 10
1362− * if called under rcu_read_lock.
1323+ * page_cache_next_miss() - Find the next gap in the page cache.
1324+ * @mapping: Mapping.
1325+ * @index: Index.
1326+ * @max_scan: Maximum range to search.
1327+ *
1328+ * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1329+ * gap with the lowest index.
1330+ *
1331+ * This function may be called under the rcu_read_lock. However, this will
1332+ * not atomically search a snapshot of the cache at a single point in time.
1333+ * For example, if a gap is created at index 5, then subsequently a gap is
1334+ * created at index 10, page_cache_next_miss covering both indices may
1335+ * return 10 if called under the rcu_read_lock.
1336+ *
1337+ * Return: The index of the gap if found, otherwise an index outside the
1338+ * range specified (in which case 'return - index >= max_scan' will be true).
1339+ * In the rare case of index wrap-around, 0 will be returned.
13631340 */
1364−pgoff_t page_cache_next_hole(struct address_space *mapping,
1341+pgoff_t page_cache_next_miss(struct address_space *mapping,
13651342 pgoff_t index, unsigned long max_scan)
13661343 {
1367− unsigned long i;
1368−
1369− for (i = 0; i < max_scan; i++) {
1370− struct page *page;
1344+ XA_STATE(xas, &mapping->i_pages, index);
13711345
1372− page = radix_tree_lookup(&mapping->i_pages, index);
1373− if (!page || radix_tree_exceptional_entry(page))
1346+ while (max_scan--) {
1347+ void *entry = xas_next(&xas);
1348+ if (!entry || xa_is_value(entry))
13741349 break;
1375− index++;
1376− if (index == 0)
1350+ if (xas.xa_index == 0)
13771351 break;
13781352 }
13791353
1380− return index;
1354+ return xas.xa_index;
13811355 }
1382−EXPORT_SYMBOL(page_cache_next_hole);
1356+EXPORT_SYMBOL(page_cache_next_miss);
13831357
13841358 /**
1385− * page_cache_prev_hole - find the prev hole (not-present entry)
1386− * @mapping: mapping
1387− * @index: index
1388− * @max_scan: maximum range to search
1389− *
1390− * Search backwards in the range [max(index-max_scan+1, 0), index] for
1391− * the first hole.
1392− *
1393− * Returns: the index of the hole if found, otherwise returns an index
1394− * outside of the set specified (in which case 'index - return >=
1395− * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
1396− * will be returned.
1397− *
1398− * page_cache_prev_hole may be called under rcu_read_lock. However,
1399− * like radix_tree_gang_lookup, this will not atomically search a
1400− * snapshot of the tree at a single point in time. For example, if a
1401− * hole is created at index 10, then subsequently a hole is created at
1402− * index 5, page_cache_prev_hole covering both indexes may return 5 if
1403− * called under rcu_read_lock.
1359+ * page_cache_prev_miss() - Find the next gap in the page cache.
1360+ * @mapping: Mapping.
1361+ * @index: Index.
1362+ * @max_scan: Maximum range to search.
1363+ *
1364+ * Search the range [max(index - max_scan + 1, 0), index] for the
1365+ * gap with the highest index.
1366+ *
1367+ * This function may be called under the rcu_read_lock. However, this will
1368+ * not atomically search a snapshot of the cache at a single point in time.
1369+ * For example, if a gap is created at index 10, then subsequently a gap is
1370+ * created at index 5, page_cache_prev_miss() covering both indices may
1371+ * return 5 if called under the rcu_read_lock.
1372+ *
1373+ * Return: The index of the gap if found, otherwise an index outside the
1374+ * range specified (in which case 'index - return >= max_scan' will be true).
1375+ * In the rare case of wrap-around, ULONG_MAX will be returned.
14041376 */
1405−pgoff_t page_cache_prev_hole(struct address_space *mapping,
1377+pgoff_t page_cache_prev_miss(struct address_space *mapping,
14061378 pgoff_t index, unsigned long max_scan)
14071379 {
1408− unsigned long i;
1409−
1410− for (i = 0; i < max_scan; i++) {
1411− struct page *page;
1380+ XA_STATE(xas, &mapping->i_pages, index);
14121381
1413− page = radix_tree_lookup(&mapping->i_pages, index);
1414− if (!page || radix_tree_exceptional_entry(page))
1382+ while (max_scan--) {
1383+ void *entry = xas_prev(&xas);
1384+ if (!entry || xa_is_value(entry))
14151385 break;
1416− index--;
1417− if (index == ULONG_MAX)
1386+ if (xas.xa_index == ULONG_MAX)
14181387 break;
14191388 }
14201389
1421− return index;
1390+ return xas.xa_index;
14221391 }
1423−EXPORT_SYMBOL(page_cache_prev_hole);
1392+EXPORT_SYMBOL(page_cache_prev_miss);
14241393
14251394 /**
14261395 * find_get_entry - find and get a page cache entry
@@ -1437,47 +1406,40 @@ EXPORT_SYMBOL(page_cache_prev_hole); · 10 unmodified lines
14371406 */
14381407 struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
14391408 {
1440− void **pagep;
1409+ XA_STATE(xas, &mapping->i_pages, offset);
14411410 struct page *head, *page;
14421411
14431412 rcu_read_lock();
14441413 repeat:
1445− page = NULL;
1446− pagep = radix_tree_lookup_slot(&mapping->i_pages, offset);
1447− if (pagep) {
1448− page = radix_tree_deref_slot(pagep);
1449− if (unlikely(!page))
1450− goto out;
1451− if (radix_tree_exception(page)) {
1452− if (radix_tree_deref_retry(page))
1453− goto repeat;
1454− /*
1455− * A shadow entry of a recently evicted page,
1456− * or a swap entry from shmem/tmpfs. Return
1457− * it without attempting to raise page count.
1458− */
1459− goto out;
1460− }
1414+ xas_reset(&xas);
1415+ page = xas_load(&xas);
1416+ if (xas_retry(&xas, page))
1417+ goto repeat;
1418+ /*
1419+ * A shadow entry of a recently evicted page, or a swap entry from
1420+ * shmem/tmpfs. Return it without attempting to raise page count.
1421+ */
1422+ if (!page || xa_is_value(page))
1423+ goto out;
14611424
1462− head = compound_head(page);
1463− if (!page_cache_get_speculative(head))
1464− goto repeat;
1425+ head = compound_head(page);
1426+ if (!page_cache_get_speculative(head))
1427+ goto repeat;
14651428
1466− /* The page was split under us? */
1467− if (compound_head(page) != head) {
1468− put_page(head);
1469− goto repeat;
1470− }
1429+ /* The page was split under us? */
1430+ if (compound_head(page) != head) {
1431+ put_page(head);
1432+ goto repeat;
1433+ }
14711434
1472− /*
1473− * Has the page moved?
1474− * This is part of the lockless pagecache protocol. See
1475− * include/linux/pagemap.h for details.
1476− */
1477− if (unlikely(page != *pagep)) {
1478− put_page(head);
1479− goto repeat;
1480− }
1435+ /*
1436+ * Has the page moved?
1437+ * This is part of the lockless pagecache protocol. See
1438+ * include/linux/pagemap.h for details.
1439+ */
1440+ if (unlikely(page != xas_reload(&xas))) {
1441+ put_page(head);
1442+ goto repeat;
14811443 }
14821444 out:
14831445 rcu_read_unlock();
@@ -1508,7 +1470,7 @@ struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset) · 24 unmodified lines
15081470
15091471 repeat:
15101472 page = find_get_entry(mapping, offset);
1511− if (page && !radix_tree_exception(page)) {
1473+ if (page && !xa_is_value(page)) {
15121474 lock_page(page);
15131475 /* Has the page been truncated? */
15141476 if (unlikely(page_mapping(page) != mapping)) {
@@ -1554,7 +1516,7 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, · 39 unmodified lines
15541516
15551517 repeat:
15561518 page = find_get_entry(mapping, offset);
1557− if (radix_tree_exceptional_entry(page))
1519+ if (xa_is_value(page))
15581520 page = NULL;
15591521 if (!page)
15601522 goto no_page;
@@ -1640,53 +1602,48 @@ unsigned find_get_entries(struct address_space *mapping, · 79 unmodified lines
16401602 pgoff_t start, unsigned int nr_entries,
16411603 struct page **entries, pgoff_t *indices)
16421604 {
1643− void **slot;
1605+ XA_STATE(xas, &mapping->i_pages, start);
1606+ struct page *page;
16441607 unsigned int ret = 0;
1645− struct radix_tree_iter iter;
16461608
16471609 if (!nr_entries)
16481610 return 0;
16491611
16501612 rcu_read_lock();
1651− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1652− struct page *head, *page;
1653−repeat:
1654− page = radix_tree_deref_slot(slot);
1655− if (unlikely(!page))
1613+ xas_for_each(&xas, page, ULONG_MAX) {
1614+ struct page *head;
1615+ if (xas_retry(&xas, page))
16561616 continue;
1657− if (radix_tree_exception(page)) {
1658− if (radix_tree_deref_retry(page)) {
1659− slot = radix_tree_iter_retry(&iter);
1660− continue;
1661− }
1662− /*
1663− * A shadow entry of a recently evicted page, a swap
1664− * entry from shmem/tmpfs or a DAX entry. Return it
1665− * without attempting to raise page count.
1666− */
1617+ /*
1618+ * A shadow entry of a recently evicted page, a swap
1619+ * entry from shmem/tmpfs or a DAX entry. Return it
1620+ * without attempting to raise page count.
1621+ */
1622+ if (xa_is_value(page))
16671623 goto export;
1668− }
16691624
16701625 head = compound_head(page);
16711626 if (!page_cache_get_speculative(head))
1672− goto repeat;
1627+ goto retry;
16731628
16741629 /* The page was split under us? */
1675− if (compound_head(page) != head) {
1676− put_page(head);
1677− goto repeat;
1678− }
1630+ if (compound_head(page) != head)
1631+ goto put_page;
16791632
16801633 /* Has the page moved? */
1681− if (unlikely(page != *slot)) {
1682− put_page(head);
1683− goto repeat;
1684− }
1634+ if (unlikely(page != xas_reload(&xas)))
1635+ goto put_page;
1636+
16851637 export:
1686− indices[ret] = iter.index;
1638+ indices[ret] = xas.xa_index;
16871639 entries[ret] = page;
16881640 if (++ret == nr_entries)
16891641 break;
1642+ continue;
1643+put_page:
1644+ put_page(head);
1645+retry:
1646+ xas_reset(&xas);
16901647 }
16911648 rcu_read_unlock();
16921649 return ret;
@@ -1717,64 +1674,50 @@ unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, · 24 unmodified lines
17171674 pgoff_t end, unsigned int nr_pages,
17181675 struct page **pages)
17191676 {
1720− struct radix_tree_iter iter;
1721− void **slot;
1677+ XA_STATE(xas, &mapping->i_pages, *start);
1678+ struct page *page;
17221679 unsigned ret = 0;
17231680
17241681 if (unlikely(!nr_pages))
17251682 return 0;
17261683
17271684 rcu_read_lock();
1728− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, *start) {
1729− struct page *head, *page;
1730−
1731− if (iter.index > end)
1732− break;
1733−repeat:
1734− page = radix_tree_deref_slot(slot);
1735− if (unlikely(!page))
1685+ xas_for_each(&xas, page, end) {
1686+ struct page *head;
1687+ if (xas_retry(&xas, page))
17361688 continue;
1737−
1738− if (radix_tree_exception(page)) {
1739− if (radix_tree_deref_retry(page)) {
1740− slot = radix_tree_iter_retry(&iter);
1741− continue;
1742− }
1743− /*
1744− * A shadow entry of a recently evicted page,
1745− * or a swap entry from shmem/tmpfs. Skip
1746− * over it.
1747− */
1689+ /* Skip over shadow, swap and DAX entries */
1690+ if (xa_is_value(page))
17481691 continue;
1749− }
17501692
17511693 head = compound_head(page);
17521694 if (!page_cache_get_speculative(head))
1753− goto repeat;
1695+ goto retry;
17541696
17551697 /* The page was split under us? */
1756− if (compound_head(page) != head) {
1757− put_page(head);
1758− goto repeat;
1759− }
1698+ if (compound_head(page) != head)
1699+ goto put_page;
17601700
17611701 /* Has the page moved? */
1762− if (unlikely(page != *slot)) {
1763− put_page(head);
1764− goto repeat;
1765− }
1702+ if (unlikely(page != xas_reload(&xas)))
1703+ goto put_page;
17661704
17671705 pages[ret] = page;
17681706 if (++ret == nr_pages) {
1769− *start = pages[ret - 1]->index + 1;
1707+ *start = page->index + 1;
17701708 goto out;
17711709 }
1710+ continue;
1711+put_page:
1712+ put_page(head);
1713+retry:
1714+ xas_reset(&xas);
17721715 }
17731716
17741717 /*
17751718 * We come here when there is no page beyond @end. We take care to not
17761719 * overflow the index @start as it confuses some of the callers. This
1777− * breaks the iteration when there is page at index -1 but that is
1720+ * breaks the iteration when there is a page at index -1 but that is
17781721 * already broken anyway.
17791722 */
17801723 if (end == (pgoff_t)-1)
@@ -1802,57 +1745,43 @@ out: · 21 unmodified lines
18021745 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
18031746 unsigned int nr_pages, struct page **pages)
18041747 {
1805− struct radix_tree_iter iter;
1806− void **slot;
1748+ XA_STATE(xas, &mapping->i_pages, index);
1749+ struct page *page;
18071750 unsigned int ret = 0;
18081751
18091752 if (unlikely(!nr_pages))
18101753 return 0;
18111754
18121755 rcu_read_lock();
1813− radix_tree_for_each_contig(slot, &mapping->i_pages, &iter, index) {
1814− struct page *head, *page;
1815−repeat:
1816− page = radix_tree_deref_slot(slot);
1817− /* The hole, there no reason to continue */
1818− if (unlikely(!page))
1819− break;
1820−
1821− if (radix_tree_exception(page)) {
1822− if (radix_tree_deref_retry(page)) {
1823− slot = radix_tree_iter_retry(&iter);
1824− continue;
1825− }
1826− /*
1827− * A shadow entry of a recently evicted page,
1828− * or a swap entry from shmem/tmpfs. Stop
1829− * looking for contiguous pages.
1830− */
1756+ for (page = xas_load(&xas); page; page = xas_next(&xas)) {
1757+ struct page *head;
1758+ if (xas_retry(&xas, page))
1759+ continue;
1760+ /*
1761+ * If the entry has been swapped out, we can stop looking.
1762+ * No current caller is looking for DAX entries.
1763+ */
1764+ if (xa_is_value(page))
18311765 break;
1832− }
18331766
18341767 head = compound_head(page);
18351768 if (!page_cache_get_speculative(head))
1836− goto repeat;
1769+ goto retry;
18371770
18381771 /* The page was split under us? */
1839− if (compound_head(page) != head) {
1840− put_page(head);
1841− goto repeat;
1842− }
1772+ if (compound_head(page) != head)
1773+ goto put_page;
18431774
18441775 /* Has the page moved? */
1845− if (unlikely(page != *slot)) {
1846− put_page(head);
1847− goto repeat;
1848− }
1776+ if (unlikely(page != xas_reload(&xas)))
1777+ goto put_page;
18491778
18501779 /*
18511780 * must check mapping and index after taking the ref.
18521781 * otherwise we can get both false positives and false
18531782 * negatives, which is just confusing to the caller.
18541783 */
1855− if (page->mapping == NULL || page_to_pgoff(page) != iter.index) {
1784+ if (!page->mapping || page_to_pgoff(page) != xas.xa_index) {
18561785 put_page(page);
18571786 break;
18581787 }
@@ -1860,6 +1789,11 @@ repeat: · 1 unmodified line
18601789 pages[ret] = page;
18611790 if (++ret == nr_pages)
18621791 break;
1792+ continue;
1793+put_page:
1794+ put_page(head);
1795+retry:
1796+ xas_reset(&xas);
18631797 }
18641798 rcu_read_unlock();
18651799 return ret;
@@ -1879,74 +1813,58 @@ EXPORT_SYMBOL(find_get_pages_contig); · 13 unmodified lines
18791813 * @tag. We update @index to index the next page for the traversal.
18801814 */
18811815 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
1882− pgoff_t end, int tag, unsigned int nr_pages,
1816+ pgoff_t end, xa_mark_t tag, unsigned int nr_pages,
18831817 struct page **pages)
18841818 {
1885− struct radix_tree_iter iter;
1886− void **slot;
1819+ XA_STATE(xas, &mapping->i_pages, *index);
1820+ struct page *page;
18871821 unsigned ret = 0;
18881822
18891823 if (unlikely(!nr_pages))
18901824 return 0;
18911825
18921826 rcu_read_lock();
1893− radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, *index, tag) {
1894− struct page *head, *page;
1895−
1896− if (iter.index > end)
1897− break;
1898−repeat:
1899− page = radix_tree_deref_slot(slot);
1900− if (unlikely(!page))
1827+ xas_for_each_marked(&xas, page, end, tag) {
1828+ struct page *head;
1829+ if (xas_retry(&xas, page))
19011830 continue;
1902−
1903− if (radix_tree_exception(page)) {
1904− if (radix_tree_deref_retry(page)) {
1905− slot = radix_tree_iter_retry(&iter);
1906− continue;
1907− }
1908− /*
1909− * A shadow entry of a recently evicted page.
1910− *
1911− * Those entries should never be tagged, but
1912− * this tree walk is lockless and the tags are
1913− * looked up in bulk, one radix tree node at a
1914− * time, so there is a sizable window for page
1915− * reclaim to evict a page we saw tagged.
1916− *
1917− * Skip over it.
1918− */
1831+ /*
1832+ * Shadow entries should never be tagged, but this iteration
1833+ * is lockless so there is a window for page reclaim to evict
1834+ * a page we saw tagged. Skip over it.
1835+ */
1836+ if (xa_is_value(page))
19191837 continue;
1920− }
19211838
19221839 head = compound_head(page);
19231840 if (!page_cache_get_speculative(head))
1924− goto repeat;
1841+ goto retry;
19251842
19261843 /* The page was split under us? */
1927− if (compound_head(page) != head) {
1928− put_page(head);
1929− goto repeat;
1930− }
1844+ if (compound_head(page) != head)
1845+ goto put_page;
19311846
19321847 /* Has the page moved? */
1933− if (unlikely(page != *slot)) {
1934− put_page(head);
1935− goto repeat;
1936− }
1848+ if (unlikely(page != xas_reload(&xas)))
1849+ goto put_page;
19371850
19381851 pages[ret] = page;
19391852 if (++ret == nr_pages) {
1940− *index = pages[ret - 1]->index + 1;
1853+ *index = page->index + 1;
19411854 goto out;
19421855 }
1856+ continue;
1857+put_page:
1858+ put_page(head);
1859+retry:
1860+ xas_reset(&xas);
19431861 }
19441862
19451863 /*
1946− * We come here when we got at @end. We take care to not overflow the
1864+ * We come here when we got to @end. We take care to not overflow the
19471865 * index @index as it confuses some of the callers. This breaks the
1948− * iteration when there is page at index -1 but that is already broken
1949− * anyway.
1866+ * iteration when there is a page at index -1 but that is already
1867+ * broken anyway.
19501868 */
19511869 if (end == (pgoff_t)-1)
19521870 *index = (pgoff_t)-1;
@@ -1972,57 +1890,51 @@ EXPORT_SYMBOL(find_get_pages_range_tag); · 19 unmodified lines
19721890 * @tag.
19731891 */
19741892 unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
1975− int tag, unsigned int nr_entries,
1893+ xa_mark_t tag, unsigned int nr_entries,
19761894 struct page **entries, pgoff_t *indices)
19771895 {
1978− void **slot;
1896+ XA_STATE(xas, &mapping->i_pages, start);
1897+ struct page *page;
19791898 unsigned int ret = 0;
1980− struct radix_tree_iter iter;
19811899
19821900 if (!nr_entries)
19831901 return 0;
19841902
19851903 rcu_read_lock();
1986− radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, tag) {
1987− struct page *head, *page;
1988−repeat:
1989− page = radix_tree_deref_slot(slot);
1990− if (unlikely(!page))
1904+ xas_for_each_marked(&xas, page, ULONG_MAX, tag) {
1905+ struct page *head;
1906+ if (xas_retry(&xas, page))
19911907 continue;
1992− if (radix_tree_exception(page)) {
1993− if (radix_tree_deref_retry(page)) {
1994− slot = radix_tree_iter_retry(&iter);
1995− continue;
1996− }
1997−
1998− /*
1999− * A shadow entry of a recently evicted page, a swap
2000− * entry from shmem/tmpfs or a DAX entry. Return it
2001− * without attempting to raise page count.
2002− */
1908+ /*
1909+ * A shadow entry of a recently evicted page, a swap
1910+ * entry from shmem/tmpfs or a DAX entry. Return it
1911+ * without attempting to raise page count.
1912+ */
1913+ if (xa_is_value(page))
20031914 goto export;
2004− }
20051915
20061916 head = compound_head(page);
20071917 if (!page_cache_get_speculative(head))
2008− goto repeat;
1918+ goto retry;
20091919
20101920 /* The page was split under us? */
2011− if (compound_head(page) != head) {
2012− put_page(head);
2013− goto repeat;
2014− }
1921+ if (compound_head(page) != head)
1922+ goto put_page;
20151923
20161924 /* Has the page moved? */
2017− if (unlikely(page != *slot)) {
2018− put_page(head);
2019− goto repeat;
2020− }
1925+ if (unlikely(page != xas_reload(&xas)))
1926+ goto put_page;
1927+
20211928 export:
2022− indices[ret] = iter.index;
1929+ indices[ret] = xas.xa_index;
20231930 entries[ret] = page;
20241931 if (++ret == nr_entries)
20251932 break;
1933+ continue;
1934+put_page:
1935+ put_page(head);
1936+retry:
1937+ xas_reset(&xas);
20261938 }
20271939 rcu_read_unlock();
20281940 return ret;
@@ -2626,45 +2538,31 @@ EXPORT_SYMBOL(filemap_fault); · 597 unmodified lines
26262538 void filemap_map_pages(struct vm_fault *vmf,
26272539 pgoff_t start_pgoff, pgoff_t end_pgoff)
26282540 {
2629− struct radix_tree_iter iter;
2630− void **slot;
26312541 struct file *file = vmf->vma->vm_file;
26322542 struct address_space *mapping = file->f_mapping;
26332543 pgoff_t last_pgoff = start_pgoff;
26342544 unsigned long max_idx;
2545+ XA_STATE(xas, &mapping->i_pages, start_pgoff);
26352546 struct page *head, *page;
26362547
26372548 rcu_read_lock();
2638− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start_pgoff) {
2639− if (iter.index > end_pgoff)
2640− break;
2641−repeat:
2642− page = radix_tree_deref_slot(slot);
2643− if (unlikely(!page))
2644− goto next;
2645− if (radix_tree_exception(page)) {
2646− if (radix_tree_deref_retry(page)) {
2647− slot = radix_tree_iter_retry(&iter);
2648− continue;
2649− }
2549+ xas_for_each(&xas, page, end_pgoff) {
2550+ if (xas_retry(&xas, page))
2551+ continue;
2552+ if (xa_is_value(page))
26502553 goto next;
2651− }
26522554
26532555 head = compound_head(page);
26542556 if (!page_cache_get_speculative(head))
2655− goto repeat;
2557+ goto next;
26562558
26572559 /* The page was split under us? */
2658− if (compound_head(page) != head) {
2659− put_page(head);
2660− goto repeat;
2661− }
2560+ if (compound_head(page) != head)
2561+ goto skip;
26622562
26632563 /* Has the page moved? */
2664− if (unlikely(page != *slot)) {
2665− put_page(head);
2666− goto repeat;
2667− }
2564+ if (unlikely(page != xas_reload(&xas)))
2565+ goto skip;
26682566
26692567 if (!PageUptodate(page) ||
26702568 PageReadahead(page) ||
@@ -2683,10 +2581,10 @@ repeat: · 12 unmodified lines
26832581 if (file->f_ra.mmap_miss > 0)
26842582 file->f_ra.mmap_miss--;
26852583
2686− vmf->address += (iter.index - last_pgoff) << PAGE_SHIFT;
2584+ vmf->address += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
26872585 if (vmf->pte)
2688− vmf->pte += iter.index - last_pgoff;
2689− last_pgoff = iter.index;
2586+ vmf->pte += xas.xa_index - last_pgoff;
2587+ last_pgoff = xas.xa_index;
26902588 if (alloc_set_pte(vmf, NULL, page))
26912589 goto unlock;
26922590 unlock_page(page);
@@ -2699,8 +2597,6 @@ next: · 6 unmodified lines
26992597 /* Huge page is mapped? No need to proceed. */
27002598 if (pmd_trans_huge(*vmf->pmd))
27012599 break;
2702− if (iter.index == end_pgoff)
2703− break;
27042600 }
27052601 rcu_read_unlock();
27062602 }
@@ -2810,7 +2706,7 @@ repeat: · 103 unmodified lines
28102706 put_page(page);
28112707 if (err == -EEXIST)
28122708 goto repeat;
2813− /* Presumably ENOMEM for radix tree node */
2709+ /* Presumably ENOMEM for xarray node */
28142710 return ERR_PTR(err);
28152711 }
28162712
mm/huge_memory.c+7−10
@@ -2450,13 +2450,13 @@ static void __split_huge_page(struct page *page, struct list_head *list, · 2449 unmodified lines
24502450 ClearPageCompound(head);
24512451 /* See comment in __split_huge_page_tail() */
24522452 if (PageAnon(head)) {
2453− /* Additional pin to radix tree of swap cache */
2453+ /* Additional pin to swap cache */
24542454 if (PageSwapCache(head))
24552455 page_ref_add(head, 2);
24562456 else
24572457 page_ref_inc(head);
24582458 } else {
2459− /* Additional pin to radix tree */
2459+ /* Additional pin to page cache */
24602460 page_ref_add(head, 2);
24612461 xa_unlock(&head->mapping->i_pages);
24622462 }
@@ -2568,7 +2568,7 @@ bool can_split_huge_page(struct page *page, int *pextra_pins) · 105 unmodified lines
25682568 {
25692569 int extra_pins;
25702570
2571− /* Additional pins from radix tree */
2571+ /* Additional pins from page cache */
25722572 if (PageAnon(page))
25732573 extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0;
25742574 else
@@ -2664,17 +2664,14 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) · 89 unmodified lines
26642664 spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags);
26652665
26662666 if (mapping) {
2667− void **pslot;
2667+ XA_STATE(xas, &mapping->i_pages, page_index(head));
26682668
2669− xa_lock(&mapping->i_pages);
2670− pslot = radix_tree_lookup_slot(&mapping->i_pages,
2671− page_index(head));
26722669 /*
2673− * Check if the head page is present in radix tree.
2670+ * Check if the head page is present in page cache.
26742671 * We assume all tail are present too, if head is there.
26752672 */
2676− if (radix_tree_deref_slot_protected(pslot,
2677− &mapping->i_pages.xa_lock) != head)
2673+ xa_lock(&mapping->i_pages);
2674+ if (xas_load(&xas) != head)
26782675 goto fail;
26792676 }
26802677
mm/khugepaged.c+72−106
@@ -1288,17 +1288,17 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) · 1287 unmodified lines
12881288 *
12891289 * Basic scheme is simple, details are more complex:
12901290 * - allocate and freeze a new huge page;
1291− * - scan over radix tree replacing old pages the new one
1291+ * - scan page cache replacing old pages with the new one
12921292 * + swap in pages if necessary;
12931293 * + fill in gaps;
1294− * + keep old pages around in case if rollback is required;
1295− * - if replacing succeed:
1294+ * + keep old pages around in case rollback is required;
1295+ * - if replacing succeeds:
12961296 * + copy data over;
12971297 * + free old pages;
12981298 * + unfreeze huge page;
12991299 * - if replacing failed;
13001300 * + put all pages back and unfreeze them;
1301− * + restore gaps in the radix-tree;
1301+ * + restore gaps in the page cache;
13021302 * + free huge page;
13031303 */
13041304 static void collapse_shmem(struct mm_struct *mm,
@@ -1306,12 +1306,11 @@ static void collapse_shmem(struct mm_struct *mm, · 1 unmodified line
13061306 struct page **hpage, int node)
13071307 {
13081308 gfp_t gfp;
1309− struct page *page, *new_page, *tmp;
1309+ struct page *new_page;
13101310 struct mem_cgroup *memcg;
13111311 pgoff_t index, end = start + HPAGE_PMD_NR;
13121312 LIST_HEAD(pagelist);
1313− struct radix_tree_iter iter;
1314− void **slot;
1313+ XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
13151314 int nr_none = 0, result = SCAN_SUCCEED;
13161315
13171316 VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
@@ -1336,48 +1335,49 @@ static void collapse_shmem(struct mm_struct *mm, · 18 unmodified lines
13361335 __SetPageLocked(new_page);
13371336 BUG_ON(!page_ref_freeze(new_page, 1));
13381337
1339−
13401338 /*
1341− * At this point the new_page is 'frozen' (page_count() is zero), locked
1342− * and not up-to-date. It's safe to insert it into radix tree, because
1343− * nobody would be able to map it or use it in other way until we
1344− * unfreeze it.
1339+ * At this point the new_page is 'frozen' (page_count() is zero),
1340+ * locked and not up-to-date. It's safe to insert it into the page
1341+ * cache, because nobody would be able to map it or use it in other
1342+ * way until we unfreeze it.
13451343 */
13461344
1347− index = start;
1348− xa_lock_irq(&mapping->i_pages);
1349− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1350− int n = min(iter.index, end) - index;
1351−
1352− /*
1353− * Handle holes in the radix tree: charge it from shmem and
1354− * insert relevant subpage of new_page into the radix-tree.
1355− */
1356− if (n && !shmem_charge(mapping->host, n)) {
1357− result = SCAN_FAIL;
1345+ /* This will be less messy when we use multi-index entries */
1346+ do {
1347+ xas_lock_irq(&xas);
1348+ xas_create_range(&xas);
1349+ if (!xas_error(&xas))
13581350 break;
1359− }
1360− nr_none += n;
1361− for (; index < min(iter.index, end); index++) {
1362− radix_tree_insert(&mapping->i_pages, index,
1363− new_page + (index % HPAGE_PMD_NR));
1364− }
1351+ xas_unlock_irq(&xas);
1352+ if (!xas_nomem(&xas, GFP_KERNEL))
1353+ goto out;
1354+ } while (1);
13651355
1366− /* We are done. */
1367− if (index >= end)
1368− break;
1356+ xas_set(&xas, start);
1357+ for (index = start; index < end; index++) {
1358+ struct page *page = xas_next(&xas);
1359+
1360+ VM_BUG_ON(index != xas.xa_index);
1361+ if (!page) {
1362+ if (!shmem_charge(mapping->host, 1)) {
1363+ result = SCAN_FAIL;
1364+ break;
1365+ }
1366+ xas_store(&xas, new_page + (index % HPAGE_PMD_NR));
1367+ nr_none++;
1368+ continue;
1369+ }
13691370
1370− page = radix_tree_deref_slot_protected(slot,
1371− &mapping->i_pages.xa_lock);
1372− if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) {
1373− xa_unlock_irq(&mapping->i_pages);
1371+ if (xa_is_value(page) || !PageUptodate(page)) {
1372+ xas_unlock_irq(&xas);
13741373 /* swap in or instantiate fallocated page */
13751374 if (shmem_getpage(mapping->host, index, &page,
13761375 SGP_NOHUGE)) {
13771376 result = SCAN_FAIL;
1378− goto tree_unlocked;
1377+ goto xa_unlocked;
13791378 }
1380− xa_lock_irq(&mapping->i_pages);
1379+ xas_lock_irq(&xas);
1380+ xas_set(&xas, index);
13811381 } else if (trylock_page(page)) {
13821382 get_page(page);
13831383 } else {
@@ -1397,7 +1397,7 @@ static void collapse_shmem(struct mm_struct *mm, · 13 unmodified lines
13971397 result = SCAN_TRUNCATED;
13981398 goto out_unlock;
13991399 }
1400− xa_unlock_irq(&mapping->i_pages);
1400+ xas_unlock_irq(&xas);
14011401
14021402 if (isolate_lru_page(page)) {
14031403 result = SCAN_DEL_PAGE_LRU;
@@ -1407,17 +1407,16 @@ static void collapse_shmem(struct mm_struct *mm, · 3 unmodified lines
14071407 if (page_mapped(page))
14081408 unmap_mapping_pages(mapping, index, 1, false);
14091409
1410− xa_lock_irq(&mapping->i_pages);
1410+ xas_lock_irq(&xas);
1411+ xas_set(&xas, index);
14111412
1412− slot = radix_tree_lookup_slot(&mapping->i_pages, index);
1413− VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot,
1414− &mapping->i_pages.xa_lock), page);
1413+ VM_BUG_ON_PAGE(page != xas_load(&xas), page);
14151414 VM_BUG_ON_PAGE(page_mapped(page), page);
14161415
14171416 /*
14181417 * The page is expected to have page_count() == 3:
14191418 * - we hold a pin on it;
1420− * - one reference from radix tree;
1419+ * - one reference from page cache;
14211420 * - one from isolate_lru_page;
14221421 */
14231422 if (!page_ref_freeze(page, 3)) {
@@ -1432,56 +1431,30 @@ static void collapse_shmem(struct mm_struct *mm, · 8 unmodified lines
14321431 list_add_tail(&page->lru, &pagelist);
14331432
14341433 /* Finally, replace with the new page. */
1435− radix_tree_replace_slot(&mapping->i_pages, slot,
1436− new_page + (index % HPAGE_PMD_NR));
1437−
1438− slot = radix_tree_iter_resume(slot, &iter);
1439− index++;
1434+ xas_store(&xas, new_page + (index % HPAGE_PMD_NR));
14401435 continue;
14411436 out_lru:
1442− xa_unlock_irq(&mapping->i_pages);
1437+ xas_unlock_irq(&xas);
14431438 putback_lru_page(page);
14441439 out_isolate_failed:
14451440 unlock_page(page);
14461441 put_page(page);
1447− goto tree_unlocked;
1442+ goto xa_unlocked;
14481443 out_unlock:
14491444 unlock_page(page);
14501445 put_page(page);
14511446 break;
14521447 }
1448+ xas_unlock_irq(&xas);
14531449
1454− /*
1455− * Handle hole in radix tree at the end of the range.
1456− * This code only triggers if there's nothing in radix tree
1457− * beyond 'end'.
1458− */
1459− if (result == SCAN_SUCCEED && index < end) {
1460− int n = end - index;
1461−
1462− if (!shmem_charge(mapping->host, n)) {
1463− result = SCAN_FAIL;
1464− goto tree_locked;
1465− }
1466−
1467− for (; index < end; index++) {
1468− radix_tree_insert(&mapping->i_pages, index,
1469− new_page + (index % HPAGE_PMD_NR));
1470− }
1471− nr_none += n;
1472− }
1473−
1474−tree_locked:
1475− xa_unlock_irq(&mapping->i_pages);
1476−tree_unlocked:
1477−
1450+xa_unlocked:
14781451 if (result == SCAN_SUCCEED) {
1479− unsigned long flags;
1452+ struct page *page, *tmp;
14801453 struct zone *zone = page_zone(new_page);
14811454
14821455 /*
1483− * Replacing old pages with new one has succeed, now we need to
1484− * copy the content and free old pages.
1456+ * Replacing old pages with new one has succeeded, now we
1457+ * need to copy the content and free the old pages.
14851458 */
14861459 list_for_each_entry_safe(page, tmp, &pagelist, lru) {
14871460 copy_highpage(new_page + (page->index % HPAGE_PMD_NR),
@@ -1495,16 +1468,16 @@ tree_unlocked: · 7 unmodified lines
14951468 put_page(page);
14961469 }
14971470
1498− local_irq_save(flags);
1471+ local_irq_disable();
14991472 __inc_node_page_state(new_page, NR_SHMEM_THPS);
15001473 if (nr_none) {
15011474 __mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);
15021475 __mod_node_page_state(zone->zone_pgdat, NR_SHMEM, nr_none);
15031476 }
1504− local_irq_restore(flags);
1477+ local_irq_enable();
15051478
15061479 /*
1507− * Remove pte page tables, so we can re-faulti
1480+ * Remove pte page tables, so we can re-fault
15081481 * the page as huge.
15091482 */
15101483 retract_page_tables(mapping, start);
@@ -1521,37 +1494,37 @@ tree_unlocked: · 10 unmodified lines
15211494
15221495 khugepaged_pages_collapsed++;
15231496 } else {
1524− /* Something went wrong: rollback changes to the radix-tree */
1497+ struct page *page;
1498+ /* Something went wrong: roll back page cache changes */
15251499 shmem_uncharge(mapping->host, nr_none);
1526− xa_lock_irq(&mapping->i_pages);
1527− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1528− if (iter.index >= end)
1529− break;
1500+ xas_lock_irq(&xas);
1501+ xas_set(&xas, start);
1502+ xas_for_each(&xas, page, end - 1) {
15301503 page = list_first_entry_or_null(&pagelist,
15311504 struct page, lru);
1532− if (!page || iter.index < page->index) {
1505+ if (!page || xas.xa_index < page->index) {
15331506 if (!nr_none)
15341507 break;
15351508 nr_none--;
15361509 /* Put holes back where they were */
1537− radix_tree_delete(&mapping->i_pages, iter.index);
1510+ xas_store(&xas, NULL);
15381511 continue;
15391512 }
15401513
1541− VM_BUG_ON_PAGE(page->index != iter.index, page);
1514+ VM_BUG_ON_PAGE(page->index != xas.xa_index, page);
15421515
15431516 /* Unfreeze the page. */
15441517 list_del(&page->lru);
15451518 page_ref_unfreeze(page, 2);
1546− radix_tree_replace_slot(&mapping->i_pages, slot, page);
1547− slot = radix_tree_iter_resume(slot, &iter);
1548− xa_unlock_irq(&mapping->i_pages);
1519+ xas_store(&xas, page);
1520+ xas_pause(&xas);
1521+ xas_unlock_irq(&xas);
15491522 putback_lru_page(page);
15501523 unlock_page(page);
1551− xa_lock_irq(&mapping->i_pages);
1524+ xas_lock_irq(&xas);
15521525 }
15531526 VM_BUG_ON(nr_none);
1554− xa_unlock_irq(&mapping->i_pages);
1527+ xas_unlock_irq(&xas);
15551528
15561529 /* Unfreeze new_page, caller would take care about freeing it */
15571530 page_ref_unfreeze(new_page, 1);
@@ -1569,8 +1542,7 @@ static void khugepaged_scan_shmem(struct mm_struct *mm, · 11 unmodified lines
15691542 pgoff_t start, struct page **hpage)
15701543 {
15711544 struct page *page = NULL;
1572− struct radix_tree_iter iter;
1573− void **slot;
1545+ XA_STATE(xas, &mapping->i_pages, start);
15741546 int present, swap;
15751547 int node = NUMA_NO_NODE;
15761548 int result = SCAN_SUCCEED;
@@ -1579,17 +1551,11 @@ static void khugepaged_scan_shmem(struct mm_struct *mm, · 2 unmodified lines
15791551 swap = 0;
15801552 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
15811553 rcu_read_lock();
1582− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1583− if (iter.index >= start + HPAGE_PMD_NR)
1584− break;
1585−
1586− page = radix_tree_deref_slot(slot);
1587− if (radix_tree_deref_retry(page)) {
1588− slot = radix_tree_iter_retry(&iter);
1554+ xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) {
1555+ if (xas_retry(&xas, page))
15891556 continue;
1590− }
15911557
1592− if (radix_tree_exception(page)) {
1558+ if (xa_is_value(page)) {
15931559 if (++swap > khugepaged_max_ptes_swap) {
15941560 result = SCAN_EXCEED_SWAP_PTE;
15951561 break;
@@ -1628,7 +1594,7 @@ static void khugepaged_scan_shmem(struct mm_struct *mm, · 32 unmodified lines
16281594 present++;
16291595
16301596 if (need_resched()) {
1631− slot = radix_tree_iter_resume(slot, &iter);
1597+ xas_pause(&xas);
16321598 cond_resched_rcu();
16331599 }
16341600 }
mm/madvise.c+1−1
@@ -251,7 +251,7 @@ static void force_shm_swapin_readahead(struct vm_area_struct *vma, · 250 unmodified lines
251251 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
252252
253253 page = find_get_entry(mapping, index);
254− if (!radix_tree_exceptional_entry(page)) {
254+ if (!xa_is_value(page)) {
255255 if (page)
256256 put_page(page);
257257 continue;
mm/memcontrol.c+1−1
@@ -4728,7 +4728,7 @@ static struct page *mc_handle_file_pte(struct vm_area_struct *vma, · 4727 unmodified lines
47284728 /* shmem/tmpfs may report page out on swap: account for that too. */
47294729 if (shmem_mapping(mapping)) {
47304730 page = find_get_entry(mapping, pgoff);
4731− if (radix_tree_exceptional_entry(page)) {
4731+ if (xa_is_value(page)) {
47324732 swp_entry_t swp = radix_to_swp_entry(page);
47334733 if (do_memsw_account())
47344734 *entry = swp;
mm/memfd.c+43−62
@@ -21,44 +21,36 @@ · 20 unmodified lines
2121 #include <uapi/linux/memfd.h>
2222
2323 /*
24− * We need a tag: a new tag would expand every radix_tree_node by 8 bytes,
24+ * We need a tag: a new tag would expand every xa_node by 8 bytes,
2525 * so reuse a tag which we firmly believe is never set or cleared on tmpfs
2626 * or hugetlbfs because they are memory only filesystems.
2727 */
2828 #define MEMFD_TAG_PINNED PAGECACHE_TAG_TOWRITE
2929 #define LAST_SCAN 4 /* about 150ms max */
3030
31−static void memfd_tag_pins(struct address_space *mapping)
31+static void memfd_tag_pins(struct xa_state *xas)
3232 {
33− struct radix_tree_iter iter;
34− void __rcu **slot;
35− pgoff_t start;
3633 struct page *page;
34+ unsigned int tagged = 0;
3735
3836 lru_add_drain();
39− start = 0;
40− rcu_read_lock();
41−
42− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
43− page = radix_tree_deref_slot(slot);
44− if (!page || radix_tree_exception(page)) {
45− if (radix_tree_deref_retry(page)) {
46− slot = radix_tree_iter_retry(&iter);
47− continue;
48− }
49− } else if (page_count(page) - page_mapcount(page) > 1) {
50− xa_lock_irq(&mapping->i_pages);
51− radix_tree_tag_set(&mapping->i_pages, iter.index,
52− MEMFD_TAG_PINNED);
53− xa_unlock_irq(&mapping->i_pages);
54− }
5537
56− if (need_resched()) {
57− slot = radix_tree_iter_resume(slot, &iter);
58− cond_resched_rcu();
59− }
38+ xas_lock_irq(xas);
39+ xas_for_each(xas, page, ULONG_MAX) {
40+ if (xa_is_value(page))
41+ continue;
42+ if (page_count(page) - page_mapcount(page) > 1)
43+ xas_set_mark(xas, MEMFD_TAG_PINNED);
44+
45+ if (++tagged % XA_CHECK_SCHED)
46+ continue;
47+
48+ xas_pause(xas);
49+ xas_unlock_irq(xas);
50+ cond_resched();
51+ xas_lock_irq(xas);
6052 }
61− rcu_read_unlock();
53+ xas_unlock_irq(xas);
6254 }
6355
6456 /*
@@ -72,17 +64,17 @@ static void memfd_tag_pins(struct address_space *mapping) · 7 unmodified lines
7264 */
7365 static int memfd_wait_for_pins(struct address_space *mapping)
7466 {
75− struct radix_tree_iter iter;
76− void __rcu **slot;
77− pgoff_t start;
67+ XA_STATE(xas, &mapping->i_pages, 0);
7868 struct page *page;
7969 int error, scan;
8070
81− memfd_tag_pins(mapping);
71+ memfd_tag_pins(&xas);
8272
8373 error = 0;
8474 for (scan = 0; scan <= LAST_SCAN; scan++) {
85− if (!radix_tree_tagged(&mapping->i_pages, MEMFD_TAG_PINNED))
75+ unsigned int tagged = 0;
76+
77+ if (!xas_marked(&xas, MEMFD_TAG_PINNED))
8678 break;
8779
8880 if (!scan)
@@ -90,45 +82,34 @@ static int memfd_wait_for_pins(struct address_space *mapping) · 1 unmodified line
9082 else if (schedule_timeout_killable((HZ << scan) / 200))
9183 scan = LAST_SCAN;
9284
93− start = 0;
94− rcu_read_lock();
95− radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter,
96− start, MEMFD_TAG_PINNED) {
97−
98− page = radix_tree_deref_slot(slot);
99− if (radix_tree_exception(page)) {
100− if (radix_tree_deref_retry(page)) {
101− slot = radix_tree_iter_retry(&iter);
102− continue;
103− }
104−
105− page = NULL;
106− }
107−
108− if (page &&
109− page_count(page) - page_mapcount(page) != 1) {
110− if (scan < LAST_SCAN)
111− goto continue_resched;
112−
85+ xas_set(&xas, 0);
86+ xas_lock_irq(&xas);
87+ xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
88+ bool clear = true;
89+ if (xa_is_value(page))
90+ continue;
91+ if (page_count(page) - page_mapcount(page) != 1) {
11392 /*
11493 * On the last scan, we clean up all those tags
11594 * we inserted; but make a note that we still
11695 * found pages pinned.
11796 */
118− error = -EBUSY;
97+ if (scan == LAST_SCAN)
98+ error = -EBUSY;
99+ else
100+ clear = false;
119101 }
102+ if (clear)
103+ xas_clear_mark(&xas, MEMFD_TAG_PINNED);
104+ if (++tagged % XA_CHECK_SCHED)
105+ continue;
120106
121− xa_lock_irq(&mapping->i_pages);
122− radix_tree_tag_clear(&mapping->i_pages,
123− iter.index, MEMFD_TAG_PINNED);
124− xa_unlock_irq(&mapping->i_pages);
125−continue_resched:
126− if (need_resched()) {
127− slot = radix_tree_iter_resume(slot, &iter);
128− cond_resched_rcu();
129− }
107+ xas_pause(&xas);
108+ xas_unlock_irq(&xas);
109+ cond_resched();
110+ xas_lock_irq(&xas);
130111 }
131− rcu_read_unlock();
112+ xas_unlock_irq(&xas);
132113 }
133114
134115 return error;
mm/migrate.c+18−30
@@ -326,7 +326,7 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, · 325 unmodified lines
326326 page = migration_entry_to_page(entry);
327327
328328 /*
329− * Once radix-tree replacement of page migration started, page_count
329+ * Once page cache replacement of page migration started, page_count
330330 * *must* be zero. And, we don't want to call wait_on_page_locked()
331331 * against a page without get_page().
332332 * So, we use get_page_unless_zero(), here. Even failed, page fault
@@ -441,10 +441,10 @@ int migrate_page_move_mapping(struct address_space *mapping, · 108 unmodified lines
441441 struct buffer_head *head, enum migrate_mode mode,
442442 int extra_count)
443443 {
444+ XA_STATE(xas, &mapping->i_pages, page_index(page));
444445 struct zone *oldzone, *newzone;
445446 int dirty;
446447 int expected_count = 1 + extra_count;
447− void **pslot;
448448
449449 /*
450450 * Device public or private pages have an extra refcount as they are
@@ -470,21 +470,16 @@ int migrate_page_move_mapping(struct address_space *mapping, · 19 unmodified lines
470470 oldzone = page_zone(page);
471471 newzone = page_zone(newpage);
472472
473− xa_lock_irq(&mapping->i_pages);
474−
475− pslot = radix_tree_lookup_slot(&mapping->i_pages,
476− page_index(page));
473+ xas_lock_irq(&xas);
477474
478475 expected_count += hpage_nr_pages(page) + page_has_private(page);
479− if (page_count(page) != expected_count ||
480− radix_tree_deref_slot_protected(pslot,
481− &mapping->i_pages.xa_lock) != page) {
482− xa_unlock_irq(&mapping->i_pages);
476+ if (page_count(page) != expected_count || xas_load(&xas) != page) {
477+ xas_unlock_irq(&xas);
483478 return -EAGAIN;
484479 }
485480
486481 if (!page_ref_freeze(page, expected_count)) {
487− xa_unlock_irq(&mapping->i_pages);
482+ xas_unlock_irq(&xas);
488483 return -EAGAIN;
489484 }
490485
@@ -498,7 +493,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 7 unmodified lines
498493 if (mode == MIGRATE_ASYNC && head &&
499494 !buffer_migrate_lock_buffers(head, mode)) {
500495 page_ref_unfreeze(page, expected_count);
501− xa_unlock_irq(&mapping->i_pages);
496+ xas_unlock_irq(&xas);
502497 return -EAGAIN;
503498 }
504499
@@ -526,16 +521,13 @@ int migrate_page_move_mapping(struct address_space *mapping, · 21 unmodified lines
526521 SetPageDirty(newpage);
527522 }
528523
529− radix_tree_replace_slot(&mapping->i_pages, pslot, newpage);
524+ xas_store(&xas, newpage);
530525 if (PageTransHuge(page)) {
531526 int i;
532− int index = page_index(page);
533527
534528 for (i = 1; i < HPAGE_PMD_NR; i++) {
535− pslot = radix_tree_lookup_slot(&mapping->i_pages,
536− index + i);
537− radix_tree_replace_slot(&mapping->i_pages, pslot,
538− newpage + i);
529+ xas_next(&xas);
530+ xas_store(&xas, newpage + i);
539531 }
540532 }
541533
@@ -546,7 +538,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 4 unmodified lines
546538 */
547539 page_ref_unfreeze(page, expected_count - hpage_nr_pages(page));
548540
549− xa_unlock(&mapping->i_pages);
541+ xas_unlock(&xas);
550542 /* Leave irq disabled to prevent preemption while updating stats */
551543
552544 /*
@@ -586,22 +578,18 @@ EXPORT_SYMBOL(migrate_page_move_mapping); · 33 unmodified lines
586578 int migrate_huge_page_move_mapping(struct address_space *mapping,
587579 struct page *newpage, struct page *page)
588580 {
581+ XA_STATE(xas, &mapping->i_pages, page_index(page));
589582 int expected_count;
590− void **pslot;
591−
592− xa_lock_irq(&mapping->i_pages);
593−
594− pslot = radix_tree_lookup_slot(&mapping->i_pages, page_index(page));
595583
584+ xas_lock_irq(&xas);
596585 expected_count = 2 + page_has_private(page);
597− if (page_count(page) != expected_count ||
598− radix_tree_deref_slot_protected(pslot, &mapping->i_pages.xa_lock) != page) {
599− xa_unlock_irq(&mapping->i_pages);
586+ if (page_count(page) != expected_count || xas_load(&xas) != page) {
587+ xas_unlock_irq(&xas);
600588 return -EAGAIN;
601589 }
602590
603591 if (!page_ref_freeze(page, expected_count)) {
604− xa_unlock_irq(&mapping->i_pages);
592+ xas_unlock_irq(&xas);
605593 return -EAGAIN;
606594 }
607595
@@ -610,11 +598,11 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, · 2 unmodified lines
610598
611599 get_page(newpage);
612600
613− radix_tree_replace_slot(&mapping->i_pages, pslot, newpage);
601+ xas_store(&xas, newpage);
614602
615603 page_ref_unfreeze(page, expected_count - 1);
616604
617− xa_unlock_irq(&mapping->i_pages);
605+ xas_unlock_irq(&xas);
618606
619607 return MIGRATEPAGE_SUCCESS;
620608 }
mm/mincore.c+1−1
@@ -66,7 +66,7 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t pgoff) · 65 unmodified lines
6666 * shmem/tmpfs may return swap: account for swapcache
6767 * page too.
6868 */
69− if (radix_tree_exceptional_entry(page)) {
69+ if (xa_is_value(page)) {
7070 swp_entry_t swp = radix_to_swp_entry(page);
7171 page = find_get_page(swap_address_space(swp),
7272 swp_offset(swp));
mm/page-writeback.c+26−46
@@ -2097,34 +2097,25 @@ void __init page_writeback_init(void) · 2096 unmodified lines
20972097 * dirty pages in the file (thus it is important for this function to be quick
20982098 * so that it can tag pages faster than a dirtying process can create them).
20992099 */
2100−/*
2101− * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce the i_pages lock
2102− * latency.
2103− */
21042100 void tag_pages_for_writeback(struct address_space *mapping,
21052101 pgoff_t start, pgoff_t end)
21062102 {
2107−#define WRITEBACK_TAG_BATCH 4096
2108− unsigned long tagged = 0;
2109− struct radix_tree_iter iter;
2110− void **slot;
2103+ XA_STATE(xas, &mapping->i_pages, start);
2104+ unsigned int tagged = 0;
2105+ void *page;
21112106
2112− xa_lock_irq(&mapping->i_pages);
2113− radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start,
2114− PAGECACHE_TAG_DIRTY) {
2115− if (iter.index > end)
2116− break;
2117− radix_tree_iter_tag_set(&mapping->i_pages, &iter,
2118− PAGECACHE_TAG_TOWRITE);
2119− tagged++;
2120− if ((tagged % WRITEBACK_TAG_BATCH) != 0)
2107+ xas_lock_irq(&xas);
2108+ xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) {
2109+ xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
2110+ if (++tagged % XA_CHECK_SCHED)
21212111 continue;
2122− slot = radix_tree_iter_resume(slot, &iter);
2123− xa_unlock_irq(&mapping->i_pages);
2112+
2113+ xas_pause(&xas);
2114+ xas_unlock_irq(&xas);
21242115 cond_resched();
2125− xa_lock_irq(&mapping->i_pages);
2116+ xas_lock_irq(&xas);
21262117 }
2127− xa_unlock_irq(&mapping->i_pages);
2118+ xas_unlock_irq(&xas);
21282119 }
21292120 EXPORT_SYMBOL(tag_pages_for_writeback);
21302121
@@ -2170,7 +2161,7 @@ int write_cache_pages(struct address_space *mapping, · 39 unmodified lines
21702161 pgoff_t end; /* Inclusive */
21712162 pgoff_t done_index;
21722163 int range_whole = 0;
2173− int tag;
2164+ xa_mark_t tag;
21742165
21752166 pagevec_init(&pvec);
21762167 if (wbc->range_cyclic) {
@@ -2442,7 +2433,7 @@ void account_page_cleaned(struct page *page, struct address_space *mapping, · 265 unmodified lines
24422433
24432434 /*
24442435 * For address_spaces which do not use buffers. Just tag the page as dirty in
2445− * its radix tree.
2436+ * the xarray.
24462437 *
24472438 * This is also used when a single buffer is being dirtied: we want to set the
24482439 * page dirty in that case, but not all the buffers. This is a "bottom-up"
@@ -2468,7 +2459,7 @@ int __set_page_dirty_nobuffers(struct page *page) · 19 unmodified lines
24682459 BUG_ON(page_mapping(page) != mapping);
24692460 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
24702461 account_page_dirtied(page, mapping);
2471− radix_tree_tag_set(&mapping->i_pages, page_index(page),
2462+ __xa_set_mark(&mapping->i_pages, page_index(page),
24722463 PAGECACHE_TAG_DIRTY);
24732464 xa_unlock_irqrestore(&mapping->i_pages, flags);
24742465 unlock_page_memcg(page);
@@ -2631,13 +2622,13 @@ EXPORT_SYMBOL(__cancel_dirty_page); · 156 unmodified lines
26312622 * Returns true if the page was previously dirty.
26322623 *
26332624 * This is for preparing to put the page under writeout. We leave the page
2634− * tagged as dirty in the radix tree so that a concurrent write-for-sync
2625+ * tagged as dirty in the xarray so that a concurrent write-for-sync
26352626 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage
26362627 * implementation will run either set_page_writeback() or set_page_dirty(),
2637− * at which stage we bring the page's dirty flag and radix-tree dirty tag
2628+ * at which stage we bring the page's dirty flag and xarray dirty tag
26382629 * back into sync.
26392630 *
2640− * This incoherency between the page's dirty flag and radix-tree tag is
2631+ * This incoherency between the page's dirty flag and xarray tag is
26412632 * unfortunate, but it only exists while the page is locked.
26422633 */
26432634 int clear_page_dirty_for_io(struct page *page)
@@ -2718,7 +2709,7 @@ int test_clear_page_writeback(struct page *page) · 74 unmodified lines
27182709 xa_lock_irqsave(&mapping->i_pages, flags);
27192710 ret = TestClearPageWriteback(page);
27202711 if (ret) {
2721− radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2712+ __xa_clear_mark(&mapping->i_pages, page_index(page),
27222713 PAGECACHE_TAG_WRITEBACK);
27232714 if (bdi_cap_account_writeback(bdi)) {
27242715 struct bdi_writeback *wb = inode_to_wb(inode);
@@ -2758,11 +2749,13 @@ int __test_set_page_writeback(struct page *page, bool keep_write) · 33 unmodified lines
27582749
27592750 lock_page_memcg(page);
27602751 if (mapping && mapping_use_writeback_tags(mapping)) {
2752+ XA_STATE(xas, &mapping->i_pages, page_index(page));
27612753 struct inode *inode = mapping->host;
27622754 struct backing_dev_info *bdi = inode_to_bdi(inode);
27632755 unsigned long flags;
27642756
2765− xa_lock_irqsave(&mapping->i_pages, flags);
2757+ xas_lock_irqsave(&xas, flags);
2758+ xas_load(&xas);
27662759 ret = TestSetPageWriteback(page);
27672760 if (!ret) {
27682761 bool on_wblist;
@@ -2770,8 +2763,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write) · 1 unmodified line
27702763 on_wblist = mapping_tagged(mapping,
27712764 PAGECACHE_TAG_WRITEBACK);
27722765
2773− radix_tree_tag_set(&mapping->i_pages, page_index(page),
2774− PAGECACHE_TAG_WRITEBACK);
2766+ xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
27752767 if (bdi_cap_account_writeback(bdi))
27762768 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);
27772769
@@ -2784,12 +2776,10 @@ int __test_set_page_writeback(struct page *page, bool keep_write) · 6 unmodified lines
27842776 sb_mark_inode_writeback(mapping->host);
27852777 }
27862778 if (!PageDirty(page))
2787− radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2788− PAGECACHE_TAG_DIRTY);
2779+ xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
27892780 if (!keep_write)
2790− radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2791− PAGECACHE_TAG_TOWRITE);
2792− xa_unlock_irqrestore(&mapping->i_pages, flags);
2781+ xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
2782+ xas_unlock_irqrestore(&xas, flags);
27932783 } else {
27942784 ret = TestSetPageWriteback(page);
27952785 }
@@ -2803,16 +2793,6 @@ int __test_set_page_writeback(struct page *page, bool keep_write) · 7 unmodified lines
28032793 }
28042794 EXPORT_SYMBOL(__test_set_page_writeback);
28052795
2806−/*
2807− * Return true if any of the pages in the mapping are marked with the
2808− * passed tag.
2809− */
2810−int mapping_tagged(struct address_space *mapping, int tag)
2811−{
2812− return radix_tree_tagged(&mapping->i_pages, tag);
2813−}
2814−EXPORT_SYMBOL(mapping_tagged);
2815−
28162796 /**
28172797 * wait_for_stable_page() - wait for writeback to finish, if necessary.
28182798 * @page: The page to wait on.
mm/readahead.c+4−6
@@ -176,10 +176,8 @@ unsigned int __do_page_cache_readahead(struct address_space *mapping, · 175 unmodified lines
176176 if (page_offset > end_index)
177177 break;
178178
179− rcu_read_lock();
180− page = radix_tree_lookup(&mapping->i_pages, page_offset);
181− rcu_read_unlock();
182− if (page && !radix_tree_exceptional_entry(page)) {
179+ page = xa_load(&mapping->i_pages, page_offset);
180+ if (page && !xa_is_value(page)) {
183181 /*
184182 * Page already present? Kick off the current batch of
185183 * contiguous pages before continuing with the next
@@ -336,7 +334,7 @@ static pgoff_t count_history_pages(struct address_space *mapping, · 150 unmodified lines
336334 pgoff_t head;
337335
338336 rcu_read_lock();
339− head = page_cache_prev_hole(mapping, offset - 1, max);
337+ head = page_cache_prev_miss(mapping, offset - 1, max);
340338 rcu_read_unlock();
341339
342340 return offset - 1 - head;
@@ -425,7 +423,7 @@ ondemand_readahead(struct address_space *mapping, · 82 unmodified lines
425423 pgoff_t start;
426424
427425 rcu_read_lock();
428− start = page_cache_next_hole(mapping, offset + 1, max_pages);
426+ start = page_cache_next_miss(mapping, offset + 1, max_pages);
429427 rcu_read_unlock();
430428
431429 if (!start || start - offset > max_pages)
mm/shmem.c+73−120
@@ -322,24 +322,20 @@ void shmem_uncharge(struct inode *inode, long pages) · 321 unmodified lines
322322 }
323323
324324 /*
325− * Replace item expected in radix tree by a new item, while holding tree lock.
325+ * Replace item expected in xarray by a new item, while holding xa_lock.
326326 */
327−static int shmem_radix_tree_replace(struct address_space *mapping,
327+static int shmem_replace_entry(struct address_space *mapping,
328328 pgoff_t index, void *expected, void *replacement)
329329 {
330− struct radix_tree_node *node;
331− void __rcu **pslot;
330+ XA_STATE(xas, &mapping->i_pages, index);
332331 void *item;
333332
334333 VM_BUG_ON(!expected);
335334 VM_BUG_ON(!replacement);
336− item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot);
337− if (!item)
338− return -ENOENT;
335+ item = xas_load(&xas);
339336 if (item != expected)
340337 return -ENOENT;
341− __radix_tree_replace(&mapping->i_pages, node, pslot,
342− replacement, NULL);
338+ xas_store(&xas, replacement);
343339 return 0;
344340 }
345341
@@ -353,12 +349,7 @@ static int shmem_radix_tree_replace(struct address_space *mapping, · 7 unmodified lines
353349 static bool shmem_confirm_swap(struct address_space *mapping,
354350 pgoff_t index, swp_entry_t swap)
355351 {
356− void *item;
357−
358− rcu_read_lock();
359− item = radix_tree_lookup(&mapping->i_pages, index);
360− rcu_read_unlock();
361− return item == swp_to_radix_entry(swap);
352+ return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
362353 }
363354
364355 /*
@@ -586,9 +577,11 @@ static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo) · 221 unmodified lines
586577 */
587578 static int shmem_add_to_page_cache(struct page *page,
588579 struct address_space *mapping,
589− pgoff_t index, void *expected)
580+ pgoff_t index, void *expected, gfp_t gfp)
590581 {
591− int error, nr = hpage_nr_pages(page);
582+ XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
583+ unsigned long i = 0;
584+ unsigned long nr = 1UL << compound_order(page);
592585
593586 VM_BUG_ON_PAGE(PageTail(page), page);
594587 VM_BUG_ON_PAGE(index != round_down(index, nr), page);
@@ -600,47 +593,39 @@ static int shmem_add_to_page_cache(struct page *page, · 5 unmodified lines
600593 page->mapping = mapping;
601594 page->index = index;
602595
603− xa_lock_irq(&mapping->i_pages);
604− if (PageTransHuge(page)) {
605− void __rcu **results;
606− pgoff_t idx;
607− int i;
608−
609− error = 0;
610− if (radix_tree_gang_lookup_slot(&mapping->i_pages,
611− &results, &idx, index, 1) &&
612− idx < index + HPAGE_PMD_NR) {
613− error = -EEXIST;
596+ do {
597+ void *entry;
598+ xas_lock_irq(&xas);
599+ entry = xas_find_conflict(&xas);
600+ if (entry != expected)
601+ xas_set_err(&xas, -EEXIST);
602+ xas_create_range(&xas);
603+ if (xas_error(&xas))
604+ goto unlock;
605+next:
606+ xas_store(&xas, page + i);
607+ if (++i < nr) {
608+ xas_next(&xas);
609+ goto next;
614610 }
615−
616− if (!error) {
617− for (i = 0; i < HPAGE_PMD_NR; i++) {
618− error = radix_tree_insert(&mapping->i_pages,
619− index + i, page + i);
620− VM_BUG_ON(error);
621− }
611+ if (PageTransHuge(page)) {
622612 count_vm_event(THP_FILE_ALLOC);
613+ __inc_node_page_state(page, NR_SHMEM_THPS);
623614 }
624− } else if (!expected) {
625− error = radix_tree_insert(&mapping->i_pages, index, page);
626− } else {
627− error = shmem_radix_tree_replace(mapping, index, expected,
628− page);
629− }
630−
631− if (!error) {
632615 mapping->nrpages += nr;
633− if (PageTransHuge(page))
634− __inc_node_page_state(page, NR_SHMEM_THPS);
635616 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
636617 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
637− xa_unlock_irq(&mapping->i_pages);
638− } else {
618+unlock:
619+ xas_unlock_irq(&xas);
620+ } while (xas_nomem(&xas, gfp));
621+
622+ if (xas_error(&xas)) {
639623 page->mapping = NULL;
640− xa_unlock_irq(&mapping->i_pages);
641624 page_ref_sub(page, nr);
625+ return xas_error(&xas);
642626 }
643− return error;
627+
628+ return 0;
644629 }
645630
646631 /*
@@ -654,7 +639,7 @@ static void shmem_delete_from_page_cache(struct page *page, void *radswap) · 7 unmodified lines
654639 VM_BUG_ON_PAGE(PageCompound(page), page);
655640
656641 xa_lock_irq(&mapping->i_pages);
657− error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
642+ error = shmem_replace_entry(mapping, page->index, page, radswap);
658643 page->mapping = NULL;
659644 mapping->nrpages--;
660645 __dec_node_page_state(page, NR_FILE_PAGES);
@@ -665,7 +650,7 @@ static void shmem_delete_from_page_cache(struct page *page, void *radswap) · 4 unmodified lines
665650 }
666651
667652 /*
668− * Remove swap entry from radix tree, free the swap and its page cache.
653+ * Remove swap entry from page cache, free the swap and its page cache.
669654 */
670655 static int shmem_free_swap(struct address_space *mapping,
671656 pgoff_t index, void *radswap)
@@ -673,7 +658,7 @@ static int shmem_free_swap(struct address_space *mapping, · 1 unmodified line
673658 void *old;
674659
675660 xa_lock_irq(&mapping->i_pages);
676− old = radix_tree_delete_item(&mapping->i_pages, index, radswap);
661+ old = __xa_cmpxchg(&mapping->i_pages, index, radswap, NULL, 0);
677662 xa_unlock_irq(&mapping->i_pages);
678663 if (old != radswap)
679664 return -ENOENT;
@@ -691,29 +676,19 @@ static int shmem_free_swap(struct address_space *mapping, · 11 unmodified lines
691676 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
692677 pgoff_t start, pgoff_t end)
693678 {
694− struct radix_tree_iter iter;
695− void __rcu **slot;
679+ XA_STATE(xas, &mapping->i_pages, start);
696680 struct page *page;
697681 unsigned long swapped = 0;
698682
699683 rcu_read_lock();
700−
701− radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
702− if (iter.index >= end)
703− break;
704−
705− page = radix_tree_deref_slot(slot);
706−
707− if (radix_tree_deref_retry(page)) {
708− slot = radix_tree_iter_retry(&iter);
684+ xas_for_each(&xas, page, end - 1) {
685+ if (xas_retry(&xas, page))
709686 continue;
710− }
711−
712− if (radix_tree_exceptional_entry(page))
687+ if (xa_is_value(page))
713688 swapped++;
714689
715690 if (need_resched()) {
716− slot = radix_tree_iter_resume(slot, &iter);
691+ xas_pause(&xas);
717692 cond_resched_rcu();
718693 }
719694 }
@@ -788,7 +763,7 @@ void shmem_unlock_mapping(struct address_space *mapping) · 68 unmodified lines
788763 }
789764
790765 /*
791− * Remove range of pages and swap entries from radix tree, and free them.
766+ * Remove range of pages and swap entries from page cache, and free them.
792767 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
793768 */
794769 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
@@ -824,7 +799,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 29 unmodified lines
824799 if (index >= end)
825800 break;
826801
827− if (radix_tree_exceptional_entry(page)) {
802+ if (xa_is_value(page)) {
828803 if (unfalloc)
829804 continue;
830805 nr_swaps_freed += !shmem_free_swap(mapping,
@@ -921,7 +896,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 90 unmodified lines
921896 if (index >= end)
922897 break;
923898
924− if (radix_tree_exceptional_entry(page)) {
899+ if (xa_is_value(page)) {
925900 if (unfalloc)
926901 continue;
927902 if (shmem_free_swap(mapping, index, page)) {
@@ -1110,34 +1085,27 @@ static void shmem_evict_inode(struct inode *inode) · 182 unmodified lines
11101085 clear_inode(inode);
11111086 }
11121087
1113−static unsigned long find_swap_entry(struct radix_tree_root *root, void *item)
1088+static unsigned long find_swap_entry(struct xarray *xa, void *item)
11141089 {
1115− struct radix_tree_iter iter;
1116− void __rcu **slot;
1117− unsigned long found = -1;
1090+ XA_STATE(xas, xa, 0);
11181091 unsigned int checked = 0;
1092+ void *entry;
11191093
11201094 rcu_read_lock();
1121− radix_tree_for_each_slot(slot, root, &iter, 0) {
1122− void *entry = radix_tree_deref_slot(slot);
1123−
1124− if (radix_tree_deref_retry(entry)) {
1125− slot = radix_tree_iter_retry(&iter);
1095+ xas_for_each(&xas, entry, ULONG_MAX) {
1096+ if (xas_retry(&xas, entry))
11261097 continue;
1127− }
1128− if (entry == item) {
1129− found = iter.index;
1098+ if (entry == item)
11301099 break;
1131− }
11321100 checked++;
1133− if ((checked % 4096) != 0)
1101+ if ((checked % XA_CHECK_SCHED) != 0)
11341102 continue;
1135− slot = radix_tree_iter_resume(slot, &iter);
1103+ xas_pause(&xas);
11361104 cond_resched_rcu();
11371105 }
1138−
11391106 rcu_read_unlock();
1140− return found;
1107+
1108+ return entry ? xas.xa_index : -1;
11411109 }
11421110
11431111 /*
@@ -1175,10 +1143,10 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, · 31 unmodified lines
11751143 * We needed to drop mutex to make that restrictive page
11761144 * allocation, but the inode might have been freed while we
11771145 * dropped it: although a racing shmem_evict_inode() cannot
1178− * complete without emptying the radix_tree, our page lock
1146+ * complete without emptying the page cache, our page lock
11791147 * on this swapcache page is not enough to prevent that -
11801148 * free_swap_and_cache() of our swap entry will only
1181− * trylock_page(), removing swap from radix_tree whatever.
1149+ * trylock_page(), removing swap from page cache whatever.
11821150 *
11831151 * We must not proceed to shmem_add_to_page_cache() if the
11841152 * inode has been freed, but of course we cannot rely on
@@ -1200,7 +1168,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, · 15 unmodified lines
12001168 */
12011169 if (!error)
12021170 error = shmem_add_to_page_cache(*pagep, mapping, index,
1203− radswap);
1171+ radswap, gfp);
12041172 if (error != -ENOMEM) {
12051173 /*
12061174 * Truncation and eviction use free_swap_and_cache(), which
@@ -1244,7 +1212,7 @@ int shmem_unuse(swp_entry_t swap, struct page *page) · 37 unmodified lines
12441212 &memcg, false);
12451213 if (error)
12461214 goto out;
1247− /* No radix_tree_preload: swap entry keeps a place for page in tree */
1215+ /* No memory allocation: swap entry occupies the slot for the page */
12481216 error = -EAGAIN;
12491217
12501218 mutex_lock(&shmem_swaplist_mutex);
@@ -1453,23 +1421,17 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp, · 202 unmodified lines
14531421 struct shmem_inode_info *info, pgoff_t index)
14541422 {
14551423 struct vm_area_struct pvma;
1456− struct inode *inode = &info->vfs_inode;
1457− struct address_space *mapping = inode->i_mapping;
1458− pgoff_t idx, hindex;
1459− void __rcu **results;
1424+ struct address_space *mapping = info->vfs_inode.i_mapping;
1425+ pgoff_t hindex;
14601426 struct page *page;
14611427
14621428 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
14631429 return NULL;
14641430
14651431 hindex = round_down(index, HPAGE_PMD_NR);
1466− rcu_read_lock();
1467− if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx,
1468− hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1469− rcu_read_unlock();
1432+ if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1433+ XA_PRESENT))
14701434 return NULL;
1471− }
1472− rcu_read_unlock();
14731435
14741436 shmem_pseudo_vma_init(&pvma, info, hindex);
14751437 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
@@ -1578,8 +1540,7 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp, · 102 unmodified lines
15781540 * a nice clean interface for us to replace oldpage by newpage there.
15791541 */
15801542 xa_lock_irq(&swap_mapping->i_pages);
1581− error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1582− newpage);
1543+ error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
15831544 if (!error) {
15841545 __inc_node_page_state(newpage, NR_FILE_PAGES);
15851546 __dec_node_page_state(oldpage, NR_FILE_PAGES);
@@ -1643,7 +1604,7 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, · 57 unmodified lines
16431604 repeat:
16441605 swap.val = 0;
16451606 page = find_lock_entry(mapping, index);
1646− if (radix_tree_exceptional_entry(page)) {
1607+ if (xa_is_value(page)) {
16471608 swap = radix_to_swp_entry(page);
16481609 page = NULL;
16491610 }
@@ -1718,7 +1679,7 @@ repeat: · 68 unmodified lines
17181679 false);
17191680 if (!error) {
17201681 error = shmem_add_to_page_cache(page, mapping, index,
1721− swp_to_radix_entry(swap));
1682+ swp_to_radix_entry(swap), gfp);
17221683 /*
17231684 * We already confirmed swap under page lock, and make
17241685 * no memory allocation here, so usually no possibility
@@ -1824,13 +1785,8 @@ alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, inode, · 99 unmodified lines
18241785 PageTransHuge(page));
18251786 if (error)
18261787 goto unacct;
1827− error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK,
1828− compound_order(page));
1829− if (!error) {
1830− error = shmem_add_to_page_cache(page, mapping, hindex,
1831− NULL);
1832− radix_tree_preload_end();
1833− }
1788+ error = shmem_add_to_page_cache(page, mapping, hindex,
1789+ NULL, gfp & GFP_RECLAIM_MASK);
18341790 if (error) {
18351791 mem_cgroup_cancel_charge(page, memcg,
18361792 PageTransHuge(page));
@@ -1931,7 +1887,7 @@ unlock: · 94 unmodified lines
19311887 spin_unlock_irq(&info->lock);
19321888 goto repeat;
19331889 }
1934− if (error == -EEXIST) /* from above or from radix_tree_insert */
1890+ if (error == -EEXIST)
19351891 goto repeat;
19361892 return error;
19371893 }
@@ -2299,11 +2255,8 @@ static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, · 361 unmodified lines
22992255 if (ret)
23002256 goto out_release;
23012257
2302− ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
2303− if (!ret) {
2304− ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL);
2305− radix_tree_preload_end();
2306− }
2258+ ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2259+ gfp & GFP_RECLAIM_MASK);
23072260 if (ret)
23082261 goto out_release_uncharge;
23092262
@@ -2548,7 +2501,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · 238 unmodified lines
25482501 }
25492502
25502503 /*
2551− * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
2504+ * llseek SEEK_DATA or SEEK_HOLE through the page cache.
25522505 */
25532506 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
25542507 pgoff_t index, pgoff_t end, int whence)
@@ -2578,7 +2531,7 @@ static pgoff_t shmem_seek_hole_data(struct address_space *mapping, · 23 unmodified lines
25782531 index = indices[i];
25792532 }
25802533 page = pvec.pages[i];
2581− if (page && !radix_tree_exceptional_entry(page)) {
2534+ if (page && !xa_is_value(page)) {
25822535 if (!PageUptodate(page))
25832536 page = NULL;
25842537 }
mm/swap.c+3−3
@@ -964,7 +964,7 @@ void pagevec_remove_exceptionals(struct pagevec *pvec) · 963 unmodified lines
964964
965965 for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
966966 struct page *page = pvec->pages[i];
967− if (!radix_tree_exceptional_entry(page))
967+ if (!xa_is_value(page))
968968 pvec->pages[j++] = page;
969969 }
970970 pvec->nr = j;
@@ -1001,7 +1001,7 @@ EXPORT_SYMBOL(pagevec_lookup_range); · 30 unmodified lines
10011001
10021002 unsigned pagevec_lookup_range_tag(struct pagevec *pvec,
10031003 struct address_space *mapping, pgoff_t *index, pgoff_t end,
1004− int tag)
1004+ xa_mark_t tag)
10051005 {
10061006 pvec->nr = find_get_pages_range_tag(mapping, index, end, tag,
10071007 PAGEVEC_SIZE, pvec->pages);
@@ -1011,7 +1011,7 @@ EXPORT_SYMBOL(pagevec_lookup_range_tag); · 3 unmodified lines
10111011
10121012 unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec,
10131013 struct address_space *mapping, pgoff_t *index, pgoff_t end,
1014− int tag, unsigned max_pages)
1014+ xa_mark_t tag, unsigned max_pages)
10151015 {
10161016 pvec->nr = find_get_pages_range_tag(mapping, index, end, tag,
10171017 min_t(unsigned int, max_pages, PAGEVEC_SIZE), pvec->pages);
mm/swap_state.c+40−79
@@ -107,14 +107,15 @@ void show_swap_cache_info(void) · 106 unmodified lines
107107 }
108108
109109 /*
110− * __add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
110+ * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
111111 * but sets SwapCache flag and private instead of mapping and index.
112112 */
113−int __add_to_swap_cache(struct page *page, swp_entry_t entry)
113+int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp)
114114 {
115− int error, i, nr = hpage_nr_pages(page);
116− struct address_space *address_space;
115+ struct address_space *address_space = swap_address_space(entry);
117116 pgoff_t idx = swp_offset(entry);
117+ XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page));
118+ unsigned long i, nr = 1UL << compound_order(page);
118119
119120 VM_BUG_ON_PAGE(!PageLocked(page), page);
120121 VM_BUG_ON_PAGE(PageSwapCache(page), page);
@@ -123,73 +124,52 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry) · 2 unmodified lines
123124 page_ref_add(page, nr);
124125 SetPageSwapCache(page);
125126
126− address_space = swap_address_space(entry);
127− xa_lock_irq(&address_space->i_pages);
128− for (i = 0; i < nr; i++) {
129− set_page_private(page + i, entry.val + i);
130− error = radix_tree_insert(&address_space->i_pages,
131− idx + i, page + i);
132− if (unlikely(error))
133− break;
134− }
135− if (likely(!error)) {
127+ do {
128+ xas_lock_irq(&xas);
129+ xas_create_range(&xas);
130+ if (xas_error(&xas))
131+ goto unlock;
132+ for (i = 0; i < nr; i++) {
133+ VM_BUG_ON_PAGE(xas.xa_index != idx + i, page);
134+ set_page_private(page + i, entry.val + i);
135+ xas_store(&xas, page + i);
136+ xas_next(&xas);
137+ }
136138 address_space->nrpages += nr;
137139 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
138140 ADD_CACHE_INFO(add_total, nr);
139− } else {
140− /*
141− * Only the context which have set SWAP_HAS_CACHE flag
142− * would call add_to_swap_cache().
143− * So add_to_swap_cache() doesn't returns -EEXIST.
144− */
145− VM_BUG_ON(error == -EEXIST);
146− set_page_private(page + i, 0UL);
147− while (i--) {
148− radix_tree_delete(&address_space->i_pages, idx + i);
149− set_page_private(page + i, 0UL);
150− }
151− ClearPageSwapCache(page);
152− page_ref_sub(page, nr);
153− }
154− xa_unlock_irq(&address_space->i_pages);
141+unlock:
142+ xas_unlock_irq(&xas);
143+ } while (xas_nomem(&xas, gfp));
155144
156− return error;
157−}
158−
159−
160−int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask)
161−{
162− int error;
145+ if (!xas_error(&xas))
146+ return 0;
163147
164− error = radix_tree_maybe_preload_order(gfp_mask, compound_order(page));
165− if (!error) {
166− error = __add_to_swap_cache(page, entry);
167− radix_tree_preload_end();
168− }
169− return error;
148+ ClearPageSwapCache(page);
149+ page_ref_sub(page, nr);
150+ return xas_error(&xas);
170151 }
171152
172153 /*
173154 * This must be called only on pages that have
174155 * been verified to be in the swap cache.
175156 */
176−void __delete_from_swap_cache(struct page *page)
157+void __delete_from_swap_cache(struct page *page, swp_entry_t entry)
177158 {
178− struct address_space *address_space;
159+ struct address_space *address_space = swap_address_space(entry);
179160 int i, nr = hpage_nr_pages(page);
180− swp_entry_t entry;
181− pgoff_t idx;
161+ pgoff_t idx = swp_offset(entry);
162+ XA_STATE(xas, &address_space->i_pages, idx);
182163
183164 VM_BUG_ON_PAGE(!PageLocked(page), page);
184165 VM_BUG_ON_PAGE(!PageSwapCache(page), page);
185166 VM_BUG_ON_PAGE(PageWriteback(page), page);
186167
187− entry.val = page_private(page);
188− address_space = swap_address_space(entry);
189− idx = swp_offset(entry);
190168 for (i = 0; i < nr; i++) {
191− radix_tree_delete(&address_space->i_pages, idx + i);
169+ void *entry = xas_store(&xas, NULL);
170+ VM_BUG_ON_PAGE(entry != page + i, entry);
192171 set_page_private(page + i, 0);
172+ xas_next(&xas);
193173 }
194174 ClearPageSwapCache(page);
195175 address_space->nrpages -= nr;
@@ -217,7 +197,7 @@ int add_to_swap(struct page *page) · 21 unmodified lines
217197 return 0;
218198
219199 /*
220− * Radix-tree node allocations from PF_MEMALLOC contexts could
200+ * XArray node allocations from PF_MEMALLOC contexts could
221201 * completely exhaust the page allocator. __GFP_NOMEMALLOC
222202 * stops emergency reserves from being allocated.
223203 *
@@ -229,7 +209,6 @@ int add_to_swap(struct page *page) · 5 unmodified lines
229209 */
230210 err = add_to_swap_cache(page, entry,
231211 __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN);
232− /* -ENOMEM radix-tree allocation failure */
233212 if (err)
234213 /*
235214 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
@@ -263,14 +242,11 @@ fail: · 27 unmodified lines
263242 */
264243 void delete_from_swap_cache(struct page *page)
265244 {
266− swp_entry_t entry;
267− struct address_space *address_space;
245+ swp_entry_t entry = { .val = page_private(page) };
246+ struct address_space *address_space = swap_address_space(entry);
268247
269− entry.val = page_private(page);
270−
271− address_space = swap_address_space(entry);
272248 xa_lock_irq(&address_space->i_pages);
273− __delete_from_swap_cache(page);
249+ __delete_from_swap_cache(page, entry);
274250 xa_unlock_irq(&address_space->i_pages);
275251
276252 put_swap_page(page, entry);
@@ -413,19 +389,11 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, · 136 unmodified lines
413389 break; /* Out of memory */
414390 }
415391
416− /*
417− * call radix_tree_preload() while we can wait.
418− */
419− err = radix_tree_maybe_preload(gfp_mask & GFP_KERNEL);
420− if (err)
421− break;
422−
423392 /*
424393 * Swap entry may have been freed since our caller observed it.
425394 */
426395 err = swapcache_prepare(entry);
427396 if (err == -EEXIST) {
428− radix_tree_preload_end();
429397 /*
430398 * We might race against get_swap_page() and stumble
431399 * across a SWAP_HAS_CACHE swap_map entry whose page
@@ -433,27 +401,20 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, · 1 unmodified line
433401 */
434402 cond_resched();
435403 continue;
436− }
437− if (err) { /* swp entry is obsolete ? */
438− radix_tree_preload_end();
404+ } else if (err) /* swp entry is obsolete ? */
439405 break;
440− }
441406
442− /* May fail (-ENOMEM) if radix-tree node allocation failed. */
407+ /* May fail (-ENOMEM) if XArray node allocation failed. */
443408 __SetPageLocked(new_page);
444409 __SetPageSwapBacked(new_page);
445− err = __add_to_swap_cache(new_page, entry);
410+ err = add_to_swap_cache(new_page, entry, gfp_mask & GFP_KERNEL);
446411 if (likely(!err)) {
447− radix_tree_preload_end();
448− /*
449− * Initiate read into locked page and return.
450− */
412+ /* Initiate read into locked page */
451413 SetPageWorkingset(new_page);
452414 lru_cache_add_anon(new_page);
453415 *new_page_allocated = true;
454416 return new_page;
455417 }
456− radix_tree_preload_end();
457418 __ClearPageLocked(new_page);
458419 /*
459420 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
@@ -626,7 +587,7 @@ int init_swap_address_space(unsigned int type, unsigned long nr_pages) · 166 unmodified lines
626587 return -ENOMEM;
627588 for (i = 0; i < nr; i++) {
628589 space = spaces + i;
629− INIT_RADIX_TREE(&space->i_pages, GFP_ATOMIC|__GFP_NOWARN);
590+ xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ);
630591 atomic_set(&space->i_mmap_writable, 0);
631592 space->a_ops = &swap_aops;
632593 /* swap cache doesn't use writeback related tags */
mm/truncate.c+12−15
@@ -33,15 +33,12 @@ · 32 unmodified lines
3333 static inline void __clear_shadow_entry(struct address_space *mapping,
3434 pgoff_t index, void *entry)
3535 {
36− struct radix_tree_node *node;
37− void **slot;
36+ XA_STATE(xas, &mapping->i_pages, index);
3837
39− if (!__radix_tree_lookup(&mapping->i_pages, index, &node, &slot))
38+ xas_set_update(&xas, workingset_update_node);
39+ if (xas_load(&xas) != entry)
4040 return;
41− if (*slot != entry)
42− return;
43− __radix_tree_replace(&mapping->i_pages, node, slot, NULL,
44− workingset_update_node);
41+ xas_store(&xas, NULL);
4542 mapping->nrexceptional--;
4643 }
4744
@@ -70,7 +67,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, · 22 unmodified lines
7067 return;
7168
7269 for (j = 0; j < pagevec_count(pvec); j++)
73− if (radix_tree_exceptional_entry(pvec->pages[j]))
70+ if (xa_is_value(pvec->pages[j]))
7471 break;
7572
7673 if (j == pagevec_count(pvec))
@@ -85,7 +82,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, · 8 unmodified lines
8582 struct page *page = pvec->pages[i];
8683 pgoff_t index = indices[i];
8784
88− if (!radix_tree_exceptional_entry(page)) {
85+ if (!xa_is_value(page)) {
8986 pvec->pages[j++] = page;
9087 continue;
9188 }
@@ -347,7 +344,7 @@ void truncate_inode_pages_range(struct address_space *mapping, · 255 unmodified lines
347344 if (index >= end)
348345 break;
349346
350− if (radix_tree_exceptional_entry(page))
347+ if (xa_is_value(page))
351348 continue;
352349
353350 if (!trylock_page(page))
@@ -442,7 +439,7 @@ void truncate_inode_pages_range(struct address_space *mapping, · 88 unmodified lines
442439 break;
443440 }
444441
445− if (radix_tree_exceptional_entry(page))
442+ if (xa_is_value(page))
446443 continue;
447444
448445 lock_page(page);
@@ -561,7 +558,7 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping, · 112 unmodified lines
561558 if (index > end)
562559 break;
563560
564− if (radix_tree_exceptional_entry(page)) {
561+ if (xa_is_value(page)) {
565562 invalidate_exceptional_entry(mapping, index,
566563 page);
567564 continue;
@@ -692,7 +689,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping, · 124 unmodified lines
692689 if (index > end)
693690 break;
694691
695− if (radix_tree_exceptional_entry(page)) {
692+ if (xa_is_value(page)) {
696693 if (!invalidate_exceptional_entry2(mapping,
697694 index, page))
698695 ret = -EBUSY;
@@ -738,10 +735,10 @@ int invalidate_inode_pages2_range(struct address_space *mapping, · 39 unmodified lines
738735 index++;
739736 }
740737 /*
741− * For DAX we invalidate page tables after invalidating radix tree. We
738+ * For DAX we invalidate page tables after invalidating page cache. We
742739 * could invalidate page tables while invalidating each entry however
743740 * that would be expensive. And doing range unmapping before doesn't
744− * work as we have no cheap way to find whether radix tree entry didn't
741+ * work as we have no cheap way to find whether page cache entry didn't
745742 * get remapped later.
746743 */
747744 if (dax_mapping(mapping)) {
mm/vmscan.c+5−5
@@ -751,12 +751,12 @@ static inline int is_page_cache_freeable(struct page *page) · 750 unmodified lines
751751 {
752752 /*
753753 * A freeable page cache page is referenced only by the caller
754− * that isolated the page, the page cache radix tree and
755− * optional buffer heads at page->private.
754+ * that isolated the page, the page cache and optional buffer
755+ * heads at page->private.
756756 */
757− int radix_pins = PageTransHuge(page) && PageSwapCache(page) ?
757+ int page_cache_pins = PageTransHuge(page) && PageSwapCache(page) ?
758758 HPAGE_PMD_NR : 1;
759− return page_count(page) - page_has_private(page) == 1 + radix_pins;
759+ return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
760760 }
761761
762762 static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
@@ -932,7 +932,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, · 169 unmodified lines
932932 if (PageSwapCache(page)) {
933933 swp_entry_t swap = { .val = page_private(page) };
934934 mem_cgroup_swapout(page, swap);
935− __delete_from_swap_cache(page);
935+ __delete_from_swap_cache(page, swap);
936936 xa_unlock_irqrestore(&mapping->i_pages, flags);
937937 put_swap_page(page, swap);
938938 } else {
mm/workingset.c+29−39
@@ -160,20 +160,20 @@ · 159 unmodified lines
160160 * and activations is maintained (node->inactive_age).
161161 *
162162 * On eviction, a snapshot of this counter (along with some bits to
163− * identify the node) is stored in the now empty page cache radix tree
163+ * identify the node) is stored in the now empty page cache
164164 * slot of the evicted page. This is called a shadow entry.
165165 *
166166 * On cache misses for which there are shadow entries, an eligible
167167 * refault distance will immediately activate the refaulting page.
168168 */
169169
170−#define EVICTION_SHIFT (RADIX_TREE_EXCEPTIONAL_ENTRY + \
170+#define EVICTION_SHIFT ((BITS_PER_LONG - BITS_PER_XA_VALUE) + \
171171 1 + NODES_SHIFT + MEM_CGROUP_ID_SHIFT)
172172 #define EVICTION_MASK (~0UL >> EVICTION_SHIFT)
173173
174174 /*
175175 * Eviction timestamps need to be able to cover the full range of
176− * actionable refaults. However, bits are tight in the radix tree
176+ * actionable refaults. However, bits are tight in the xarray
177177 * entry, and after storing the identifier for the lruvec there might
178178 * not be enough left to represent every single actionable refault. In
179179 * that case, we have to sacrifice granularity for distance, and group
@@ -185,22 +185,21 @@ static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, · 5 unmodified lines
185185 bool workingset)
186186 {
187187 eviction >>= bucket_order;
188+ eviction &= EVICTION_MASK;
188189 eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
189190 eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
190191 eviction = (eviction << 1) | workingset;
191− eviction = (eviction << RADIX_TREE_EXCEPTIONAL_SHIFT);
192192
193− return (void *)(eviction | RADIX_TREE_EXCEPTIONAL_ENTRY);
193+ return xa_mk_value(eviction);
194194 }
195195
196196 static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
197197 unsigned long *evictionp, bool *workingsetp)
198198 {
199− unsigned long entry = (unsigned long)shadow;
199+ unsigned long entry = xa_to_value(shadow);
200200 int memcgid, nid;
201201 bool workingset;
202202
203− entry >>= RADIX_TREE_EXCEPTIONAL_SHIFT;
204203 workingset = entry & 1;
205204 entry >>= 1;
206205 nid = entry & ((1UL << NODES_SHIFT) - 1);
@@ -367,7 +366,7 @@ out: · 160 unmodified lines
367366
368367 static struct list_lru shadow_nodes;
369368
370−void workingset_update_node(struct radix_tree_node *node)
369+void workingset_update_node(struct xa_node *node)
371370 {
372371 /*
373372 * Track non-empty nodes that contain only shadow entries;
@@ -379,7 +378,7 @@ void workingset_update_node(struct radix_tree_node *node) · 5 unmodified lines
379378 */
380379 VM_WARN_ON_ONCE(!irqs_disabled()); /* For __inc_lruvec_page_state */
381380
382− if (node->count && node->count == node->exceptional) {
381+ if (node->count && node->count == node->nr_values) {
383382 if (list_empty(&node->private_list)) {
384383 list_lru_add(&shadow_nodes, &node->private_list);
385384 __inc_lruvec_page_state(virt_to_page(node),
@@ -404,7 +403,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 18 unmodified lines
404403 nodes = list_lru_shrink_count(&shadow_nodes, sc);
405404
406405 /*
407− * Approximate a reasonable limit for the radix tree nodes
406+ * Approximate a reasonable limit for the nodes
408407 * containing shadow entries. We don't need to keep more
409408 * shadow entries than possible pages on the active list,
410409 * since refault distances bigger than that are dismissed.
@@ -419,11 +418,11 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 8 unmodified lines
419418 * worst-case density of 1/8th. Below that, not all eligible
420419 * refaults can be detected anymore.
421420 *
422− * On 64-bit with 7 radix_tree_nodes per page and 64 slots
421+ * On 64-bit with 7 xa_nodes per page and 64 slots
423422 * each, this will reclaim shadow entries when they consume
424423 * ~1.8% of available memory:
425424 *
426− * PAGE_SIZE / radix_tree_nodes / node_entries * 8 / PAGE_SIZE
425+ * PAGE_SIZE / xa_nodes / node_entries * 8 / PAGE_SIZE
427426 */
428427 #ifdef CONFIG_MEMCG
429428 if (sc->memcg) {
@@ -438,7 +437,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 8 unmodified lines
438437 #endif
439438 pages = node_present_pages(sc->nid);
440439
441− max_nodes = pages >> (RADIX_TREE_MAP_SHIFT - 3);
440+ max_nodes = pages >> (XA_CHUNK_SHIFT - 3);
442441
443442 if (!nodes)
444443 return SHRINK_EMPTY;
@@ -451,11 +450,11 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 6 unmodified lines
451450 static enum lru_status shadow_lru_isolate(struct list_head *item,
452451 struct list_lru_one *lru,
453452 spinlock_t *lru_lock,
454− void *arg)
453+ void *arg) __must_hold(lru_lock)
455454 {
455+ struct xa_node *node = container_of(item, struct xa_node, private_list);
456+ XA_STATE(xas, node->array, 0);
456457 struct address_space *mapping;
457− struct radix_tree_node *node;
458− unsigned int i;
459458 int ret;
460459
461460 /*
@@ -463,15 +462,14 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, · 1 unmodified line
463462 * the shadow node LRU under the i_pages lock and the
464463 * lru_lock. Because the page cache tree is emptied before
465464 * the inode can be destroyed, holding the lru_lock pins any
466− * address_space that has radix tree nodes on the LRU.
465+ * address_space that has nodes on the LRU.
467466 *
468467 * We can then safely transition to the i_pages lock to
469468 * pin only the address_space of the particular node we want
470469 * to reclaim, take the node off-LRU, and drop the lru_lock.
471470 */
472471
473− node = container_of(item, struct radix_tree_node, private_list);
474− mapping = container_of(node->root, struct address_space, i_pages);
472+ mapping = container_of(node->array, struct address_space, i_pages);
475473
476474 /* Coming from the list, invert the lock order */
477475 if (!xa_trylock(&mapping->i_pages)) {
@@ -490,29 +488,21 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, · 12 unmodified lines
490488 * no pages, so we expect to be able to remove them all and
491489 * delete and free the empty node afterwards.
492490 */
493− if (WARN_ON_ONCE(!node->exceptional))
491+ if (WARN_ON_ONCE(!node->nr_values))
494492 goto out_invalid;
495− if (WARN_ON_ONCE(node->count != node->exceptional))
496− goto out_invalid;
497− for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) {
498− if (node->slots[i]) {
499− if (WARN_ON_ONCE(!radix_tree_exceptional_entry(node->slots[i])))
500− goto out_invalid;
501− if (WARN_ON_ONCE(!node->exceptional))
502− goto out_invalid;
503− if (WARN_ON_ONCE(!mapping->nrexceptional))
504− goto out_invalid;
505− node->slots[i] = NULL;
506− node->exceptional--;
507− node->count--;
508− mapping->nrexceptional--;
509− }
510− }
511− if (WARN_ON_ONCE(node->exceptional))
493+ if (WARN_ON_ONCE(node->count != node->nr_values))
512494 goto out_invalid;
495+ mapping->nrexceptional -= node->nr_values;
496+ xas.xa_node = xa_parent_locked(&mapping->i_pages, node);
497+ xas.xa_offset = node->offset;
498+ xas.xa_shift = node->shift + XA_CHUNK_SHIFT;
499+ xas_set_update(&xas, workingset_update_node);
500+ /*
501+ * We could store a shadow entry here which was the minimum of the
502+ * shadow entries we were tracking ...
503+ */
504+ xas_store(&xas, NULL);
513505 __inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM);
514− __radix_tree_delete_node(&mapping->i_pages, node,
515− workingset_lookup_update(mapping));
516506
517507 out_invalid:
518508 xa_unlock_irq(&mapping->i_pages);
tools/include/asm-generic/bitops.h+1−0
@@ -27,5 +27,6 @@ · 26 unmodified lines
2727 #include <asm-generic/bitops/hweight.h>
2828
2929 #include <asm-generic/bitops/atomic.h>
30+#include <asm-generic/bitops/non-atomic.h>
3031
3132 #endif /* __TOOLS_ASM_GENERIC_BITOPS_H */
tools/include/asm-generic/bitops/atomic.h+0−9
@@ -15,13 +15,4 @@ static inline void clear_bit(int nr, unsigned long *addr) · 14 unmodified lines
1515 addr[nr / __BITS_PER_LONG] &= ~(1UL << (nr % __BITS_PER_LONG));
1616 }
1717
18−static __always_inline int test_bit(unsigned int nr, const unsigned long *addr)
19−{
20− return ((1UL << (nr % __BITS_PER_LONG)) &
21− (((unsigned long *)addr)[nr / __BITS_PER_LONG])) != 0;
22−}
23−
24−#define __set_bit(nr, addr) set_bit(nr, addr)
25−#define __clear_bit(nr, addr) clear_bit(nr, addr)
26−
2718 #endif /* _TOOLS_LINUX_ASM_GENERIC_BITOPS_ATOMIC_H_ */
tools/include/asm-generic/bitops/non-atomic.h+109−0
@@ -0,0 +1,109 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
3+#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
4+
5+#include <asm/types.h>
6+
7+/**
8+ * __set_bit - Set a bit in memory
9+ * @nr: the bit to set
10+ * @addr: the address to start counting from
11+ *
12+ * Unlike set_bit(), this function is non-atomic and may be reordered.
13+ * If it's called on the same region of memory simultaneously, the effect
14+ * may be that only one operation succeeds.
15+ */
16+static inline void __set_bit(int nr, volatile unsigned long *addr)
17+{
18+ unsigned long mask = BIT_MASK(nr);
19+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
20+
21+ *p |= mask;
22+}
23+
24+static inline void __clear_bit(int nr, volatile unsigned long *addr)
25+{
26+ unsigned long mask = BIT_MASK(nr);
27+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
28+
29+ *p &= ~mask;
30+}
31+
32+/**
33+ * __change_bit - Toggle a bit in memory
34+ * @nr: the bit to change
35+ * @addr: the address to start counting from
36+ *
37+ * Unlike change_bit(), this function is non-atomic and may be reordered.
38+ * If it's called on the same region of memory simultaneously, the effect
39+ * may be that only one operation succeeds.
40+ */
41+static inline void __change_bit(int nr, volatile unsigned long *addr)
42+{
43+ unsigned long mask = BIT_MASK(nr);
44+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
45+
46+ *p ^= mask;
47+}
48+
49+/**
50+ * __test_and_set_bit - Set a bit and return its old value
51+ * @nr: Bit to set
52+ * @addr: Address to count from
53+ *
54+ * This operation is non-atomic and can be reordered.
55+ * If two examples of this operation race, one can appear to succeed
56+ * but actually fail. You must protect multiple accesses with a lock.
57+ */
58+static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
59+{
60+ unsigned long mask = BIT_MASK(nr);
61+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
62+ unsigned long old = *p;
63+
64+ *p = old | mask;
65+ return (old & mask) != 0;
66+}
67+
68+/**
69+ * __test_and_clear_bit - Clear a bit and return its old value
70+ * @nr: Bit to clear
71+ * @addr: Address to count from
72+ *
73+ * This operation is non-atomic and can be reordered.
74+ * If two examples of this operation race, one can appear to succeed
75+ * but actually fail. You must protect multiple accesses with a lock.
76+ */
77+static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
78+{
79+ unsigned long mask = BIT_MASK(nr);
80+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
81+ unsigned long old = *p;
82+
83+ *p = old & ~mask;
84+ return (old & mask) != 0;
85+}
86+
87+/* WARNING: non atomic and it can be reordered! */
88+static inline int __test_and_change_bit(int nr,
89+ volatile unsigned long *addr)
90+{
91+ unsigned long mask = BIT_MASK(nr);
92+ unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
93+ unsigned long old = *p;
94+
95+ *p = old ^ mask;
96+ return (old & mask) != 0;
97+}
98+
99+/**
100+ * test_bit - Determine whether a bit is set
101+ * @nr: bit number to test
102+ * @addr: Address to start counting from
103+ */
104+static inline int test_bit(int nr, const volatile unsigned long *addr)
105+{
106+ return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
107+}
108+
109+#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */
tools/include/linux/bitmap.h+1−0
@@ -15,6 +15,7 @@ void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, · 14 unmodified lines
1515 const unsigned long *bitmap2, int bits);
1616 int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
1717 const unsigned long *bitmap2, unsigned int bits);
18+void bitmap_clear(unsigned long *map, unsigned int start, int len);
1819
1920 #define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1)))
2021
tools/include/linux/kernel.h+1−0
@@ -70,6 +70,7 @@ · 69 unmodified lines
7070 #define BUG_ON(cond) assert(!(cond))
7171 #endif
7272 #endif
73+#define BUG() BUG_ON(1)
7374
7475 #if __BYTE_ORDER == __BIG_ENDIAN
7576 #define cpu_to_le16 bswap_16
tools/include/linux/spinlock.h+10−2
@@ -8,8 +8,14 @@ · 7 unmodified lines
88 #define spinlock_t pthread_mutex_t
99 #define DEFINE_SPINLOCK(x) pthread_mutex_t x = PTHREAD_MUTEX_INITIALIZER
1010 #define __SPIN_LOCK_UNLOCKED(x) (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER
11−#define spin_lock_init(x) pthread_mutex_init(x, NULL)
12−
11+#define spin_lock_init(x) pthread_mutex_init(x, NULL)
12+
13+#define spin_lock(x) pthread_mutex_lock(x)
14+#define spin_unlock(x) pthread_mutex_unlock(x)
15+#define spin_lock_bh(x) pthread_mutex_lock(x)
16+#define spin_unlock_bh(x) pthread_mutex_unlock(x)
17+#define spin_lock_irq(x) pthread_mutex_lock(x)
18+#define spin_unlock_irq(x) pthread_mutex_unlock(x)
1319 #define spin_lock_irqsave(x, f) (void)f, pthread_mutex_lock(x)
1420 #define spin_unlock_irqrestore(x, f) (void)f, pthread_mutex_unlock(x)
1521
@@ -31,4 +37,6 @@ static inline bool arch_spin_is_locked(arch_spinlock_t *mutex) · 15 unmodified lines
3137 return true;
3238 }
3339
40+#include <linux/lockdep.h>
41+
3442 #endif
tools/testing/radix-tree/.gitignore+1−0
@@ -4,3 +4,4 @@ idr-test · 3 unmodified lines
44 main
55 multiorder
66 radix-tree.c
7+xarray
tools/testing/radix-tree/Makefile+8−3
@@ -4,8 +4,8 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ · 3 unmodified lines
44 -fsanitize=undefined
55 LDFLAGS += -fsanitize=address -fsanitize=undefined
66 LDLIBS+= -lpthread -lurcu
7−TARGETS = main idr-test multiorder
8−CORE_OFILES := radix-tree.o idr.o linux.o test.o find_bit.o
7+TARGETS = main idr-test multiorder xarray
8+CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o
99 OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \
1010 tag_check.o multiorder.o idr-test.o iteration_check.o benchmark.o
1111
@@ -25,6 +25,8 @@ main: $(OFILES) · 13 unmodified lines
2525 idr-test.o: ../../../lib/test_ida.c
2626 idr-test: idr-test.o $(CORE_OFILES)
2727
28+xarray: $(CORE_OFILES)
29+
2830 multiorder: multiorder.o $(CORE_OFILES)
2931
3032 clean:
@@ -35,6 +37,7 @@ vpath %.c ../../lib · 4 unmodified lines
3537 $(OFILES): Makefile *.h */*.h generated/map-shift.h \
3638 ../../include/linux/*.h \
3739 ../../include/asm/*.h \
40+ ../../../include/linux/xarray.h \
3841 ../../../include/linux/radix-tree.h \
3942 ../../../include/linux/idr.h
4043
@@ -44,8 +47,10 @@ radix-tree.c: ../../../lib/radix-tree.c · 3 unmodified lines
4447 idr.c: ../../../lib/idr.c
4548 sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@
4649
50+xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c
51+
4752 generated/map-shift.h:
4853 @if ! grep -qws $(SHIFT) generated/map-shift.h; then \
49− echo "#define RADIX_TREE_MAP_SHIFT $(SHIFT)" > \
54+ echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \
5055 generated/map-shift.h; \
5156 fi
tools/testing/radix-tree/benchmark.c+21−120
@@ -17,9 +17,6 @@ · 16 unmodified lines
1717 #include <time.h>
1818 #include "test.h"
1919
20−#define for_each_index(i, base, order) \
21− for (i = base; i < base + (1 << order); i++)
22−
2320 #define NSEC_PER_SEC 1000000000L
2421
2522 static long long benchmark_iter(struct radix_tree_root *root, bool tagged)
@@ -61,7 +58,7 @@ again: · 35 unmodified lines
6158 }
6259
6360 static void benchmark_insert(struct radix_tree_root *root,
64− unsigned long size, unsigned long step, int order)
61+ unsigned long size, unsigned long step)
6562 {
6663 struct timespec start, finish;
6764 unsigned long index;
@@ -70,19 +67,19 @@ static void benchmark_insert(struct radix_tree_root *root, · 2 unmodified lines
7067 clock_gettime(CLOCK_MONOTONIC, &start);
7168
7269 for (index = 0 ; index < size ; index += step)
73− item_insert_order(root, index, order);
70+ item_insert(root, index);
7471
7572 clock_gettime(CLOCK_MONOTONIC, &finish);
7673
7774 nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
7875 (finish.tv_nsec - start.tv_nsec);
7976
80− printv(2, "Size: %8ld, step: %8ld, order: %d, insertion: %15lld ns\n",
81− size, step, order, nsec);
77+ printv(2, "Size: %8ld, step: %8ld, insertion: %15lld ns\n",
78+ size, step, nsec);
8279 }
8380
8481 static void benchmark_tagging(struct radix_tree_root *root,
85− unsigned long size, unsigned long step, int order)
82+ unsigned long size, unsigned long step)
8683 {
8784 struct timespec start, finish;
8885 unsigned long index;
@@ -98,138 +95,53 @@ static void benchmark_tagging(struct radix_tree_root *root, · 9 unmodified lines
9895 nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
9996 (finish.tv_nsec - start.tv_nsec);
10097
101− printv(2, "Size: %8ld, step: %8ld, order: %d, tagging: %17lld ns\n",
102− size, step, order, nsec);
98+ printv(2, "Size: %8ld, step: %8ld, tagging: %17lld ns\n",
99+ size, step, nsec);
103100 }
104101
105102 static void benchmark_delete(struct radix_tree_root *root,
106− unsigned long size, unsigned long step, int order)
103+ unsigned long size, unsigned long step)
107104 {
108105 struct timespec start, finish;
109− unsigned long index, i;
106+ unsigned long index;
110107 long long nsec;
111108
112109 clock_gettime(CLOCK_MONOTONIC, &start);
113110
114111 for (index = 0 ; index < size ; index += step)
115− for_each_index(i, index, order)
116− item_delete(root, i);
112+ item_delete(root, index);
117113
118114 clock_gettime(CLOCK_MONOTONIC, &finish);
119115
120116 nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
121117 (finish.tv_nsec - start.tv_nsec);
122118
123− printv(2, "Size: %8ld, step: %8ld, order: %d, deletion: %16lld ns\n",
124− size, step, order, nsec);
119+ printv(2, "Size: %8ld, step: %8ld, deletion: %16lld ns\n",
120+ size, step, nsec);
125121 }
126122
127−static void benchmark_size(unsigned long size, unsigned long step, int order)
123+static void benchmark_size(unsigned long size, unsigned long step)
128124 {
129125 RADIX_TREE(tree, GFP_KERNEL);
130126 long long normal, tagged;
131127
132− benchmark_insert(&tree, size, step, order);
133− benchmark_tagging(&tree, size, step, order);
128+ benchmark_insert(&tree, size, step);
129+ benchmark_tagging(&tree, size, step);
134130
135131 tagged = benchmark_iter(&tree, true);
136132 normal = benchmark_iter(&tree, false);
137133
138− printv(2, "Size: %8ld, step: %8ld, order: %d, tagged iteration: %8lld ns\n",
139− size, step, order, tagged);
140− printv(2, "Size: %8ld, step: %8ld, order: %d, normal iteration: %8lld ns\n",
141− size, step, order, normal);
134+ printv(2, "Size: %8ld, step: %8ld, tagged iteration: %8lld ns\n",
135+ size, step, tagged);
136+ printv(2, "Size: %8ld, step: %8ld, normal iteration: %8lld ns\n",
137+ size, step, normal);
142138
143− benchmark_delete(&tree, size, step, order);
139+ benchmark_delete(&tree, size, step);
144140
145141 item_kill_tree(&tree);
146142 rcu_barrier();
147143 }
148144
149−static long long __benchmark_split(unsigned long index,
150− int old_order, int new_order)
151−{
152− struct timespec start, finish;
153− long long nsec;
154− RADIX_TREE(tree, GFP_ATOMIC);
155−
156− item_insert_order(&tree, index, old_order);
157−
158− clock_gettime(CLOCK_MONOTONIC, &start);
159− radix_tree_split(&tree, index, new_order);
160− clock_gettime(CLOCK_MONOTONIC, &finish);
161− nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
162− (finish.tv_nsec - start.tv_nsec);
163−
164− item_kill_tree(&tree);
165−
166− return nsec;
167−
168−}
169−
170−static void benchmark_split(unsigned long size, unsigned long step)
171−{
172− int i, j, idx;
173− long long nsec = 0;
174−
175−
176− for (idx = 0; idx < size; idx += step) {
177− for (i = 3; i < 11; i++) {
178− for (j = 0; j < i; j++) {
179− nsec += __benchmark_split(idx, i, j);
180− }
181− }
182− }
183−
184− printv(2, "Size %8ld, step %8ld, split time %10lld ns\n",
185− size, step, nsec);
186−
187−}
188−
189−static long long __benchmark_join(unsigned long index,
190− unsigned order1, unsigned order2)
191−{
192− unsigned long loc;
193− struct timespec start, finish;
194− long long nsec;
195− void *item, *item2 = item_create(index + 1, order1);
196− RADIX_TREE(tree, GFP_KERNEL);
197−
198− item_insert_order(&tree, index, order2);
199− item = radix_tree_lookup(&tree, index);
200−
201− clock_gettime(CLOCK_MONOTONIC, &start);
202− radix_tree_join(&tree, index + 1, order1, item2);
203− clock_gettime(CLOCK_MONOTONIC, &finish);
204− nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
205− (finish.tv_nsec - start.tv_nsec);
206−
207− loc = find_item(&tree, item);
208− if (loc == -1)
209− free(item);
210−
211− item_kill_tree(&tree);
212−
213− return nsec;
214−}
215−
216−static void benchmark_join(unsigned long step)
217−{
218− int i, j, idx;
219− long long nsec = 0;
220−
221− for (idx = 0; idx < 1 << 10; idx += step) {
222− for (i = 1; i < 15; i++) {
223− for (j = 0; j < i; j++) {
224− nsec += __benchmark_join(idx, i, j);
225− }
226− }
227− }
228−
229− printv(2, "Size %8d, step %8ld, join time %10lld ns\n",
230− 1 << 10, step, nsec);
231−}
232−
233145 void benchmark(void)
234146 {
235147 unsigned long size[] = {1 << 10, 1 << 20, 0};
@@ -242,16 +154,5 @@ void benchmark(void) · 6 unmodified lines
242154
243155 for (c = 0; size[c]; c++)
244156 for (s = 0; step[s]; s++)
245− benchmark_size(size[c], step[s], 0);
246−
247− for (c = 0; size[c]; c++)
248− for (s = 0; step[s]; s++)
249− benchmark_size(size[c], step[s] << 9, 9);
250−
251− for (c = 0; size[c]; c++)
252− for (s = 0; step[s]; s++)
253− benchmark_split(size[c], step[s]);
254−
255− for (s = 0; step[s]; s++)
256− benchmark_join(step[s]);
157+ benchmark_size(size[c], step[s]);
257158 }
tools/testing/radix-tree/bitmap.c+23−0
@@ -0,0 +1,23 @@
1+/* lib/bitmap.c pulls in at least two other files. */
2+
3+#include <linux/bitmap.h>
4+
5+void bitmap_clear(unsigned long *map, unsigned int start, int len)
6+{
7+ unsigned long *p = map + BIT_WORD(start);
8+ const unsigned int size = start + len;
9+ int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
10+ unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
11+
12+ while (len - bits_to_clear >= 0) {
13+ *p &= ~mask_to_clear;
14+ len -= bits_to_clear;
15+ bits_to_clear = BITS_PER_LONG;
16+ mask_to_clear = ~0UL;
17+ p++;
18+ }
19+ if (len) {
20+ mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
21+ *p &= ~mask_to_clear;
22+ }
23+}
tools/testing/radix-tree/generated/autoconf.h+1−1
@@ -1 +1 @@
1−#define CONFIG_RADIX_TREE_MULTIORDER 1
1+#define CONFIG_XARRAY_MULTI 1
tools/testing/radix-tree/idr-test.c+66−5
@@ -19,7 +19,7 @@ · 18 unmodified lines
1919
2020 #include "test.h"
2121
22−#define DUMMY_PTR ((void *)0x12)
22+#define DUMMY_PTR ((void *)0x10)
2323
2424 int item_idr_free(int id, void *p, void *data)
2525 {
@@ -227,6 +227,66 @@ void idr_u32_test(int base) · 201 unmodified lines
227227 idr_u32_test1(&idr, 0xffffffff);
228228 }
229229
230+static void idr_align_test(struct idr *idr)
231+{
232+ char name[] = "Motorola 68000";
233+ int i, id;
234+ void *entry;
235+
236+ for (i = 0; i < 9; i++) {
237+ BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i);
238+ idr_for_each_entry(idr, entry, id);
239+ }
240+ idr_destroy(idr);
241+
242+ for (i = 1; i < 10; i++) {
243+ BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 1);
244+ idr_for_each_entry(idr, entry, id);
245+ }
246+ idr_destroy(idr);
247+
248+ for (i = 2; i < 11; i++) {
249+ BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 2);
250+ idr_for_each_entry(idr, entry, id);
251+ }
252+ idr_destroy(idr);
253+
254+ for (i = 3; i < 12; i++) {
255+ BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 3);
256+ idr_for_each_entry(idr, entry, id);
257+ }
258+ idr_destroy(idr);
259+
260+ for (i = 0; i < 8; i++) {
261+ BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != 0);
262+ BUG_ON(idr_alloc(idr, &name[i + 1], 0, 0, GFP_KERNEL) != 1);
263+ idr_for_each_entry(idr, entry, id);
264+ idr_remove(idr, 1);
265+ idr_for_each_entry(idr, entry, id);
266+ idr_remove(idr, 0);
267+ BUG_ON(!idr_is_empty(idr));
268+ }
269+
270+ for (i = 0; i < 8; i++) {
271+ BUG_ON(idr_alloc(idr, NULL, 0, 0, GFP_KERNEL) != 0);
272+ idr_for_each_entry(idr, entry, id);
273+ idr_replace(idr, &name[i], 0);
274+ idr_for_each_entry(idr, entry, id);
275+ BUG_ON(idr_find(idr, 0) != &name[i]);
276+ idr_remove(idr, 0);
277+ }
278+
279+ for (i = 0; i < 8; i++) {
280+ BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != 0);
281+ BUG_ON(idr_alloc(idr, NULL, 0, 0, GFP_KERNEL) != 1);
282+ idr_remove(idr, 1);
283+ idr_for_each_entry(idr, entry, id);
284+ idr_replace(idr, &name[i + 1], 0);
285+ idr_for_each_entry(idr, entry, id);
286+ idr_remove(idr, 0);
287+ }
288+}
289+
230290 void idr_checks(void)
231291 {
232292 unsigned long i;
@@ -307,6 +367,7 @@ void idr_checks(void) · 74 unmodified lines
307367 idr_u32_test(4);
308368 idr_u32_test(1);
309369 idr_u32_test(0);
370+ idr_align_test(&idr);
310371 }
311372
312373 #define module_init(x)
@@ -344,16 +405,16 @@ void ida_check_conv_user(void) · 31 unmodified lines
344405 DEFINE_IDA(ida);
345406 unsigned long i;
346407
347− radix_tree_cpu_dead(1);
348408 for (i = 0; i < 1000000; i++) {
349409 int id = ida_alloc(&ida, GFP_NOWAIT);
350410 if (id == -ENOMEM) {
351− IDA_BUG_ON(&ida, (i % IDA_BITMAP_BITS) !=
352− BITS_PER_LONG - 2);
411+ IDA_BUG_ON(&ida, ((i % IDA_BITMAP_BITS) !=
412+ BITS_PER_XA_VALUE) &&
413+ ((i % IDA_BITMAP_BITS) != 0));
353414 id = ida_alloc(&ida, GFP_KERNEL);
354415 } else {
355416 IDA_BUG_ON(&ida, (i % IDA_BITMAP_BITS) ==
356− BITS_PER_LONG - 2);
417+ BITS_PER_XA_VALUE);
357418 }
358419 IDA_BUG_ON(&ida, id != i);
359420 }
tools/testing/radix-tree/iteration_check.c+53−56
@@ -1,5 +1,5 @@
11 /*
2− * iteration_check.c: test races having to do with radix tree iteration
2+ * iteration_check.c: test races having to do with xarray iteration
33 * Copyright (c) 2016 Intel Corporation
44 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
55 *
@@ -12,41 +12,54 @@ · 6 unmodified lines
1212 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
1313 * more details.
1414 */
15−#include <linux/radix-tree.h>
1615 #include <pthread.h>
1716 #include "test.h"
1817
1918 #define NUM_THREADS 5
2019 #define MAX_IDX 100
21−#define TAG 0
22−#define NEW_TAG 1
20+#define TAG XA_MARK_0
21+#define NEW_TAG XA_MARK_1
2322
24−static pthread_mutex_t tree_lock = PTHREAD_MUTEX_INITIALIZER;
2523 static pthread_t threads[NUM_THREADS];
2624 static unsigned int seeds[3];
27−static RADIX_TREE(tree, GFP_KERNEL);
25+static DEFINE_XARRAY(array);
2826 static bool test_complete;
2927 static int max_order;
3028
31−/* relentlessly fill the tree with tagged entries */
29+void my_item_insert(struct xarray *xa, unsigned long index)
30+{
31+ XA_STATE(xas, xa, index);
32+ struct item *item = item_create(index, 0);
33+ int order;
34+
35+retry:
36+ xas_lock(&xas);
37+ for (order = max_order; order >= 0; order--) {
38+ xas_set_order(&xas, index, order);
39+ item->order = order;
40+ if (xas_find_conflict(&xas))
41+ continue;
42+ xas_store(&xas, item);
43+ xas_set_mark(&xas, TAG);
44+ break;
45+ }
46+ xas_unlock(&xas);
47+ if (xas_nomem(&xas, GFP_KERNEL))
48+ goto retry;
49+ if (order < 0)
50+ free(item);
51+}
52+
53+/* relentlessly fill the array with tagged entries */
3254 static void *add_entries_fn(void *arg)
3355 {
3456 rcu_register_thread();
3557
3658 while (!test_complete) {
3759 unsigned long pgoff;
38− int order;
3960
4061 for (pgoff = 0; pgoff < MAX_IDX; pgoff++) {
41− pthread_mutex_lock(&tree_lock);
42− for (order = max_order; order >= 0; order--) {
43− if (item_insert_order(&tree, pgoff, order)
44− == 0) {
45− item_tag_set(&tree, pgoff, TAG);
46− break;
47− }
48− }
49− pthread_mutex_unlock(&tree_lock);
62+ my_item_insert(&array, pgoff);
5063 }
5164 }
5265
@@ -56,33 +69,25 @@ static void *add_entries_fn(void *arg) · 3 unmodified lines
5669 }
5770
5871 /*
59− * Iterate over the tagged entries, doing a radix_tree_iter_retry() as we find
60− * things that have been removed and randomly resetting our iteration to the
61− * next chunk with radix_tree_iter_resume(). Both radix_tree_iter_retry() and
62− * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a
63− * NULL 'slot' variable.
72+ * Iterate over tagged entries, retrying when we find ourselves in a deleted
73+ * node and randomly pausing the iteration.
6474 */
6575 static void *tagged_iteration_fn(void *arg)
6676 {
67− struct radix_tree_iter iter;
68− void **slot;
77+ XA_STATE(xas, &array, 0);
78+ void *entry;
6979
7080 rcu_register_thread();
7181
7282 while (!test_complete) {
83+ xas_set(&xas, 0);
7384 rcu_read_lock();
74− radix_tree_for_each_tagged(slot, &tree, &iter, 0, TAG) {
75− void *entry = radix_tree_deref_slot(slot);
76− if (unlikely(!entry))
85+ xas_for_each_marked(&xas, entry, ULONG_MAX, TAG) {
86+ if (xas_retry(&xas, entry))
7787 continue;
7888
79− if (radix_tree_deref_retry(entry)) {
80− slot = radix_tree_iter_retry(&iter);
81− continue;
82− }
83−
8489 if (rand_r(&seeds[0]) % 50 == 0) {
85− slot = radix_tree_iter_resume(slot, &iter);
90+ xas_pause(&xas);
8691 rcu_read_unlock();
8792 rcu_barrier();
8893 rcu_read_lock();
@@ -97,33 +102,25 @@ static void *tagged_iteration_fn(void *arg) · 8 unmodified lines
97102 }
98103
99104 /*
100− * Iterate over the entries, doing a radix_tree_iter_retry() as we find things
101− * that have been removed and randomly resetting our iteration to the next
102− * chunk with radix_tree_iter_resume(). Both radix_tree_iter_retry() and
103− * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a
104− * NULL 'slot' variable.
105+ * Iterate over the entries, retrying when we find ourselves in a deleted
106+ * node and randomly pausing the iteration.
105107 */
106108 static void *untagged_iteration_fn(void *arg)
107109 {
108− struct radix_tree_iter iter;
109− void **slot;
110+ XA_STATE(xas, &array, 0);
111+ void *entry;
110112
111113 rcu_register_thread();
112114
113115 while (!test_complete) {
116+ xas_set(&xas, 0);
114117 rcu_read_lock();
115− radix_tree_for_each_slot(slot, &tree, &iter, 0) {
116− void *entry = radix_tree_deref_slot(slot);
117− if (unlikely(!entry))
118+ xas_for_each(&xas, entry, ULONG_MAX) {
119+ if (xas_retry(&xas, entry))
118120 continue;
119121
120− if (radix_tree_deref_retry(entry)) {
121− slot = radix_tree_iter_retry(&iter);
122− continue;
123− }
124−
125122 if (rand_r(&seeds[1]) % 50 == 0) {
126− slot = radix_tree_iter_resume(slot, &iter);
123+ xas_pause(&xas);
127124 rcu_read_unlock();
128125 rcu_barrier();
129126 rcu_read_lock();
@@ -138,7 +135,7 @@ static void *untagged_iteration_fn(void *arg) · 8 unmodified lines
138135 }
139136
140137 /*
141− * Randomly remove entries to help induce radix_tree_iter_retry() calls in the
138+ * Randomly remove entries to help induce retries in the
142139 * two iteration functions.
143140 */
144141 static void *remove_entries_fn(void *arg)
@@ -147,12 +144,13 @@ static void *remove_entries_fn(void *arg) · 2 unmodified lines
147144
148145 while (!test_complete) {
149146 int pgoff;
147+ struct item *item;
150148
151149 pgoff = rand_r(&seeds[2]) % MAX_IDX;
152150
153− pthread_mutex_lock(&tree_lock);
154− item_delete(&tree, pgoff);
155− pthread_mutex_unlock(&tree_lock);
151+ item = xa_erase(&array, pgoff);
152+ if (item)
153+ item_free(item, pgoff);
156154 }
157155
158156 rcu_unregister_thread();
@@ -165,8 +163,7 @@ static void *tag_entries_fn(void *arg) · 6 unmodified lines
165163 rcu_register_thread();
166164
167165 while (!test_complete) {
168− tag_tagged_items(&tree, &tree_lock, 0, MAX_IDX, 10, TAG,
169− NEW_TAG);
166+ tag_tagged_items(&array, 0, MAX_IDX, 10, TAG, NEW_TAG);
170167 }
171168 rcu_unregister_thread();
172169 return NULL;
@@ -217,5 +214,5 @@ void iteration_test(unsigned order, unsigned test_duration) · 44 unmodified lines
217214 }
218215 }
219216
220− item_kill_tree(&tree);
217+ item_kill_tree(&array);
221218 }
tools/testing/radix-tree/linux/bug.h+1−0
@@ -1 +1,2 @@
1+#include <stdio.h>
12 #include "asm/bug.h"
tools/testing/radix-tree/linux/kconfig.h+1−0
@@ -0,0 +1 @@
1+#include "../../../../include/linux/kconfig.h"
tools/testing/radix-tree/linux/kernel.h+5−0
@@ -14,7 +14,12 @@ · 13 unmodified lines
1414 #include "../../../include/linux/kconfig.h"
1515
1616 #define printk printf
17+#define pr_info printk
1718 #define pr_debug printk
1819 #define pr_cont printk
1920
21+#define __acquires(x)
22+#define __releases(x)
23+#define __must_hold(x)
24+
2025 #endif /* _KERNEL_H */
tools/testing/radix-tree/linux/lockdep.h+11−0
@@ -0,0 +1,11 @@
1+#ifndef _LINUX_LOCKDEP_H
2+#define _LINUX_LOCKDEP_H
3+struct lock_class_key {
4+ unsigned int a;
5+};
6+
7+static inline void lockdep_set_class(spinlock_t *lock,
8+ struct lock_class_key *key)
9+{
10+}
11+#endif /* _LINUX_LOCKDEP_H */
tools/testing/radix-tree/linux/radix-tree.h+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
22 #ifndef _TEST_RADIX_TREE_H
33 #define _TEST_RADIX_TREE_H
44
5−#include "generated/map-shift.h"
65 #include "../../../../include/linux/radix-tree.h"
76
87 extern int kmalloc_verbose;
tools/testing/radix-tree/linux/rcupdate.h+2−0
@@ -6,5 +6,7 @@ · 5 unmodified lines
66
77 #define rcu_dereference_raw(p) rcu_dereference(p)
88 #define rcu_dereference_protected(p, cond) rcu_dereference(p)
9+#define rcu_dereference_check(p, cond) rcu_dereference(p)
10+#define RCU_INIT_POINTER(p, v) (p) = (v)
911
1012 #endif
tools/testing/radix-tree/main.c+3−63
@@ -214,7 +214,7 @@ void copy_tag_check(void) · 213 unmodified lines
214214 }
215215
216216 // printf("\ncopying tags...\n");
217− tagged = tag_tagged_items(&tree, NULL, start, end, ITEMS, 0, 1);
217+ tagged = tag_tagged_items(&tree, start, end, ITEMS, XA_MARK_0, XA_MARK_1);
218218
219219 // printf("checking copied tags\n");
220220 assert(tagged == count);
@@ -223,7 +223,7 @@ void copy_tag_check(void) · 2 unmodified lines
223223 /* Copy tags in several rounds */
224224 // printf("\ncopying tags...\n");
225225 tmp = rand() % (count / 10 + 2);
226− tagged = tag_tagged_items(&tree, NULL, start, end, tmp, 0, 2);
226+ tagged = tag_tagged_items(&tree, start, end, tmp, XA_MARK_0, XA_MARK_2);
227227 assert(tagged == count);
228228
229229 // printf("%lu %lu %lu\n", tagged, tmp, count);
@@ -236,63 +236,6 @@ void copy_tag_check(void) · 6 unmodified lines
236236 item_kill_tree(&tree);
237237 }
238238
239−static void __locate_check(struct radix_tree_root *tree, unsigned long index,
240− unsigned order)
241−{
242− struct item *item;
243− unsigned long index2;
244−
245− item_insert_order(tree, index, order);
246− item = item_lookup(tree, index);
247− index2 = find_item(tree, item);
248− if (index != index2) {
249− printv(2, "index %ld order %d inserted; found %ld\n",
250− index, order, index2);
251− abort();
252− }
253−}
254−
255−static void __order_0_locate_check(void)
256−{
257− RADIX_TREE(tree, GFP_KERNEL);
258− int i;
259−
260− for (i = 0; i < 50; i++)
261− __locate_check(&tree, rand() % INT_MAX, 0);
262−
263− item_kill_tree(&tree);
264−}
265−
266−static void locate_check(void)
267−{
268− RADIX_TREE(tree, GFP_KERNEL);
269− unsigned order;
270− unsigned long offset, index;
271−
272− __order_0_locate_check();
273−
274− for (order = 0; order < 20; order++) {
275− for (offset = 0; offset < (1 << (order + 3));
276− offset += (1UL << order)) {
277− for (index = 0; index < (1UL << (order + 5));
278− index += (1UL << order)) {
279− __locate_check(&tree, index + offset, order);
280− }
281− if (find_item(&tree, &tree) != -1)
282− abort();
283−
284− item_kill_tree(&tree);
285− }
286− }
287−
288− if (find_item(&tree, &tree) != -1)
289− abort();
290− __locate_check(&tree, -1, 0);
291− if (find_item(&tree, &tree) != -1)
292− abort();
293− item_kill_tree(&tree);
294−}
295−
296239 static void single_thread_tests(bool long_run)
297240 {
298241 int i;
@@ -303,10 +246,6 @@ static void single_thread_tests(bool long_run) · 4 unmodified lines
303246 rcu_barrier();
304247 printv(2, "after multiorder_check: %d allocated, preempt %d\n",
305248 nr_allocated, preempt_count);
306− locate_check();
307− rcu_barrier();
308− printv(2, "after locate_check: %d allocated, preempt %d\n",
309− nr_allocated, preempt_count);
310249 tag_check();
311250 rcu_barrier();
312251 printv(2, "after tag_check: %d allocated, preempt %d\n",
@@ -365,6 +304,7 @@ int main(int argc, char **argv) · 52 unmodified lines
365304 rcu_register_thread();
366305 radix_tree_init();
367306
307+ xarray_tests();
368308 regression1_test();
369309 regression2_test();
370310 regression3_test();
tools/testing/radix-tree/multiorder.c+64−545
@@ -20,230 +20,39 @@ · 19 unmodified lines
2020
2121 #include "test.h"
2222
23−#define for_each_index(i, base, order) \
24− for (i = base; i < base + (1 << order); i++)
25−
26−static void __multiorder_tag_test(int index, int order)
27−{
28− RADIX_TREE(tree, GFP_KERNEL);
29− int base, err, i;
30−
31− /* our canonical entry */
32− base = index & ~((1 << order) - 1);
33−
34− printv(2, "Multiorder tag test with index %d, canonical entry %d\n",
35− index, base);
36−
37− err = item_insert_order(&tree, index, order);
38− assert(!err);
39−
40− /*
41− * Verify we get collisions for covered indices. We try and fail to
42− * insert an exceptional entry so we don't leak memory via
43− * item_insert_order().
44− */
45− for_each_index(i, base, order) {
46− err = __radix_tree_insert(&tree, i, order,
47− (void *)(0xA0 | RADIX_TREE_EXCEPTIONAL_ENTRY));
48− assert(err == -EEXIST);
49− }
50−
51− for_each_index(i, base, order) {
52− assert(!radix_tree_tag_get(&tree, i, 0));
53− assert(!radix_tree_tag_get(&tree, i, 1));
54− }
55−
56− assert(radix_tree_tag_set(&tree, index, 0));
57−
58− for_each_index(i, base, order) {
59− assert(radix_tree_tag_get(&tree, i, 0));
60− assert(!radix_tree_tag_get(&tree, i, 1));
61− }
62−
63− assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1);
64− assert(radix_tree_tag_clear(&tree, index, 0));
65−
66− for_each_index(i, base, order) {
67− assert(!radix_tree_tag_get(&tree, i, 0));
68− assert(radix_tree_tag_get(&tree, i, 1));
69− }
70−
71− assert(radix_tree_tag_clear(&tree, index, 1));
72−
73− assert(!radix_tree_tagged(&tree, 0));
74− assert(!radix_tree_tagged(&tree, 1));
75−
76− item_kill_tree(&tree);
77−}
78−
79−static void __multiorder_tag_test2(unsigned order, unsigned long index2)
23+static int item_insert_order(struct xarray *xa, unsigned long index,
24+ unsigned order)
8025 {
81− RADIX_TREE(tree, GFP_KERNEL);
82− unsigned long index = (1 << order);
83− index2 += index;
84−
85− assert(item_insert_order(&tree, 0, order) == 0);
86− assert(item_insert(&tree, index2) == 0);
87−
88− assert(radix_tree_tag_set(&tree, 0, 0));
89− assert(radix_tree_tag_set(&tree, index2, 0));
90−
91− assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 2);
92−
93− item_kill_tree(&tree);
94−}
95−
96−static void multiorder_tag_tests(void)
97−{
98− int i, j;
99−
100− /* test multi-order entry for indices 0-7 with no sibling pointers */
101− __multiorder_tag_test(0, 3);
102− __multiorder_tag_test(5, 3);
103−
104− /* test multi-order entry for indices 8-15 with no sibling pointers */
105− __multiorder_tag_test(8, 3);
106− __multiorder_tag_test(15, 3);
107−
108− /*
109− * Our order 5 entry covers indices 0-31 in a tree with height=2.
110− * This is broken up as follows:
111− * 0-7: canonical entry
112− * 8-15: sibling 1
113− * 16-23: sibling 2
114− * 24-31: sibling 3
115− */
116− __multiorder_tag_test(0, 5);
117− __multiorder_tag_test(29, 5);
118−
119− /* same test, but with indices 32-63 */
120− __multiorder_tag_test(32, 5);
121− __multiorder_tag_test(44, 5);
122−
123− /*
124− * Our order 8 entry covers indices 0-255 in a tree with height=3.
125− * This is broken up as follows:
126− * 0-63: canonical entry
127− * 64-127: sibling 1
128− * 128-191: sibling 2
129− * 192-255: sibling 3
130− */
131− __multiorder_tag_test(0, 8);
132− __multiorder_tag_test(190, 8);
133−
134− /* same test, but with indices 256-511 */
135− __multiorder_tag_test(256, 8);
136− __multiorder_tag_test(300, 8);
137−
138− __multiorder_tag_test(0x12345678UL, 8);
139−
140− for (i = 1; i < 10; i++)
141− for (j = 0; j < (10 << i); j++)
142− __multiorder_tag_test2(i, j);
143−}
144−
145−static void multiorder_check(unsigned long index, int order)
146−{
147− unsigned long i;
148− unsigned long min = index & ~((1UL << order) - 1);
149− unsigned long max = min + (1UL << order);
150− void **slot;
151− struct item *item2 = item_create(min, order);
152− RADIX_TREE(tree, GFP_KERNEL);
153−
154− printv(2, "Multiorder index %ld, order %d\n", index, order);
155−
156− assert(item_insert_order(&tree, index, order) == 0);
157−
158− for (i = min; i < max; i++) {
159− struct item *item = item_lookup(&tree, i);
160− assert(item != 0);
161− assert(item->index == index);
162− }
163− for (i = 0; i < min; i++)
164− item_check_absent(&tree, i);
165− for (i = max; i < 2*max; i++)
166− item_check_absent(&tree, i);
167− for (i = min; i < max; i++)
168− assert(radix_tree_insert(&tree, i, item2) == -EEXIST);
169−
170− slot = radix_tree_lookup_slot(&tree, index);
171− free(*slot);
172− radix_tree_replace_slot(&tree, slot, item2);
173− for (i = min; i < max; i++) {
174− struct item *item = item_lookup(&tree, i);
175− assert(item != 0);
176− assert(item->index == min);
177− }
178−
179− assert(item_delete(&tree, min) != 0);
180−
181− for (i = 0; i < 2*max; i++)
182− item_check_absent(&tree, i);
183−}
184−
185−static void multiorder_shrink(unsigned long index, int order)
186−{
187− unsigned long i;
188− unsigned long max = 1 << order;
189− RADIX_TREE(tree, GFP_KERNEL);
190− struct radix_tree_node *node;
191−
192− printv(2, "Multiorder shrink index %ld, order %d\n", index, order);
26+ XA_STATE_ORDER(xas, xa, index, order);
27+ struct item *item = item_create(index, order);
19328
194− assert(item_insert_order(&tree, 0, order) == 0);
195−
196− node = tree.rnode;
197−
198− assert(item_insert(&tree, index) == 0);
199− assert(node != tree.rnode);
200−
201− assert(item_delete(&tree, index) != 0);
202− assert(node == tree.rnode);
203−
204− for (i = 0; i < max; i++) {
205− struct item *item = item_lookup(&tree, i);
206− assert(item != 0);
207− assert(item->index == 0);
208− }
209− for (i = max; i < 2*max; i++)
210− item_check_absent(&tree, i);
211−
212− if (!item_delete(&tree, 0)) {
213− printv(2, "failed to delete index %ld (order %d)\n", index, order);
214− abort();
215− }
216−
217− for (i = 0; i < 2*max; i++)
218− item_check_absent(&tree, i);
219−}
220−
221−static void multiorder_insert_bug(void)
222−{
223− RADIX_TREE(tree, GFP_KERNEL);
29+ do {
30+ xas_lock(&xas);
31+ xas_store(&xas, item);
32+ xas_unlock(&xas);
33+ } while (xas_nomem(&xas, GFP_KERNEL));
22434
225− item_insert(&tree, 0);
226− radix_tree_tag_set(&tree, 0, 0);
227− item_insert_order(&tree, 3 << 6, 6);
35+ if (!xas_error(&xas))
36+ return 0;
22837
229− item_kill_tree(&tree);
38+ free(item);
39+ return xas_error(&xas);
23040 }
23141
232−void multiorder_iteration(void)
42+void multiorder_iteration(struct xarray *xa)
23343 {
234− RADIX_TREE(tree, GFP_KERNEL);
235− struct radix_tree_iter iter;
236− void **slot;
44+ XA_STATE(xas, xa, 0);
45+ struct item *item;
23746 int i, j, err;
23847
239− printv(1, "Multiorder iteration test\n");
240−
24148 #define NUM_ENTRIES 11
24249 int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128};
24350 int order[NUM_ENTRIES] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7};
24451
52+ printv(1, "Multiorder iteration test\n");
53+
24554 for (i = 0; i < NUM_ENTRIES; i++) {
246− err = item_insert_order(&tree, index[i], order[i]);
55+ err = item_insert_order(xa, index[i], order[i]);
24756 assert(!err);
24857 }
24958
@@ -252,14 +61,14 @@ void multiorder_iteration(void) · 2 unmodified lines
25261 if (j <= (index[i] | ((1 << order[i]) - 1)))
25362 break;
25463
255− radix_tree_for_each_slot(slot, &tree, &iter, j) {
256− int height = order[i] / RADIX_TREE_MAP_SHIFT;
257− int shift = height * RADIX_TREE_MAP_SHIFT;
64+ xas_set(&xas, j);
65+ xas_for_each(&xas, item, ULONG_MAX) {
66+ int height = order[i] / XA_CHUNK_SHIFT;
67+ int shift = height * XA_CHUNK_SHIFT;
25868 unsigned long mask = (1UL << order[i]) - 1;
259− struct item *item = *slot;
26069
261− assert((iter.index | mask) == (index[i] | mask));
262− assert(iter.shift == shift);
70+ assert((xas.xa_index | mask) == (index[i] | mask));
71+ assert(xas.xa_node->shift == shift);
26372 assert(!radix_tree_is_internal_node(item));
26473 assert((item->index | mask) == (index[i] | mask));
26574 assert(item->order == order[i]);
@@ -267,18 +76,15 @@ void multiorder_iteration(void) · 1 unmodified line
26776 }
26877 }
26978
270− item_kill_tree(&tree);
79+ item_kill_tree(xa);
27180 }
27281
273−void multiorder_tagged_iteration(void)
82+void multiorder_tagged_iteration(struct xarray *xa)
27483 {
275− RADIX_TREE(tree, GFP_KERNEL);
276− struct radix_tree_iter iter;
277− void **slot;
84+ XA_STATE(xas, xa, 0);
85+ struct item *item;
27886 int i, j;
27987
280− printv(1, "Multiorder tagged iteration test\n");
281−
28288 #define MT_NUM_ENTRIES 9
28389 int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128};
28490 int order[MT_NUM_ENTRIES] = {1, 0, 2, 4, 3, 1, 3, 0, 7};
@@ -286,13 +92,15 @@ void multiorder_tagged_iteration(void) · 1 unmodified line
28692 #define TAG_ENTRIES 7
28793 int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128};
28894
95+ printv(1, "Multiorder tagged iteration test\n");
96+
28997 for (i = 0; i < MT_NUM_ENTRIES; i++)
290− assert(!item_insert_order(&tree, index[i], order[i]));
98+ assert(!item_insert_order(xa, index[i], order[i]));
29199
292− assert(!radix_tree_tagged(&tree, 1));
100+ assert(!xa_marked(xa, XA_MARK_1));
293101
294102 for (i = 0; i < TAG_ENTRIES; i++)
295− assert(radix_tree_tag_set(&tree, tag_index[i], 1));
103+ xa_set_mark(xa, tag_index[i], XA_MARK_1);
296104
297105 for (j = 0; j < 256; j++) {
298106 int k;
@@ -304,23 +112,23 @@ void multiorder_tagged_iteration(void) · 5 unmodified lines
304112 break;
305113 }
306114
307− radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) {
115+ xas_set(&xas, j);
116+ xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_1) {
308117 unsigned long mask;
309− struct item *item = *slot;
310118 for (k = i; index[k] < tag_index[i]; k++)
311119 ;
312120 mask = (1UL << order[k]) - 1;
313121
314− assert((iter.index | mask) == (tag_index[i] | mask));
315− assert(!radix_tree_is_internal_node(item));
122+ assert((xas.xa_index | mask) == (tag_index[i] | mask));
123+ assert(!xa_is_internal(item));
316124 assert((item->index | mask) == (tag_index[i] | mask));
317125 assert(item->order == order[k]);
318126 i++;
319127 }
320128 }
321129
322− assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) ==
323− TAG_ENTRIES);
130+ assert(tag_tagged_items(xa, 0, ULONG_MAX, TAG_ENTRIES, XA_MARK_1,
131+ XA_MARK_2) == TAG_ENTRIES);
324132
325133 for (j = 0; j < 256; j++) {
326134 int mask, k;
@@ -332,297 +140,31 @@ void multiorder_tagged_iteration(void) · 5 unmodified lines
332140 break;
333141 }
334142
335− radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) {
336− struct item *item = *slot;
143+ xas_set(&xas, j);
144+ xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_2) {
337145 for (k = i; index[k] < tag_index[i]; k++)
338146 ;
339147 mask = (1 << order[k]) - 1;
340148
341− assert((iter.index | mask) == (tag_index[i] | mask));
342− assert(!radix_tree_is_internal_node(item));
149+ assert((xas.xa_index | mask) == (tag_index[i] | mask));
150+ assert(!xa_is_internal(item));
343151 assert((item->index | mask) == (tag_index[i] | mask));
344152 assert(item->order == order[k]);
345153 i++;
346154 }
347155 }
348156
349− assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0)
350− == TAG_ENTRIES);
157+ assert(tag_tagged_items(xa, 1, ULONG_MAX, MT_NUM_ENTRIES * 2, XA_MARK_1,
158+ XA_MARK_0) == TAG_ENTRIES);
351159 i = 0;
352− radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) {
353− assert(iter.index == tag_index[i]);
160+ xas_set(&xas, 0);
161+ xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_0) {
162+ assert(xas.xa_index == tag_index[i]);
354163 i++;
355164 }
165+ assert(i == TAG_ENTRIES);
356166
357− item_kill_tree(&tree);
358−}
359−
360−/*
361− * Basic join checks: make sure we can't find an entry in the tree after
362− * a larger entry has replaced it
363− */
364−static void multiorder_join1(unsigned long index,
365− unsigned order1, unsigned order2)
366−{
367− unsigned long loc;
368− void *item, *item2 = item_create(index + 1, order1);
369− RADIX_TREE(tree, GFP_KERNEL);
370−
371− item_insert_order(&tree, index, order2);
372− item = radix_tree_lookup(&tree, index);
373− radix_tree_join(&tree, index + 1, order1, item2);
374− loc = find_item(&tree, item);
375− if (loc == -1)
376− free(item);
377− item = radix_tree_lookup(&tree, index + 1);
378− assert(item == item2);
379− item_kill_tree(&tree);
380−}
381−
382−/*
383− * Check that the accounting of exceptional entries is handled correctly
384− * by joining an exceptional entry to a normal pointer.
385− */
386−static void multiorder_join2(unsigned order1, unsigned order2)
387−{
388− RADIX_TREE(tree, GFP_KERNEL);
389− struct radix_tree_node *node;
390− void *item1 = item_create(0, order1);
391− void *item2;
392−
393− item_insert_order(&tree, 0, order2);
394− radix_tree_insert(&tree, 1 << order2, (void *)0x12UL);
395− item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL);
396− assert(item2 == (void *)0x12UL);
397− assert(node->exceptional == 1);
398−
399− item2 = radix_tree_lookup(&tree, 0);
400− free(item2);
401−
402− radix_tree_join(&tree, 0, order1, item1);
403− item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL);
404− assert(item2 == item1);
405− assert(node->exceptional == 0);
406− item_kill_tree(&tree);
407−}
408−
409−/*
410− * This test revealed an accounting bug for exceptional entries at one point.
411− * Nodes were being freed back into the pool with an elevated exception count
412− * by radix_tree_join() and then radix_tree_split() was failing to zero the
413− * count of exceptional entries.
414− */
415−static void multiorder_join3(unsigned int order)
416−{
417− RADIX_TREE(tree, GFP_KERNEL);
418− struct radix_tree_node *node;
419− void **slot;
420− struct radix_tree_iter iter;
421− unsigned long i;
422−
423− for (i = 0; i < (1 << order); i++) {
424− radix_tree_insert(&tree, i, (void *)0x12UL);
425− }
426−
427− radix_tree_join(&tree, 0, order, (void *)0x16UL);
428− rcu_barrier();
429−
430− radix_tree_split(&tree, 0, 0);
431−
432− radix_tree_for_each_slot(slot, &tree, &iter, 0) {
433− radix_tree_iter_replace(&tree, &iter, slot, (void *)0x12UL);
434− }
435−
436− __radix_tree_lookup(&tree, 0, &node, NULL);
437− assert(node->exceptional == node->count);
438−
439− item_kill_tree(&tree);
440−}
441−
442−static void multiorder_join(void)
443−{
444− int i, j, idx;
445−
446− for (idx = 0; idx < 1024; idx = idx * 2 + 3) {
447− for (i = 1; i < 15; i++) {
448− for (j = 0; j < i; j++) {
449− multiorder_join1(idx, i, j);
450− }
451− }
452− }
453−
454− for (i = 1; i < 15; i++) {
455− for (j = 0; j < i; j++) {
456− multiorder_join2(i, j);
457− }
458− }
459−
460− for (i = 3; i < 10; i++) {
461− multiorder_join3(i);
462− }
463−}
464−
465−static void check_mem(unsigned old_order, unsigned new_order, unsigned alloc)
466−{
467− struct radix_tree_preload *rtp = &radix_tree_preloads;
468− if (rtp->nr != 0)
469− printv(2, "split(%u %u) remaining %u\n", old_order, new_order,
470− rtp->nr);
471− /*
472− * Can't check for equality here as some nodes may have been
473− * RCU-freed while we ran. But we should never finish with more
474− * nodes allocated since they should have all been preloaded.
475− */
476− if (nr_allocated > alloc)
477− printv(2, "split(%u %u) allocated %u %u\n", old_order, new_order,
478− alloc, nr_allocated);
479−}
480−
481−static void __multiorder_split(int old_order, int new_order)
482−{
483− RADIX_TREE(tree, GFP_ATOMIC);
484− void **slot;
485− struct radix_tree_iter iter;
486− unsigned alloc;
487− struct item *item;
488−
489− radix_tree_preload(GFP_KERNEL);
490− assert(item_insert_order(&tree, 0, old_order) == 0);
491− radix_tree_preload_end();
492−
493− /* Wipe out the preloaded cache or it'll confuse check_mem() */
494− radix_tree_cpu_dead(0);
495−
496− item = radix_tree_tag_set(&tree, 0, 2);
497−
498− radix_tree_split_preload(old_order, new_order, GFP_KERNEL);
499− alloc = nr_allocated;
500− radix_tree_split(&tree, 0, new_order);
501− check_mem(old_order, new_order, alloc);
502− radix_tree_for_each_slot(slot, &tree, &iter, 0) {
503− radix_tree_iter_replace(&tree, &iter, slot,
504− item_create(iter.index, new_order));
505− }
506− radix_tree_preload_end();
507−
508− item_kill_tree(&tree);
509− free(item);
510−}
511−
512−static void __multiorder_split2(int old_order, int new_order)
513−{
514− RADIX_TREE(tree, GFP_KERNEL);
515− void **slot;
516− struct radix_tree_iter iter;
517− struct radix_tree_node *node;
518− void *item;
519−
520− __radix_tree_insert(&tree, 0, old_order, (void *)0x12);
521−
522− item = __radix_tree_lookup(&tree, 0, &node, NULL);
523− assert(item == (void *)0x12);
524− assert(node->exceptional > 0);
525−
526− radix_tree_split(&tree, 0, new_order);
527− radix_tree_for_each_slot(slot, &tree, &iter, 0) {
528− radix_tree_iter_replace(&tree, &iter, slot,
529− item_create(iter.index, new_order));
530− }
531−
532− item = __radix_tree_lookup(&tree, 0, &node, NULL);
533− assert(item != (void *)0x12);
534− assert(node->exceptional == 0);
535−
536− item_kill_tree(&tree);
537−}
538−
539−static void __multiorder_split3(int old_order, int new_order)
540−{
541− RADIX_TREE(tree, GFP_KERNEL);
542− void **slot;
543− struct radix_tree_iter iter;
544− struct radix_tree_node *node;
545− void *item;
546−
547− __radix_tree_insert(&tree, 0, old_order, (void *)0x12);
548−
549− item = __radix_tree_lookup(&tree, 0, &node, NULL);
550− assert(item == (void *)0x12);
551− assert(node->exceptional > 0);
552−
553− radix_tree_split(&tree, 0, new_order);
554− radix_tree_for_each_slot(slot, &tree, &iter, 0) {
555− radix_tree_iter_replace(&tree, &iter, slot, (void *)0x16);
556− }
557−
558− item = __radix_tree_lookup(&tree, 0, &node, NULL);
559− assert(item == (void *)0x16);
560− assert(node->exceptional > 0);
561−
562− item_kill_tree(&tree);
563−
564− __radix_tree_insert(&tree, 0, old_order, (void *)0x12);
565−
566− item = __radix_tree_lookup(&tree, 0, &node, NULL);
567− assert(item == (void *)0x12);
568− assert(node->exceptional > 0);
569−
570− radix_tree_split(&tree, 0, new_order);
571− radix_tree_for_each_slot(slot, &tree, &iter, 0) {
572− if (iter.index == (1 << new_order))
573− radix_tree_iter_replace(&tree, &iter, slot,
574− (void *)0x16);
575− else
576− radix_tree_iter_replace(&tree, &iter, slot, NULL);
577− }
578−
579− item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL);
580− assert(item == (void *)0x16);
581− assert(node->count == node->exceptional);
582− do {
583− node = node->parent;
584− if (!node)
585− break;
586− assert(node->count == 1);
587− assert(node->exceptional == 0);
588− } while (1);
589−
590− item_kill_tree(&tree);
591−}
592−
593−static void multiorder_split(void)
594−{
595− int i, j;
596−
597− for (i = 3; i < 11; i++)
598− for (j = 0; j < i; j++) {
599− __multiorder_split(i, j);
600− __multiorder_split2(i, j);
601− __multiorder_split3(i, j);
602− }
603−}
604−
605−static void multiorder_account(void)
606−{
607− RADIX_TREE(tree, GFP_KERNEL);
608− struct radix_tree_node *node;
609− void **slot;
610−
611− item_insert_order(&tree, 0, 5);
612−
613− __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
614− __radix_tree_lookup(&tree, 0, &node, NULL);
615− assert(node->count == node->exceptional * 2);
616− radix_tree_delete(&tree, 1 << 5);
617− assert(node->exceptional == 0);
618−
619− __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
620− __radix_tree_lookup(&tree, 1 << 5, &node, &slot);
621− assert(node->count == node->exceptional * 2);
622− __radix_tree_replace(&tree, node, slot, NULL, NULL);
623− assert(node->exceptional == 0);
624−
625− item_kill_tree(&tree);
167+ item_kill_tree(xa);
626168 }
627169
628170 bool stop_iteration = false;
@@ -645,68 +187,45 @@ static void *creator_func(void *ptr) · 16 unmodified lines
645187
646188 static void *iterator_func(void *ptr)
647189 {
648− struct radix_tree_root *tree = ptr;
649− struct radix_tree_iter iter;
190+ XA_STATE(xas, ptr, 0);
650191 struct item *item;
651− void **slot;
652192
653193 while (!stop_iteration) {
654194 rcu_read_lock();
655− radix_tree_for_each_slot(slot, tree, &iter, 0) {
656− item = radix_tree_deref_slot(slot);
657−
658− if (!item)
195+ xas_for_each(&xas, item, ULONG_MAX) {
196+ if (xas_retry(&xas, item))
659197 continue;
660− if (radix_tree_deref_retry(item)) {
661− slot = radix_tree_iter_retry(&iter);
662− continue;
663− }
664198
665− item_sanity(item, iter.index);
199+ item_sanity(item, xas.xa_index);
666200 }
667201 rcu_read_unlock();
668202 }
669203 return NULL;
670204 }
671205
672−static void multiorder_iteration_race(void)
206+static void multiorder_iteration_race(struct xarray *xa)
673207 {
674208 const int num_threads = sysconf(_SC_NPROCESSORS_ONLN);
675209 pthread_t worker_thread[num_threads];
676− RADIX_TREE(tree, GFP_KERNEL);
677210 int i;
678211
679− pthread_create(&worker_thread[0], NULL, &creator_func, &tree);
212+ pthread_create(&worker_thread[0], NULL, &creator_func, xa);
680213 for (i = 1; i < num_threads; i++)
681− pthread_create(&worker_thread[i], NULL, &iterator_func, &tree);
214+ pthread_create(&worker_thread[i], NULL, &iterator_func, xa);
682215
683216 for (i = 0; i < num_threads; i++)
684217 pthread_join(worker_thread[i], NULL);
685218
686− item_kill_tree(&tree);
219+ item_kill_tree(xa);
687220 }
688221
222+static DEFINE_XARRAY(array);
223+
689224 void multiorder_checks(void)
690225 {
691− int i;
692−
693− for (i = 0; i < 20; i++) {
694− multiorder_check(200, i);
695− multiorder_check(0, i);
696− multiorder_check((1UL << i) + 1, i);
697− }
698−
699− for (i = 0; i < 15; i++)
700− multiorder_shrink((1UL << (i + RADIX_TREE_MAP_SHIFT)), i);
701−
702− multiorder_insert_bug();
703− multiorder_tag_tests();
704− multiorder_iteration();
705− multiorder_tagged_iteration();
706− multiorder_join();
707− multiorder_split();
708− multiorder_account();
709− multiorder_iteration_race();
226+ multiorder_iteration(&array);
227+ multiorder_tagged_iteration(&array);
228+ multiorder_iteration_race(&array);
710229
711230 radix_tree_cpu_dead(0);
712231 }
tools/testing/radix-tree/regression1.c+27−48
@@ -44,7 +44,6 @@ · 43 unmodified lines
4444 #include "regression.h"
4545
4646 static RADIX_TREE(mt_tree, GFP_KERNEL);
47−static pthread_mutex_t mt_lock = PTHREAD_MUTEX_INITIALIZER;
4847
4948 struct page {
5049 pthread_mutex_t lock;
@@ -53,12 +52,12 @@ struct page { · 2 unmodified lines
5352 unsigned long index;
5453 };
5554
56−static struct page *page_alloc(void)
55+static struct page *page_alloc(int index)
5756 {
5857 struct page *p;
5958 p = malloc(sizeof(struct page));
6059 p->count = 1;
61− p->index = 1;
60+ p->index = index;
6261 pthread_mutex_init(&p->lock, NULL);
6362
6463 return p;
@@ -80,53 +79,33 @@ static void page_free(struct page *p) · 15 unmodified lines
8079 static unsigned find_get_pages(unsigned long start,
8180 unsigned int nr_pages, struct page **pages)
8281 {
83− unsigned int i;
84− unsigned int ret;
85− unsigned int nr_found;
82+ XA_STATE(xas, &mt_tree, start);
83+ struct page *page;
84+ unsigned int ret = 0;
8685
8786 rcu_read_lock();
88−restart:
89− nr_found = radix_tree_gang_lookup_slot(&mt_tree,
90− (void ***)pages, NULL, start, nr_pages);
91− ret = 0;
92− for (i = 0; i < nr_found; i++) {
93− struct page *page;
94−repeat:
95− page = radix_tree_deref_slot((void **)pages[i]);
96− if (unlikely(!page))
87+ xas_for_each(&xas, page, ULONG_MAX) {
88+ if (xas_retry(&xas, page))
9789 continue;
9890
99− if (radix_tree_exception(page)) {
100− if (radix_tree_deref_retry(page)) {
101− /*
102− * Transient condition which can only trigger
103− * when entry at index 0 moves out of or back
104− * to root: none yet gotten, safe to restart.
105− */
106− assert((start | i) == 0);
107− goto restart;
108− }
109− /*
110− * No exceptional entries are inserted in this test.
111− */
112− assert(0);
113− }
114−
11591 pthread_mutex_lock(&page->lock);
116− if (!page->count) {
117− pthread_mutex_unlock(&page->lock);
118− goto repeat;
119− }
92+ if (!page->count)
93+ goto unlock;
94+
12095 /* don't actually update page refcount */
12196 pthread_mutex_unlock(&page->lock);
12297
12398 /* Has the page moved? */
124− if (unlikely(page != *((void **)pages[i]))) {
125− goto repeat;
126− }
99+ if (unlikely(page != xas_reload(&xas)))
100+ goto put_page;
127101
128102 pages[ret] = page;
129103 ret++;
104+ continue;
105+unlock:
106+ pthread_mutex_unlock(&page->lock);
107+put_page:
108+ xas_reset(&xas);
130109 }
131110 rcu_read_unlock();
132111 return ret;
@@ -145,30 +124,30 @@ static void *regression1_fn(void *arg) · 12 unmodified lines
145124 for (j = 0; j < 1000000; j++) {
146125 struct page *p;
147126
148− p = page_alloc();
149− pthread_mutex_lock(&mt_lock);
127+ p = page_alloc(0);
128+ xa_lock(&mt_tree);
150129 radix_tree_insert(&mt_tree, 0, p);
151− pthread_mutex_unlock(&mt_lock);
130+ xa_unlock(&mt_tree);
152131
153− p = page_alloc();
154− pthread_mutex_lock(&mt_lock);
132+ p = page_alloc(1);
133+ xa_lock(&mt_tree);
155134 radix_tree_insert(&mt_tree, 1, p);
156− pthread_mutex_unlock(&mt_lock);
135+ xa_unlock(&mt_tree);
157136
158− pthread_mutex_lock(&mt_lock);
137+ xa_lock(&mt_tree);
159138 p = radix_tree_delete(&mt_tree, 1);
160139 pthread_mutex_lock(&p->lock);
161140 p->count--;
162141 pthread_mutex_unlock(&p->lock);
163− pthread_mutex_unlock(&mt_lock);
142+ xa_unlock(&mt_tree);
164143 page_free(p);
165144
166− pthread_mutex_lock(&mt_lock);
145+ xa_lock(&mt_tree);
167146 p = radix_tree_delete(&mt_tree, 0);
168147 pthread_mutex_lock(&p->lock);
169148 p->count--;
170149 pthread_mutex_unlock(&p->lock);
171− pthread_mutex_unlock(&mt_lock);
150+ xa_unlock(&mt_tree);
172151 page_free(p);
173152 }
174153 } else {
tools/testing/radix-tree/regression2.c+4−4
@@ -53,9 +53,9 @@ · 52 unmodified lines
5353 #include "regression.h"
5454 #include "test.h"
5555
56−#define PAGECACHE_TAG_DIRTY 0
57−#define PAGECACHE_TAG_WRITEBACK 1
58−#define PAGECACHE_TAG_TOWRITE 2
56+#define PAGECACHE_TAG_DIRTY XA_MARK_0
57+#define PAGECACHE_TAG_WRITEBACK XA_MARK_1
58+#define PAGECACHE_TAG_TOWRITE XA_MARK_2
5959
6060 static RADIX_TREE(mt_tree, GFP_KERNEL);
6161 unsigned long page_count = 0;
@@ -92,7 +92,7 @@ void regression2_test(void) · 30 unmodified lines
9292 /* 1. */
9393 start = 0;
9494 end = max_slots - 2;
95− tag_tagged_items(&mt_tree, NULL, start, end, 1,
95+ tag_tagged_items(&mt_tree, start, end, 1,
9696 PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
9797
9898 /* 2. */
tools/testing/radix-tree/regression3.c+0−23
@@ -69,21 +69,6 @@ void regression3_test(void) · 68 unmodified lines
6969 continue;
7070 }
7171 }
72− radix_tree_delete(&root, 1);
73−
74− first = true;
75− radix_tree_for_each_contig(slot, &root, &iter, 0) {
76− printv(2, "contig %ld %p\n", iter.index, *slot);
77− if (first) {
78− radix_tree_insert(&root, 1, ptr);
79− first = false;
80− }
81− if (radix_tree_deref_retry(*slot)) {
82− printv(2, "retry at %ld\n", iter.index);
83− slot = radix_tree_iter_retry(&iter);
84− continue;
85− }
86− }
8772
8873 radix_tree_for_each_slot(slot, &root, &iter, 0) {
8974 printv(2, "slot %ld %p\n", iter.index, *slot);
@@ -93,14 +78,6 @@ void regression3_test(void) · 3 unmodified lines
9378 }
9479 }
9580
96− radix_tree_for_each_contig(slot, &root, &iter, 0) {
97− printv(2, "contig %ld %p\n", iter.index, *slot);
98− if (!iter.index) {
99− printv(2, "next at %ld\n", iter.index);
100− slot = radix_tree_iter_resume(slot, &iter);
101− }
102− }
103−
10481 radix_tree_tag_set(&root, 0, 0);
10582 radix_tree_tag_set(&root, 1, 0);
10683 radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) {
tools/testing/radix-tree/tag_check.c+2−31
@@ -24,7 +24,7 @@ __simple_checks(struct radix_tree_root *tree, unsigned long index, int tag) · 23 unmodified lines
2424 item_tag_set(tree, index, tag);
2525 ret = item_tag_get(tree, index, tag);
2626 assert(ret != 0);
27− ret = tag_tagged_items(tree, NULL, first, ~0UL, 10, tag, !tag);
27+ ret = tag_tagged_items(tree, first, ~0UL, 10, tag, !tag);
2828 assert(ret == 1);
2929 ret = item_tag_get(tree, index, !tag);
3030 assert(ret != 0);
@@ -321,7 +321,7 @@ static void single_check(void) · 290 unmodified lines
321321 assert(ret == 0);
322322 verify_tag_consistency(&tree, 0);
323323 verify_tag_consistency(&tree, 1);
324− ret = tag_tagged_items(&tree, NULL, first, 10, 10, 0, 1);
324+ ret = tag_tagged_items(&tree, first, 10, 10, XA_MARK_0, XA_MARK_1);
325325 assert(ret == 1);
326326 ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 1);
327327 assert(ret == 1);
@@ -331,34 +331,6 @@ static void single_check(void) · 3 unmodified lines
331331 item_kill_tree(&tree);
332332 }
333333
334−void radix_tree_clear_tags_test(void)
335−{
336− unsigned long index;
337− struct radix_tree_node *node;
338− struct radix_tree_iter iter;
339− void **slot;
340−
341− RADIX_TREE(tree, GFP_KERNEL);
342−
343− item_insert(&tree, 0);
344− item_tag_set(&tree, 0, 0);
345− __radix_tree_lookup(&tree, 0, &node, &slot);
346− radix_tree_clear_tags(&tree, node, slot);
347− assert(item_tag_get(&tree, 0, 0) == 0);
348−
349− for (index = 0; index < 1000; index++) {
350− item_insert(&tree, index);
351− item_tag_set(&tree, index, 0);
352− }
353−
354− radix_tree_for_each_slot(slot, &tree, &iter, 0) {
355− radix_tree_clear_tags(&tree, iter.node, slot);
356− assert(item_tag_get(&tree, iter.index, 0) == 0);
357− }
358−
359− item_kill_tree(&tree);
360−}
361−
362334 void tag_check(void)
363335 {
364336 single_check();
@@ -376,5 +348,4 @@ void tag_check(void) · 11 unmodified lines
376348 thrash_tags();
377349 rcu_barrier();
378350 printv(2, "after thrash_tags: %d allocated\n", nr_allocated);
379− radix_tree_clear_tags_test();
380351 }
tools/testing/radix-tree/test.c+42−89
@@ -25,11 +25,6 @@ int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag) · 24 unmodified lines
2525 return radix_tree_tag_get(root, index, tag);
2626 }
2727
28−int __item_insert(struct radix_tree_root *root, struct item *item)
29−{
30− return __radix_tree_insert(root, item->index, item->order, item);
31−}
32−
3328 struct item *item_create(unsigned long index, unsigned int order)
3429 {
3530 struct item *ret = malloc(sizeof(*ret));
@@ -39,21 +34,15 @@ struct item *item_create(unsigned long index, unsigned int order) · 3 unmodified lines
3934 return ret;
4035 }
4136
42−int item_insert_order(struct radix_tree_root *root, unsigned long index,
43− unsigned order)
37+int item_insert(struct radix_tree_root *root, unsigned long index)
4438 {
45− struct item *item = item_create(index, order);
46− int err = __item_insert(root, item);
39+ struct item *item = item_create(index, 0);
40+ int err = radix_tree_insert(root, item->index, item);
4741 if (err)
4842 free(item);
4943 return err;
5044 }
5145
52−int item_insert(struct radix_tree_root *root, unsigned long index)
53−{
54− return item_insert_order(root, index, 0);
55−}
56−
5746 void item_sanity(struct item *item, unsigned long index)
5847 {
5948 unsigned long mask;
@@ -63,16 +52,21 @@ void item_sanity(struct item *item, unsigned long index) · 3 unmodified lines
6352 assert((item->index | mask) == (index | mask));
6453 }
6554
55+void item_free(struct item *item, unsigned long index)
56+{
57+ item_sanity(item, index);
58+ free(item);
59+}
60+
6661 int item_delete(struct radix_tree_root *root, unsigned long index)
6762 {
6863 struct item *item = radix_tree_delete(root, index);
6964
70− if (item) {
71− item_sanity(item, index);
72− free(item);
73− return 1;
74− }
75− return 0;
65+ if (!item)
66+ return 0;
67+
68+ item_free(item, index);
69+ return 1;
7670 }
7771
7872 static void item_free_rcu(struct rcu_head *head)
@@ -82,9 +76,9 @@ static void item_free_rcu(struct rcu_head *head) · 3 unmodified lines
8276 free(item);
8377 }
8478
85−int item_delete_rcu(struct radix_tree_root *root, unsigned long index)
79+int item_delete_rcu(struct xarray *xa, unsigned long index)
8680 {
87− struct item *item = radix_tree_delete(root, index);
81+ struct item *item = xa_erase(xa, index);
8882
8983 if (item) {
9084 item_sanity(item, index);
@@ -176,59 +170,30 @@ void item_full_scan(struct radix_tree_root *root, unsigned long start, · 85 unmodified lines
176170 }
177171
178172 /* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */
179−int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock,
180− unsigned long start, unsigned long end, unsigned batch,
181− unsigned iftag, unsigned thentag)
173+int tag_tagged_items(struct xarray *xa, unsigned long start, unsigned long end,
174+ unsigned batch, xa_mark_t iftag, xa_mark_t thentag)
182175 {
183− unsigned long tagged = 0;
184− struct radix_tree_iter iter;
185− void **slot;
176+ XA_STATE(xas, xa, start);
177+ unsigned int tagged = 0;
178+ struct item *item;
186179
187180 if (batch == 0)
188181 batch = 1;
189182
190− if (lock)
191− pthread_mutex_lock(lock);
192− radix_tree_for_each_tagged(slot, root, &iter, start, iftag) {
193− if (iter.index > end)
194− break;
195− radix_tree_iter_tag_set(root, &iter, thentag);
196− tagged++;
197− if ((tagged % batch) != 0)
183+ xas_lock_irq(&xas);
184+ xas_for_each_marked(&xas, item, end, iftag) {
185+ xas_set_mark(&xas, thentag);
186+ if (++tagged % batch)
198187 continue;
199− slot = radix_tree_iter_resume(slot, &iter);
200− if (lock) {
201− pthread_mutex_unlock(lock);
202− rcu_barrier();
203− pthread_mutex_lock(lock);
204− }
205− }
206− if (lock)
207− pthread_mutex_unlock(lock);
208−
209− return tagged;
210−}
211188
212−/* Use the same pattern as find_swap_entry() in mm/shmem.c */
213−unsigned long find_item(struct radix_tree_root *root, void *item)
214−{
215− struct radix_tree_iter iter;
216− void **slot;
217− unsigned long found = -1;
218− unsigned long checked = 0;
219−
220− radix_tree_for_each_slot(slot, root, &iter, 0) {
221− if (*slot == item) {
222− found = iter.index;
223− break;
224− }
225− checked++;
226− if ((checked % 4) != 0)
227− continue;
228− slot = radix_tree_iter_resume(slot, &iter);
189+ xas_pause(&xas);
190+ xas_unlock_irq(&xas);
191+ rcu_barrier();
192+ xas_lock_irq(&xas);
229193 }
194+ xas_unlock_irq(&xas);
230195
231− return found;
196+ return tagged;
232197 }
233198
234199 static int verify_node(struct radix_tree_node *slot, unsigned int tag,
@@ -281,43 +246,31 @@ static int verify_node(struct radix_tree_node *slot, unsigned int tag, · 46 unmodified lines
281246
282247 void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
283248 {
284− struct radix_tree_node *node = root->rnode;
249+ struct radix_tree_node *node = root->xa_head;
285250 if (!radix_tree_is_internal_node(node))
286251 return;
287252 verify_node(node, tag, !!root_tag_get(root, tag));
288253 }
289254
290−void item_kill_tree(struct radix_tree_root *root)
255+void item_kill_tree(struct xarray *xa)
291256 {
292− struct radix_tree_iter iter;
293− void **slot;
294− struct item *items[32];
295− int nfound;
296−
297− radix_tree_for_each_slot(slot, root, &iter, 0) {
298− if (radix_tree_exceptional_entry(*slot))
299− radix_tree_delete(root, iter.index);
300− }
257+ XA_STATE(xas, xa, 0);
258+ void *entry;
301259
302− while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
303− int i;
304−
305− for (i = 0; i < nfound; i++) {
306− void *ret;
307−
308− ret = radix_tree_delete(root, items[i]->index);
309− assert(ret == items[i]);
310− free(items[i]);
260+ xas_for_each(&xas, entry, ULONG_MAX) {
261+ if (!xa_is_value(entry)) {
262+ item_free(entry, xas.xa_index);
311263 }
264+ xas_store(&xas, NULL);
312265 }
313− assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
314− assert(root->rnode == NULL);
266+
267+ assert(xa_empty(xa));
315268 }
316269
317270 void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
318271 {
319272 unsigned shift;
320− struct radix_tree_node *node = root->rnode;
273+ struct radix_tree_node *node = root->xa_head;
321274 if (!radix_tree_is_internal_node(node)) {
322275 assert(maxindex == 0);
323276 return;
tools/testing/radix-tree/test.h+5−8
@@ -11,13 +11,11 @@ struct item { · 10 unmodified lines
1111 };
1212
1313 struct item *item_create(unsigned long index, unsigned int order);
14−int __item_insert(struct radix_tree_root *root, struct item *item);
1514 int item_insert(struct radix_tree_root *root, unsigned long index);
1615 void item_sanity(struct item *item, unsigned long index);
17−int item_insert_order(struct radix_tree_root *root, unsigned long index,
18− unsigned order);
16+void item_free(struct item *item, unsigned long index);
1917 int item_delete(struct radix_tree_root *root, unsigned long index);
20−int item_delete_rcu(struct radix_tree_root *root, unsigned long index);
18+int item_delete_rcu(struct xarray *xa, unsigned long index);
2119 struct item *item_lookup(struct radix_tree_root *root, unsigned long index);
2220
2321 void item_check_present(struct radix_tree_root *root, unsigned long index);
@@ -29,11 +27,10 @@ void item_full_scan(struct radix_tree_root *root, unsigned long start, · 5 unmodified lines
2927 unsigned long nr, int chunk);
3028 void item_kill_tree(struct radix_tree_root *root);
3129
32−int tag_tagged_items(struct radix_tree_root *, pthread_mutex_t *,
33− unsigned long start, unsigned long end, unsigned batch,
34− unsigned iftag, unsigned thentag);
35−unsigned long find_item(struct radix_tree_root *, void *item);
30+int tag_tagged_items(struct xarray *, unsigned long start, unsigned long end,
31+ unsigned batch, xa_mark_t iftag, xa_mark_t thentag);
3632
33+void xarray_tests(void);
3734 void tag_check(void);
3835 void multiorder_checks(void);
3936 void iteration_test(unsigned order, unsigned duration);
tools/testing/radix-tree/xarray.c+35−0
@@ -0,0 +1,35 @@
1+// SPDX-License-Identifier: GPL-2.0+
2+/*
3+ * xarray.c: Userspace shim for XArray test-suite
4+ * Copyright (c) 2018 Matthew Wilcox <willy@infradead.org>
5+ */
6+
7+#define XA_DEBUG
8+#include "test.h"
9+
10+#define module_init(x)
11+#define module_exit(x)
12+#define MODULE_AUTHOR(x)
13+#define MODULE_LICENSE(x)
14+#define dump_stack() assert(0)
15+
16+#include "../../../lib/xarray.c"
17+#undef XA_DEBUG
18+#include "../../../lib/test_xarray.c"
19+
20+void xarray_tests(void)
21+{
22+ xarray_checks();
23+ xarray_exit();
24+}
25+
26+int __weak main(void)
27+{
28+ radix_tree_init();
29+ xarray_tests();
30+ radix_tree_cpu_dead(1);
31+ rcu_barrier();
32+ if (nr_allocated)
33+ printf("nr_allocated = %d\n", nr_allocated);
34+ return 0;
35+}