History

Merge tag 'xarray-5.1-rc1' of git://git.infradead.org/users/willy/linux-dax

Pull XArray updates from Matthew Wilcox: "This pull request changes the xa_alloc() API. I'm only aware of one subsystem that has started trying to use it, and we agree on the fixup as part of the merge. The xa_insert() error code also changed to match xa_alloc() (EEXIST to EBUSY), and I added xa_alloc_cyclic(). Beyond that, the usual bugfixes, optimisations and tweaking. I now have a git tree with all users of the radix tree and IDR converted over to the XArray that I'll be feeding to maintainers over the next few weeks" * tag 'xarray-5.1-rc1' of git://git.infradead.org/users/willy/linux-dax: XArray: Fix xa_reserve for 2-byte aligned entries XArray: Fix xa_erase of 2-byte aligned entries XArray: Use xa_cmpxchg to implement xa_reserve XArray: Fix xa_release in allocating arrays XArray: Mark xa_insert and xa_reserve as must_check XArray: Add cyclic allocation XArray: Redesign xa_alloc API XArray: Add support for 1s-based allocation XArray: Change xa_insert to return -EBUSY XArray: Update xa_erase family descriptions XArray tests: RCU lock prohibits GFP_KERNEL

Linus Torvalds committed 7 years ago

Commit ea295482 parents f3124cc 4a5c8d87 files · +561 −260Browse files

Unified · Split

7 files
  1. Documentation/core-api/xarray.rst +10 −5
  2. drivers/infiniband/core/device.c +12 −20
  3. drivers/infiniband/core/restrack.c +2 −23
  4. fs/nilfs2/btnode.c +1 −1
  5. include/linux/xarray.h +212 −84
  6. lib/test_xarray.c +232 −56
  7. lib/xarray.c +92 −71
Documentation/core-api/xarray.rst+10−5
@@ -85,7 +85,7 @@ which was at that index; if it returns the same entry which was passed as · 84 unmodified lines
8585
8686 If you want to only store a new entry to an index if the current entry
8787 at that index is ``NULL``, you can use :c:func:`xa_insert` which
88−returns ``-EEXIST`` if the entry is not empty.
88+returns ``-EBUSY`` if the entry is not empty.
8989
9090 You can enquire whether a mark is set on an entry by using
9191 :c:func:`xa_get_mark`. If the entry is not ``NULL``, you can set a mark
@@ -131,17 +131,23 @@ If you use :c:func:`DEFINE_XARRAY_ALLOC` to define the XArray, or · 39 unmodified lines
131131 initialise it by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`,
132132 the XArray changes to track whether entries are in use or not.
133133
134−You can call :c:func:`xa_alloc` to store the entry at any unused index
134+You can call :c:func:`xa_alloc` to store the entry at an unused index
135135 in the XArray. If you need to modify the array from interrupt context,
136136 you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable
137137 interrupts while allocating the ID.
138138
139−Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert`
140−will mark the entry as being allocated. Unlike a normal XArray, storing
139+Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert` will
140+also mark the entry as being allocated. Unlike a normal XArray, storing
141141 ``NULL`` will mark the entry as being in use, like :c:func:`xa_reserve`.
142142 To free an entry, use :c:func:`xa_erase` (or :c:func:`xa_release` if
143143 you only want to free the entry if it's ``NULL``).
144144
145+By default, the lowest free entry is allocated starting from 0. If you
146+want to allocate entries starting at 1, it is more efficient to use
147+:c:func:`DEFINE_XARRAY_ALLOC1` or ``XA_FLAGS_ALLOC1``. If you want to
148+allocate IDs up to a maximum, then wrap back around to the lowest free
149+ID, you can use :c:func:`xa_alloc_cyclic`.
150+
145151 You cannot use ``XA_MARK_0`` with an allocating XArray as this mark
146152 is used to track whether an entry is free or not. The other marks are
147153 available for your use.
@@ -209,7 +215,6 @@ Assumes xa_lock held on entry: · 61 unmodified lines
209215 * :c:func:`__xa_erase`
210216 * :c:func:`__xa_cmpxchg`
211217 * :c:func:`__xa_alloc`
212− * :c:func:`__xa_reserve`
213218 * :c:func:`__xa_set_mark`
214219 * :c:func:`__xa_clear_mark`
215220
drivers/infiniband/core/device.c+12−20
@@ -668,19 +668,10 @@ static int assign_name(struct ib_device *device, const char *name) · 667 unmodified lines
668668 }
669669 strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX);
670670
671− /* Cyclically allocate a user visible ID for the device */
672− device->index = last_id;
673− ret = xa_alloc(&devices, &device->index, INT_MAX, device, GFP_KERNEL);
674− if (ret == -ENOSPC) {
675− device->index = 0;
676− ret = xa_alloc(&devices, &device->index, INT_MAX, device,
677− GFP_KERNEL);
678− }
679− if (ret)
680− goto out;
681− last_id = device->index + 1;
682−
683− ret = 0;
671+ ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b,
672+ &last_id, GFP_KERNEL);
673+ if (ret > 0)
674+ ret = 0;
684675
685676 out:
686677 up_write(&devices_rwsem);
@@ -1059,14 +1050,15 @@ static int assign_client_id(struct ib_client *client) · 372 unmodified lines
10591050 * to get the LIFO order. The extra linked list can go away if xarray
10601051 * learns to reverse iterate.
10611052 */
1062− if (list_empty(&client_list))
1053+ if (list_empty(&client_list)) {
10631054 client->client_id = 0;
1064− else
1065− client->client_id =
1066− list_last_entry(&client_list, struct ib_client, list)
1067− ->client_id;
1068− ret = xa_alloc(&clients, &client->client_id, INT_MAX, client,
1069− GFP_KERNEL);
1055+ } else {
1056+ struct ib_client *last;
1057+
1058+ last = list_last_entry(&client_list, struct ib_client, list);
1059+ client->client_id = last->client_id + 1;
1060+ }
1061+ ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL);
10701062 if (ret)
10711063 goto out;
10721064
drivers/infiniband/core/restrack.c+2−23
@@ -13,28 +13,6 @@ · 12 unmodified lines
1313 #include "cma_priv.h"
1414 #include "restrack.h"
1515
16−static int rt_xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
17− u32 *next)
18−{
19− int err;
20−
21− *id = *next;
22− if (*next == U32_MAX)
23− *id = 0;
24−
25− xa_lock(xa);
26− err = __xa_alloc(xa, id, U32_MAX, entry, GFP_KERNEL);
27− if (err && *next != U32_MAX) {
28− *id = 0;
29− err = __xa_alloc(xa, id, *next, entry, GFP_KERNEL);
30− }
31−
32− if (!err)
33− *next = *id + 1;
34− xa_unlock(xa);
35− return err;
36−}
37−
3816 /**
3917 * rdma_restrack_init() - initialize and allocate resource tracking
4018 * @dev: IB device
@@ -226,7 +204,8 @@ static void rdma_restrack_add(struct rdma_restrack_entry *res) · 185 unmodified lines
226204 kref_init(&res->kref);
227205 init_completion(&res->comp);
228206 if (res->type != RDMA_RESTRACK_QP)
229− ret = rt_xa_alloc_cyclic(&rt->xa, &res->id, res, &rt->next_id);
207+ ret = xa_alloc_cyclic(&rt->xa, &res->id, res, xa_limit_32b,
208+ &rt->next_id, GFP_KERNEL);
230209 else {
231210 /* Special case to ensure that LQPN points to right QP */
232211 struct ib_qp *qp = container_of(res, struct ib_qp, res);
fs/nilfs2/btnode.c+1−1
@@ -189,7 +189,7 @@ retry: · 188 unmodified lines
189189 */
190190 if (!err)
191191 return 0;
192− else if (err != -EEXIST)
192+ else if (err != -EBUSY)
193193 goto failed_unlock;
194194
195195 err = invalidate_inode_pages2_range(btnc, newkey, newkey);
include/linux/xarray.h+212−84
@@ -131,6 +131,12 @@ static inline unsigned int xa_pointer_tag(void *entry) · 130 unmodified lines
131131 * xa_mk_internal() - Create an internal entry.
132132 * @v: Value to turn into an internal entry.
133133 *
134+ * Internal entries are used for a number of purposes. Entries 0-255 are
135+ * used for sibling entries (only 0-62 are used by the current code). 256
136+ * is used for the retry entry. 257 is used for the reserved / zero entry.
137+ * Negative internal entries are used to represent errnos. Node pointers
138+ * are also tagged as internal entries in some situations.
139+ *
134140 * Context: Any context.
135141 * Return: An XArray internal entry corresponding to this value.
136142 */
@@ -163,6 +169,22 @@ static inline bool xa_is_internal(const void *entry) · 26 unmodified lines
163169 return ((unsigned long)entry & 3) == 2;
164170 }
165171
172+#define XA_ZERO_ENTRY xa_mk_internal(257)
173+
174+/**
175+ * xa_is_zero() - Is the entry a zero entry?
176+ * @entry: Entry retrieved from the XArray
177+ *
178+ * The normal API will return NULL as the contents of a slot containing
179+ * a zero entry. You can only see zero entries by using the advanced API.
180+ *
181+ * Return: %true if the entry is a zero entry.
182+ */
183+static inline bool xa_is_zero(const void *entry)
184+{
185+ return unlikely(entry == XA_ZERO_ENTRY);
186+}
187+
166188 /**
167189 * xa_is_err() - Report whether an XArray operation returned an error
168190 * @entry: Result from calling an XArray function
@@ -200,6 +222,27 @@ static inline int xa_err(void *entry) · 31 unmodified lines
200222 return 0;
201223 }
202224
225+/**
226+ * struct xa_limit - Represents a range of IDs.
227+ * @min: The lowest ID to allocate (inclusive).
228+ * @max: The maximum ID to allocate (inclusive).
229+ *
230+ * This structure is used either directly or via the XA_LIMIT() macro
231+ * to communicate the range of IDs that are valid for allocation.
232+ * Two common ranges are predefined for you:
233+ * * xa_limit_32b - [0 - UINT_MAX]
234+ * * xa_limit_31b - [0 - INT_MAX]
235+ */
236+struct xa_limit {
237+ u32 max;
238+ u32 min;
239+};
240+
241+#define XA_LIMIT(_min, _max) (struct xa_limit) { .min = _min, .max = _max }
242+
243+#define xa_limit_32b XA_LIMIT(0, UINT_MAX)
244+#define xa_limit_31b XA_LIMIT(0, INT_MAX)
245+
203246 typedef unsigned __bitwise xa_mark_t;
204247 #define XA_MARK_0 ((__force xa_mark_t)0U)
205248 #define XA_MARK_1 ((__force xa_mark_t)1U)
@@ -220,10 +263,14 @@ enum xa_lock_type { · 14 unmodified lines
220263 #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ)
221264 #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH)
222265 #define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U)
266+#define XA_FLAGS_ZERO_BUSY ((__force gfp_t)8U)
267+#define XA_FLAGS_ALLOC_WRAPPED ((__force gfp_t)16U)
223268 #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \
224269 (__force unsigned)(mark)))
225270
271+/* ALLOC is for a normal 0-based alloc. ALLOC1 is for an 1-based alloc */
226272 #define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK))
273+#define XA_FLAGS_ALLOC1 (XA_FLAGS_TRACK_FREE | XA_FLAGS_ZERO_BUSY)
227274
228275 /**
229276 * struct xarray - The anchor of the XArray.
@@ -279,7 +326,7 @@ struct xarray { · 49 unmodified lines
279326 #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0)
280327
281328 /**
282− * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs.
329+ * DEFINE_XARRAY_ALLOC() - Define an XArray which allocates IDs starting at 0.
283330 * @name: A string that names your XArray.
284331 *
285332 * This is intended for file scope definitions of allocating XArrays.
@@ -287,6 +334,15 @@ struct xarray { · 1 unmodified line
287334 */
288335 #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC)
289336
337+/**
338+ * DEFINE_XARRAY_ALLOC1() - Define an XArray which allocates IDs starting at 1.
339+ * @name: A string that names your XArray.
340+ *
341+ * This is intended for file scope definitions of allocating XArrays.
342+ * See also DEFINE_XARRAY().
343+ */
344+#define DEFINE_XARRAY_ALLOC1(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC1)
345+
290346 void *xa_load(struct xarray *, unsigned long index);
291347 void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
292348 void *xa_erase(struct xarray *, unsigned long index);
@@ -463,9 +519,12 @@ void *__xa_erase(struct xarray *, unsigned long index); · 170 unmodified lines
463519 void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
464520 void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old,
465521 void *entry, gfp_t);
466−int __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t);
467−int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t);
468−int __xa_reserve(struct xarray *, unsigned long index, gfp_t);
522+int __must_check __xa_insert(struct xarray *, unsigned long index,
523+ void *entry, gfp_t);
524+int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry,
525+ struct xa_limit, gfp_t);
526+int __must_check __xa_alloc_cyclic(struct xarray *, u32 *id, void *entry,
527+ struct xa_limit, u32 *next, gfp_t);
469528 void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
470529 void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
471530
@@ -526,9 +585,9 @@ static inline void *xa_store_irq(struct xarray *xa, unsigned long index, · 54 unmodified lines
526585 * @xa: XArray.
527586 * @index: Index of entry.
528587 *
529− * This function is the equivalent of calling xa_store() with %NULL as
530− * the third argument. The XArray does not need to allocate memory, so
531− * the user does not need to provide GFP flags.
588+ * After this function returns, loading from @index will return %NULL.
589+ * If the index is part of a multi-index entry, all indices will be erased
590+ * and none of the entries will be part of a multi-index entry.
532591 *
533592 * Context: Any context. Takes and releases the xa_lock while
534593 * disabling softirqs.
@@ -550,9 +609,9 @@ static inline void *xa_erase_bh(struct xarray *xa, unsigned long index) · 15 unmodified lines
550609 * @xa: XArray.
551610 * @index: Index of entry.
552611 *
553− * This function is the equivalent of calling xa_store() with %NULL as
554− * the third argument. The XArray does not need to allocate memory, so
555− * the user does not need to provide GFP flags.
612+ * After this function returns, loading from @index will return %NULL.
613+ * If the index is part of a multi-index entry, all indices will be erased
614+ * and none of the entries will be part of a multi-index entry.
556615 *
557616 * Context: Process context. Takes and releases the xa_lock while
558617 * disabling interrupts.
@@ -664,11 +723,11 @@ static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index, · 105 unmodified lines
664723 *
665724 * Context: Any context. Takes and releases the xa_lock. May sleep if
666725 * the @gfp flags permit.
667− * Return: 0 if the store succeeded. -EEXIST if another entry was present.
726+ * Return: 0 if the store succeeded. -EBUSY if another entry was present.
668727 * -ENOMEM if memory could not be allocated.
669728 */
670−static inline int xa_insert(struct xarray *xa, unsigned long index,
671− void *entry, gfp_t gfp)
729+static inline int __must_check xa_insert(struct xarray *xa,
730+ unsigned long index, void *entry, gfp_t gfp)
672731 {
673732 int err;
674733
@@ -693,11 +752,11 @@ static inline int xa_insert(struct xarray *xa, unsigned long index, · 18 unmodified lines
693752 *
694753 * Context: Any context. Takes and releases the xa_lock while
695754 * disabling softirqs. May sleep if the @gfp flags permit.
696− * Return: 0 if the store succeeded. -EEXIST if another entry was present.
755+ * Return: 0 if the store succeeded. -EBUSY if another entry was present.
697756 * -ENOMEM if memory could not be allocated.
698757 */
699−static inline int xa_insert_bh(struct xarray *xa, unsigned long index,
700− void *entry, gfp_t gfp)
758+static inline int __must_check xa_insert_bh(struct xarray *xa,
759+ unsigned long index, void *entry, gfp_t gfp)
701760 {
702761 int err;
703762
@@ -722,11 +781,11 @@ static inline int xa_insert_bh(struct xarray *xa, unsigned long index, · 18 unmodified lines
722781 *
723782 * Context: Process context. Takes and releases the xa_lock while
724783 * disabling interrupts. May sleep if the @gfp flags permit.
725− * Return: 0 if the store succeeded. -EEXIST if another entry was present.
784+ * Return: 0 if the store succeeded. -EBUSY if another entry was present.
726785 * -ENOMEM if memory could not be allocated.
727786 */
728−static inline int xa_insert_irq(struct xarray *xa, unsigned long index,
729− void *entry, gfp_t gfp)
787+static inline int __must_check xa_insert_irq(struct xarray *xa,
788+ unsigned long index, void *entry, gfp_t gfp)
730789 {
731790 int err;
732791
@@ -741,26 +800,26 @@ static inline int xa_insert_irq(struct xarray *xa, unsigned long index, · 8 unmodified lines
741800 * xa_alloc() - Find somewhere to store this entry in the XArray.
742801 * @xa: XArray.
743802 * @id: Pointer to ID.
744− * @max: Maximum ID to allocate (inclusive).
745803 * @entry: New entry.
804+ * @limit: Range of ID to allocate.
746805 * @gfp: Memory allocation flags.
747806 *
748− * Allocates an unused ID in the range specified by @id and @max.
749− * Updates the @id pointer with the index, then stores the entry at that
750− * index. A concurrent lookup will not see an uninitialised @id.
807+ * Finds an empty entry in @xa between @limit.min and @limit.max,
808+ * stores the index into the @id pointer, then stores the entry at
809+ * that index. A concurrent lookup will not see an uninitialised @id.
751810 *
752− * Context: Process context. Takes and releases the xa_lock. May sleep if
811+ * Context: Any context. Takes and releases the xa_lock. May sleep if
753812 * the @gfp flags permit.
754− * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
755− * there is no more space in the XArray.
813+ * Return: 0 on success, -ENOMEM if memory could not be allocated or
814+ * -EBUSY if there are no free entries in @limit.
756815 */
757−static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry,
758− gfp_t gfp)
816+static inline __must_check int xa_alloc(struct xarray *xa, u32 *id,
817+ void *entry, struct xa_limit limit, gfp_t gfp)
759818 {
760819 int err;
761820
762821 xa_lock(xa);
763− err = __xa_alloc(xa, id, max, entry, gfp);
822+ err = __xa_alloc(xa, id, entry, limit, gfp);
764823 xa_unlock(xa);
765824
766825 return err;
@@ -770,26 +829,26 @@ static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines
770829 * xa_alloc_bh() - Find somewhere to store this entry in the XArray.
771830 * @xa: XArray.
772831 * @id: Pointer to ID.
773− * @max: Maximum ID to allocate (inclusive).
774832 * @entry: New entry.
833+ * @limit: Range of ID to allocate.
775834 * @gfp: Memory allocation flags.
776835 *
777− * Allocates an unused ID in the range specified by @id and @max.
778− * Updates the @id pointer with the index, then stores the entry at that
779− * index. A concurrent lookup will not see an uninitialised @id.
836+ * Finds an empty entry in @xa between @limit.min and @limit.max,
837+ * stores the index into the @id pointer, then stores the entry at
838+ * that index. A concurrent lookup will not see an uninitialised @id.
780839 *
781840 * Context: Any context. Takes and releases the xa_lock while
782841 * disabling softirqs. May sleep if the @gfp flags permit.
783− * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
784− * there is no more space in the XArray.
842+ * Return: 0 on success, -ENOMEM if memory could not be allocated or
843+ * -EBUSY if there are no free entries in @limit.
785844 */
786−static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry,
787− gfp_t gfp)
845+static inline int __must_check xa_alloc_bh(struct xarray *xa, u32 *id,
846+ void *entry, struct xa_limit limit, gfp_t gfp)
788847 {
789848 int err;
790849
791850 xa_lock_bh(xa);
792− err = __xa_alloc(xa, id, max, entry, gfp);
851+ err = __xa_alloc(xa, id, entry, limit, gfp);
793852 xa_unlock_bh(xa);
794853
795854 return err;
@@ -799,26 +858,125 @@ static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines
799858 * xa_alloc_irq() - Find somewhere to store this entry in the XArray.
800859 * @xa: XArray.
801860 * @id: Pointer to ID.
802− * @max: Maximum ID to allocate (inclusive).
803861 * @entry: New entry.
862+ * @limit: Range of ID to allocate.
804863 * @gfp: Memory allocation flags.
805864 *
806− * Allocates an unused ID in the range specified by @id and @max.
807− * Updates the @id pointer with the index, then stores the entry at that
808− * index. A concurrent lookup will not see an uninitialised @id.
865+ * Finds an empty entry in @xa between @limit.min and @limit.max,
866+ * stores the index into the @id pointer, then stores the entry at
867+ * that index. A concurrent lookup will not see an uninitialised @id.
809868 *
810869 * Context: Process context. Takes and releases the xa_lock while
811870 * disabling interrupts. May sleep if the @gfp flags permit.
812− * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
813− * there is no more space in the XArray.
871+ * Return: 0 on success, -ENOMEM if memory could not be allocated or
872+ * -EBUSY if there are no free entries in @limit.
814873 */
815−static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry,
816− gfp_t gfp)
874+static inline int __must_check xa_alloc_irq(struct xarray *xa, u32 *id,
875+ void *entry, struct xa_limit limit, gfp_t gfp)
817876 {
818877 int err;
819878
820879 xa_lock_irq(xa);
821− err = __xa_alloc(xa, id, max, entry, gfp);
880+ err = __xa_alloc(xa, id, entry, limit, gfp);
881+ xa_unlock_irq(xa);
882+
883+ return err;
884+}
885+
886+/**
887+ * xa_alloc_cyclic() - Find somewhere to store this entry in the XArray.
888+ * @xa: XArray.
889+ * @id: Pointer to ID.
890+ * @entry: New entry.
891+ * @limit: Range of allocated ID.
892+ * @next: Pointer to next ID to allocate.
893+ * @gfp: Memory allocation flags.
894+ *
895+ * Finds an empty entry in @xa between @limit.min and @limit.max,
896+ * stores the index into the @id pointer, then stores the entry at
897+ * that index. A concurrent lookup will not see an uninitialised @id.
898+ * The search for an empty entry will start at @next and will wrap
899+ * around if necessary.
900+ *
901+ * Context: Any context. Takes and releases the xa_lock. May sleep if
902+ * the @gfp flags permit.
903+ * Return: 0 if the allocation succeeded without wrapping. 1 if the
904+ * allocation succeeded after wrapping, -ENOMEM if memory could not be
905+ * allocated or -EBUSY if there are no free entries in @limit.
906+ */
907+static inline int xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
908+ struct xa_limit limit, u32 *next, gfp_t gfp)
909+{
910+ int err;
911+
912+ xa_lock(xa);
913+ err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
914+ xa_unlock(xa);
915+
916+ return err;
917+}
918+
919+/**
920+ * xa_alloc_cyclic_bh() - Find somewhere to store this entry in the XArray.
921+ * @xa: XArray.
922+ * @id: Pointer to ID.
923+ * @entry: New entry.
924+ * @limit: Range of allocated ID.
925+ * @next: Pointer to next ID to allocate.
926+ * @gfp: Memory allocation flags.
927+ *
928+ * Finds an empty entry in @xa between @limit.min and @limit.max,
929+ * stores the index into the @id pointer, then stores the entry at
930+ * that index. A concurrent lookup will not see an uninitialised @id.
931+ * The search for an empty entry will start at @next and will wrap
932+ * around if necessary.
933+ *
934+ * Context: Any context. Takes and releases the xa_lock while
935+ * disabling softirqs. May sleep if the @gfp flags permit.
936+ * Return: 0 if the allocation succeeded without wrapping. 1 if the
937+ * allocation succeeded after wrapping, -ENOMEM if memory could not be
938+ * allocated or -EBUSY if there are no free entries in @limit.
939+ */
940+static inline int xa_alloc_cyclic_bh(struct xarray *xa, u32 *id, void *entry,
941+ struct xa_limit limit, u32 *next, gfp_t gfp)
942+{
943+ int err;
944+
945+ xa_lock_bh(xa);
946+ err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
947+ xa_unlock_bh(xa);
948+
949+ return err;
950+}
951+
952+/**
953+ * xa_alloc_cyclic_irq() - Find somewhere to store this entry in the XArray.
954+ * @xa: XArray.
955+ * @id: Pointer to ID.
956+ * @entry: New entry.
957+ * @limit: Range of allocated ID.
958+ * @next: Pointer to next ID to allocate.
959+ * @gfp: Memory allocation flags.
960+ *
961+ * Finds an empty entry in @xa between @limit.min and @limit.max,
962+ * stores the index into the @id pointer, then stores the entry at
963+ * that index. A concurrent lookup will not see an uninitialised @id.
964+ * The search for an empty entry will start at @next and will wrap
965+ * around if necessary.
966+ *
967+ * Context: Process context. Takes and releases the xa_lock while
968+ * disabling interrupts. May sleep if the @gfp flags permit.
969+ * Return: 0 if the allocation succeeded without wrapping. 1 if the
970+ * allocation succeeded after wrapping, -ENOMEM if memory could not be
971+ * allocated or -EBUSY if there are no free entries in @limit.
972+ */
973+static inline int xa_alloc_cyclic_irq(struct xarray *xa, u32 *id, void *entry,
974+ struct xa_limit limit, u32 *next, gfp_t gfp)
975+{
976+ int err;
977+
978+ xa_lock_irq(xa);
979+ err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
822980 xa_unlock_irq(xa);
823981
824982 return err;
@@ -842,16 +1000,10 @@ static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, · 17 unmodified lines
8421000 * May sleep if the @gfp flags permit.
8431001 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
8441002 */
845−static inline
1003+static inline __must_check
8461004 int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)
8471005 {
848− int ret;
849−
850− xa_lock(xa);
851− ret = __xa_reserve(xa, index, gfp);
852− xa_unlock(xa);
853−
854− return ret;
1006+ return xa_err(xa_cmpxchg(xa, index, NULL, XA_ZERO_ENTRY, gfp));
8551007 }
8561008
8571009 /**
@@ -866,16 +1018,10 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines
8661018 * disabling softirqs.
8671019 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
8681020 */
869−static inline
1021+static inline __must_check
8701022 int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp)
8711023 {
872− int ret;
873−
874− xa_lock_bh(xa);
875− ret = __xa_reserve(xa, index, gfp);
876− xa_unlock_bh(xa);
877−
878− return ret;
1024+ return xa_err(xa_cmpxchg_bh(xa, index, NULL, XA_ZERO_ENTRY, gfp));
8791025 }
8801026
8811027 /**
@@ -890,16 +1036,10 @@ int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines
8901036 * disabling interrupts.
8911037 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
8921038 */
893−static inline
1039+static inline __must_check
8941040 int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp)
8951041 {
896− int ret;
897−
898− xa_lock_irq(xa);
899− ret = __xa_reserve(xa, index, gfp);
900− xa_unlock_irq(xa);
901−
902− return ret;
1042+ return xa_err(xa_cmpxchg_irq(xa, index, NULL, XA_ZERO_ENTRY, gfp));
9031043 }
9041044
9051045 /**
@@ -913,7 +1053,7 @@ int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) · 7 unmodified lines
9131053 */
9141054 static inline void xa_release(struct xarray *xa, unsigned long index)
9151055 {
916− xa_cmpxchg(xa, index, NULL, NULL, 0);
1056+ xa_cmpxchg(xa, index, XA_ZERO_ENTRY, NULL, 0);
9171057 }
9181058
9191059 /* Everything below here is the Advanced API. Proceed with caution. */
@@ -1073,18 +1213,6 @@ static inline bool xa_is_sibling(const void *entry) · 153 unmodified lines
10731213 }
10741214
10751215 #define XA_RETRY_ENTRY xa_mk_internal(256)
1076−#define XA_ZERO_ENTRY xa_mk_internal(257)
1077−
1078−/**
1079− * xa_is_zero() - Is the entry a zero entry?
1080− * @entry: Entry retrieved from the XArray
1081− *
1082− * Return: %true if the entry is a zero entry.
1083− */
1084−static inline bool xa_is_zero(const void *entry)
1085−{
1086− return unlikely(entry == XA_ZERO_ENTRY);
1087−}
10881216
10891217 /**
10901218 * xa_is_retry() - Is the entry a retry entry?
lib/test_xarray.c+232−56
@@ -40,9 +40,9 @@ static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp) · 39 unmodified lines
4040
4141 static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp)
4242 {
43− u32 id = 0;
43+ u32 id;
4444
45− XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, xa_mk_index(index),
45+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(index), xa_limit_32b,
4646 gfp) != 0);
4747 XA_BUG_ON(xa, id != index);
4848 }
@@ -107,8 +107,11 @@ static noinline void check_xas_retry(struct xarray *xa) · 58 unmodified lines
107107 XA_BUG_ON(xa, xas.xa_node != XAS_RESTART);
108108 XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
109109 XA_BUG_ON(xa, xas.xa_node != NULL);
110+ rcu_read_unlock();
110111
111112 XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);
113+
114+ rcu_read_lock();
112115 XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas)));
113116 xas.xa_node = XAS_RESTART;
114117 XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
@@ -343,7 +346,7 @@ static noinline void check_cmpxchg(struct xarray *xa) · 228 unmodified lines
343346
344347 XA_BUG_ON(xa, !xa_empty(xa));
345348 XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL);
346− XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST);
349+ XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EBUSY);
347350 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS);
348351 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS);
349352 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE);
@@ -358,46 +361,65 @@ static noinline void check_reserve(struct xarray *xa) · 8 unmodified lines
358361 {
359362 void *entry;
360363 unsigned long index;
364+ int count;
361365
362366 /* An array with a reserved entry is not empty */
363367 XA_BUG_ON(xa, !xa_empty(xa));
364− xa_reserve(xa, 12345678, GFP_KERNEL);
368+ XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
365369 XA_BUG_ON(xa, xa_empty(xa));
366370 XA_BUG_ON(xa, xa_load(xa, 12345678));
367371 xa_release(xa, 12345678);
368372 XA_BUG_ON(xa, !xa_empty(xa));
369373
370374 /* Releasing a used entry does nothing */
371− xa_reserve(xa, 12345678, GFP_KERNEL);
375+ XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
372376 XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_NOWAIT) != NULL);
373377 xa_release(xa, 12345678);
374378 xa_erase_index(xa, 12345678);
375379 XA_BUG_ON(xa, !xa_empty(xa));
376380
377− /* cmpxchg sees a reserved entry as NULL */
378− xa_reserve(xa, 12345678, GFP_KERNEL);
379− XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, NULL, xa_mk_value(12345678),
380− GFP_NOWAIT) != NULL);
381+ /* cmpxchg sees a reserved entry as ZERO */
382+ XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
383+ XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, XA_ZERO_ENTRY,
384+ xa_mk_value(12345678), GFP_NOWAIT) != NULL);
381385 xa_release(xa, 12345678);
382386 xa_erase_index(xa, 12345678);
383387 XA_BUG_ON(xa, !xa_empty(xa));
384388
385− /* But xa_insert does not */
386− xa_reserve(xa, 12345678, GFP_KERNEL);
389+ /* xa_insert treats it as busy */
390+ XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
387391 XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) !=
388− -EEXIST);
392+ -EBUSY);
389393 XA_BUG_ON(xa, xa_empty(xa));
390394 XA_BUG_ON(xa, xa_erase(xa, 12345678) != NULL);
391395 XA_BUG_ON(xa, !xa_empty(xa));
392396
393397 /* Can iterate through a reserved entry */
394398 xa_store_index(xa, 5, GFP_KERNEL);
395− xa_reserve(xa, 6, GFP_KERNEL);
399+ XA_BUG_ON(xa, xa_reserve(xa, 6, GFP_KERNEL) != 0);
396400 xa_store_index(xa, 7, GFP_KERNEL);
397401
402+ count = 0;
398403 xa_for_each(xa, index, entry) {
399404 XA_BUG_ON(xa, index != 5 && index != 7);
405+ count++;
406+ }
407+ XA_BUG_ON(xa, count != 2);
408+
409+ /* If we free a reserved entry, we should be able to allocate it */
410+ if (xa->xa_flags & XA_FLAGS_ALLOC) {
411+ u32 id;
412+
413+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(8),
414+ XA_LIMIT(5, 10), GFP_KERNEL) != 0);
415+ XA_BUG_ON(xa, id != 8);
416+
417+ xa_release(xa, 6);
418+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(6),
419+ XA_LIMIT(5, 10), GFP_KERNEL) != 0);
420+ XA_BUG_ON(xa, id != 6);
400421 }
422+
401423 xa_destroy(xa);
402424 }
403425
@@ -586,64 +608,194 @@ static noinline void check_multi_store(struct xarray *xa) · 182 unmodified lines
586608 #endif
587609 }
588610
589−static DEFINE_XARRAY_ALLOC(xa0);
590−
591−static noinline void check_xa_alloc(void)
611+static noinline void check_xa_alloc_1(struct xarray *xa, unsigned int base)
592612 {
593613 int i;
594614 u32 id;
595615
596− /* An empty array should assign 0 to the first alloc */
597− xa_alloc_index(&xa0, 0, GFP_KERNEL);
616+ XA_BUG_ON(xa, !xa_empty(xa));
617+ /* An empty array should assign %base to the first alloc */
618+ xa_alloc_index(xa, base, GFP_KERNEL);
598619
599620 /* Erasing it should make the array empty again */
600− xa_erase_index(&xa0, 0);
601− XA_BUG_ON(&xa0, !xa_empty(&xa0));
621+ xa_erase_index(xa, base);
622+ XA_BUG_ON(xa, !xa_empty(xa));
623+
624+ /* And it should assign %base again */
625+ xa_alloc_index(xa, base, GFP_KERNEL);
626+
627+ /* Allocating and then erasing a lot should not lose base */
628+ for (i = base + 1; i < 2 * XA_CHUNK_SIZE; i++)
629+ xa_alloc_index(xa, i, GFP_KERNEL);
630+ for (i = base; i < 2 * XA_CHUNK_SIZE; i++)
631+ xa_erase_index(xa, i);
632+ xa_alloc_index(xa, base, GFP_KERNEL);
602633
603− /* And it should assign 0 again */
604− xa_alloc_index(&xa0, 0, GFP_KERNEL);
634+ /* Destroying the array should do the same as erasing */
635+ xa_destroy(xa);
636+
637+ /* And it should assign %base again */
638+ xa_alloc_index(xa, base, GFP_KERNEL);
605639
606− /* The next assigned ID should be 1 */
607− xa_alloc_index(&xa0, 1, GFP_KERNEL);
608− xa_erase_index(&xa0, 1);
640+ /* The next assigned ID should be base+1 */
641+ xa_alloc_index(xa, base + 1, GFP_KERNEL);
642+ xa_erase_index(xa, base + 1);
609643
610644 /* Storing a value should mark it used */
611− xa_store_index(&xa0, 1, GFP_KERNEL);
612− xa_alloc_index(&xa0, 2, GFP_KERNEL);
645+ xa_store_index(xa, base + 1, GFP_KERNEL);
646+ xa_alloc_index(xa, base + 2, GFP_KERNEL);
613647
614− /* If we then erase 0, it should be free */
615− xa_erase_index(&xa0, 0);
616− xa_alloc_index(&xa0, 0, GFP_KERNEL);
648+ /* If we then erase base, it should be free */
649+ xa_erase_index(xa, base);
650+ xa_alloc_index(xa, base, GFP_KERNEL);
617651
618− xa_erase_index(&xa0, 1);
619− xa_erase_index(&xa0, 2);
652+ xa_erase_index(xa, base + 1);
653+ xa_erase_index(xa, base + 2);
620654
621655 for (i = 1; i < 5000; i++) {
622− xa_alloc_index(&xa0, i, GFP_KERNEL);
656+ xa_alloc_index(xa, base + i, GFP_KERNEL);
623657 }
624658
625− xa_destroy(&xa0);
659+ xa_destroy(xa);
626660
627− id = 0xfffffffeU;
628− XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id),
661+ /* Check that we fail properly at the limit of allocation */
662+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX - 1),
663+ XA_LIMIT(UINT_MAX - 1, UINT_MAX),
629664 GFP_KERNEL) != 0);
630− XA_BUG_ON(&xa0, id != 0xfffffffeU);
631− XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id),
665+ XA_BUG_ON(xa, id != 0xfffffffeU);
666+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX),
667+ XA_LIMIT(UINT_MAX - 1, UINT_MAX),
632668 GFP_KERNEL) != 0);
633− XA_BUG_ON(&xa0, id != 0xffffffffU);
634− XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id),
635− GFP_KERNEL) != -ENOSPC);
636− XA_BUG_ON(&xa0, id != 0xffffffffU);
637− xa_destroy(&xa0);
638−
639− id = 10;
640− XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id),
641− GFP_KERNEL) != -ENOSPC);
642− XA_BUG_ON(&xa0, xa_store_index(&xa0, 3, GFP_KERNEL) != 0);
643− XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id),
644− GFP_KERNEL) != -ENOSPC);
645− xa_erase_index(&xa0, 3);
646− XA_BUG_ON(&xa0, !xa_empty(&xa0));
669+ XA_BUG_ON(xa, id != 0xffffffffU);
670+ id = 3;
671+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(0),
672+ XA_LIMIT(UINT_MAX - 1, UINT_MAX),
673+ GFP_KERNEL) != -EBUSY);
674+ XA_BUG_ON(xa, id != 3);
675+ xa_destroy(xa);
676+
677+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5),
678+ GFP_KERNEL) != -EBUSY);
679+ XA_BUG_ON(xa, xa_store_index(xa, 3, GFP_KERNEL) != 0);
680+ XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5),
681+ GFP_KERNEL) != -EBUSY);
682+ xa_erase_index(xa, 3);
683+ XA_BUG_ON(xa, !xa_empty(xa));
684+}
685+
686+static noinline void check_xa_alloc_2(struct xarray *xa, unsigned int base)
687+{
688+ unsigned int i, id;
689+ unsigned long index;
690+ void *entry;
691+
692+ /* Allocate and free a NULL and check xa_empty() behaves */
693+ XA_BUG_ON(xa, !xa_empty(xa));
694+ XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0);
695+ XA_BUG_ON(xa, id != base);
696+ XA_BUG_ON(xa, xa_empty(xa));
697+ XA_BUG_ON(xa, xa_erase(xa, id) != NULL);
698+ XA_BUG_ON(xa, !xa_empty(xa));
699+
700+ /* Ditto, but check destroy instead of erase */
701+ XA_BUG_ON(xa, !xa_empty(xa));
702+ XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0);
703+ XA_BUG_ON(xa, id != base);
704+ XA_BUG_ON(xa, xa_empty(xa));
705+ xa_destroy(xa);
706+ XA_BUG_ON(xa, !xa_empty(xa));
707+
708+ for (i = base; i < base + 10; i++) {
709+ XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b,
710+ GFP_KERNEL) != 0);
711+ XA_BUG_ON(xa, id != i);
712+ }
713+
714+ XA_BUG_ON(xa, xa_store(xa, 3, xa_mk_index(3), GFP_KERNEL) != NULL);
715+ XA_BUG_ON(xa, xa_store(xa, 4, xa_mk_index(4), GFP_KERNEL) != NULL);
716+ XA_BUG_ON(xa, xa_store(xa, 4, NULL, GFP_KERNEL) != xa_mk_index(4));
717+ XA_BUG_ON(xa, xa_erase(xa, 5) != NULL);
718+ XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0);
719+ XA_BUG_ON(xa, id != 5);
720+
721+ xa_for_each(xa, index, entry) {
722+ xa_erase_index(xa, index);
723+ }
724+
725+ for (i = base; i < base + 9; i++) {
726+ XA_BUG_ON(xa, xa_erase(xa, i) != NULL);
727+ XA_BUG_ON(xa, xa_empty(xa));
728+ }
729+ XA_BUG_ON(xa, xa_erase(xa, 8) != NULL);
730+ XA_BUG_ON(xa, xa_empty(xa));
731+ XA_BUG_ON(xa, xa_erase(xa, base + 9) != NULL);
732+ XA_BUG_ON(xa, !xa_empty(xa));
733+
734+ xa_destroy(xa);
735+}
736+
737+static noinline void check_xa_alloc_3(struct xarray *xa, unsigned int base)
738+{
739+ struct xa_limit limit = XA_LIMIT(1, 0x3fff);
740+ u32 next = 0;
741+ unsigned int i, id;
742+ unsigned long index;
743+ void *entry;
744+
745+ XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(1), limit,
746+ &next, GFP_KERNEL) != 0);
747+ XA_BUG_ON(xa, id != 1);
748+
749+ next = 0x3ffd;
750+ XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(0x3ffd), limit,
751+ &next, GFP_KERNEL) != 0);
752+ XA_BUG_ON(xa, id != 0x3ffd);
753+ xa_erase_index(xa, 0x3ffd);
754+ xa_erase_index(xa, 1);
755+ XA_BUG_ON(xa, !xa_empty(xa));
756+
757+ for (i = 0x3ffe; i < 0x4003; i++) {
758+ if (i < 0x4000)
759+ entry = xa_mk_index(i);
760+ else
761+ entry = xa_mk_index(i - 0x3fff);
762+ XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, entry, limit,
763+ &next, GFP_KERNEL) != (id == 1));
764+ XA_BUG_ON(xa, xa_mk_index(id) != entry);
765+ }
766+
767+ /* Check wrap-around is handled correctly */
768+ if (base != 0)
769+ xa_erase_index(xa, base);
770+ xa_erase_index(xa, base + 1);
771+ next = UINT_MAX;
772+ XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(UINT_MAX),
773+ xa_limit_32b, &next, GFP_KERNEL) != 0);
774+ XA_BUG_ON(xa, id != UINT_MAX);
775+ XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base),
776+ xa_limit_32b, &next, GFP_KERNEL) != 1);
777+ XA_BUG_ON(xa, id != base);
778+ XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base + 1),
779+ xa_limit_32b, &next, GFP_KERNEL) != 0);
780+ XA_BUG_ON(xa, id != base + 1);
781+
782+ xa_for_each(xa, index, entry)
783+ xa_erase_index(xa, index);
784+
785+ XA_BUG_ON(xa, !xa_empty(xa));
786+}
787+
788+static DEFINE_XARRAY_ALLOC(xa0);
789+static DEFINE_XARRAY_ALLOC1(xa1);
790+
791+static noinline void check_xa_alloc(void)
792+{
793+ check_xa_alloc_1(&xa0, 0);
794+ check_xa_alloc_1(&xa1, 1);
795+ check_xa_alloc_2(&xa0, 0);
796+ check_xa_alloc_2(&xa1, 1);
797+ check_xa_alloc_3(&xa0, 0);
798+ check_xa_alloc_3(&xa1, 1);
647799 }
648800
649801 static noinline void __check_store_iter(struct xarray *xa, unsigned long start,
@@ -1194,9 +1346,8 @@ static void check_align_1(struct xarray *xa, char *name) · 544 unmodified lines
11941346 void *entry;
11951347
11961348 for (i = 0; i < 8; i++) {
1197− id = 0;
1198− XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, name + i, GFP_KERNEL)
1199− != 0);
1349+ XA_BUG_ON(xa, xa_alloc(xa, &id, name + i, xa_limit_32b,
1350+ GFP_KERNEL) != 0);
12001351 XA_BUG_ON(xa, id != i);
12011352 }
12021353 xa_for_each(xa, index, entry)
@@ -1204,6 +1355,30 @@ static void check_align_1(struct xarray *xa, char *name) · 1 unmodified line
12041355 xa_destroy(xa);
12051356 }
12061357
1358+/*
1359+ * We should always be able to store without allocating memory after
1360+ * reserving a slot.
1361+ */
1362+static void check_align_2(struct xarray *xa, char *name)
1363+{
1364+ int i;
1365+
1366+ XA_BUG_ON(xa, !xa_empty(xa));
1367+
1368+ for (i = 0; i < 8; i++) {
1369+ XA_BUG_ON(xa, xa_store(xa, 0, name + i, GFP_KERNEL) != NULL);
1370+ xa_erase(xa, 0);
1371+ }
1372+
1373+ for (i = 0; i < 8; i++) {
1374+ XA_BUG_ON(xa, xa_reserve(xa, 0, GFP_KERNEL) != 0);
1375+ XA_BUG_ON(xa, xa_store(xa, 0, name + i, 0) != NULL);
1376+ xa_erase(xa, 0);
1377+ }
1378+
1379+ XA_BUG_ON(xa, !xa_empty(xa));
1380+}
1381+
12071382 static noinline void check_align(struct xarray *xa)
12081383 {
12091384 char name[] = "Motorola 68000";
@@ -1212,7 +1387,7 @@ static noinline void check_align(struct xarray *xa) · 2 unmodified lines
12121387 check_align_1(xa, name + 1);
12131388 check_align_1(xa, name + 2);
12141389 check_align_1(xa, name + 3);
1215−// check_align_2(xa, name);
1390+ check_align_2(xa, name);
12161391 }
12171392
12181393 static LIST_HEAD(shadow_nodes);
@@ -1354,6 +1529,7 @@ static int xarray_checks(void) · 135 unmodified lines
13541529 check_xas_erase(&array);
13551530 check_cmpxchg(&array);
13561531 check_reserve(&array);
1532+ check_reserve(&xa0);
13571533 check_multi_store(&array);
13581534 check_xa_alloc();
13591535 check_find(&array);
lib/xarray.c+92−71
@@ -57,6 +57,11 @@ static inline bool xa_track_free(const struct xarray *xa) · 56 unmodified lines
5757 return xa->xa_flags & XA_FLAGS_TRACK_FREE;
5858 }
5959
60+static inline bool xa_zero_busy(const struct xarray *xa)
61+{
62+ return xa->xa_flags & XA_FLAGS_ZERO_BUSY;
63+}
64+
6065 static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)
6166 {
6267 if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))
@@ -432,6 +437,8 @@ static void xas_shrink(struct xa_state *xas) · 369 unmodified lines
432437 break;
433438 if (!xa_is_node(entry) && node->shift)
434439 break;
440+ if (xa_is_zero(entry) && xa_zero_busy(xa))
441+ entry = NULL;
435442 xas->xa_node = XAS_BOUNDS;
436443
437444 RCU_INIT_POINTER(xa->xa_head, entry);
@@ -628,6 +635,8 @@ static void *xas_create(struct xa_state *xas, bool allow_root) · 190 unmodified lines
628635 if (xas_top(node)) {
629636 entry = xa_head_locked(xa);
630637 xas->xa_node = NULL;
638+ if (!entry && xa_zero_busy(xa))
639+ entry = XA_ZERO_ENTRY;
631640 shift = xas_expand(xas, entry);
632641 if (shift < 0)
633642 return NULL;
@@ -758,10 +767,12 @@ void *xas_store(struct xa_state *xas, void *entry) · 124 unmodified lines
758767 void *first, *next;
759768 bool value = xa_is_value(entry);
760769
761− if (entry)
762− first = xas_create(xas, !xa_is_node(entry));
763− else
770+ if (entry) {
771+ bool allow_root = !xa_is_node(entry) && !xa_is_zero(entry);
772+ first = xas_create(xas, allow_root);
773+ } else {
764774 first = xas_load(xas);
775+ }
765776
766777 if (xas_invalid(xas))
767778 return first;
@@ -791,7 +802,7 @@ void *xas_store(struct xa_state *xas, void *entry) · 23 unmodified lines
791802 * entry is set to NULL.
792803 */
793804 rcu_assign_pointer(*slot, entry);
794− if (xa_is_node(next))
805+ if (xa_is_node(next) && (!node || node->shift))
795806 xas_free_nodes(xas, xa_to_node(next));
796807 if (!node)
797808 break;
@@ -1294,13 +1305,12 @@ static void *xas_result(struct xa_state *xas, void *curr) · 496 unmodified lines
12941305 * @xa: XArray.
12951306 * @index: Index into array.
12961307 *
1297− * If the entry at this index is a multi-index entry then all indices will
1298− * be erased, and the entry will no longer be a multi-index entry.
1299− * This function expects the xa_lock to be held on entry.
1308+ * After this function returns, loading from @index will return %NULL.
1309+ * If the index is part of a multi-index entry, all indices will be erased
1310+ * and none of the entries will be part of a multi-index entry.
13001311 *
1301− * Context: Any context. Expects xa_lock to be held on entry. May
1302− * release and reacquire xa_lock if @gfp flags permit.
1303− * Return: The old entry at this index.
1312+ * Context: Any context. Expects xa_lock to be held on entry.
1313+ * Return: The entry which used to be at this index.
13041314 */
13051315 void *__xa_erase(struct xarray *xa, unsigned long index)
13061316 {
@@ -1314,9 +1324,9 @@ EXPORT_SYMBOL(__xa_erase); · 7 unmodified lines
13141324 * @xa: XArray.
13151325 * @index: Index of entry.
13161326 *
1317− * This function is the equivalent of calling xa_store() with %NULL as
1318− * the third argument. The XArray does not need to allocate memory, so
1319− * the user does not need to provide GFP flags.
1327+ * After this function returns, loading from @index will return %NULL.
1328+ * If the index is part of a multi-index entry, all indices will be erased
1329+ * and none of the entries will be part of a multi-index entry.
13201330 *
13211331 * Context: Any context. Takes and releases the xa_lock.
13221332 * Return: The entry which used to be at this index.
@@ -1421,16 +1431,12 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, · 98 unmodified lines
14211431
14221432 if (WARN_ON_ONCE(xa_is_advanced(entry)))
14231433 return XA_ERROR(-EINVAL);
1424− if (xa_track_free(xa) && !entry)
1425− entry = XA_ZERO_ENTRY;
14261434
14271435 do {
14281436 curr = xas_load(&xas);
1429− if (curr == XA_ZERO_ENTRY)
1430− curr = NULL;
14311437 if (curr == old) {
14321438 xas_store(&xas, entry);
1433− if (xa_track_free(xa))
1439+ if (xa_track_free(xa) && entry && !curr)
14341440 xas_clear_mark(&xas, XA_FREE_MARK);
14351441 }
14361442 } while (__xas_nomem(&xas, gfp));
@@ -1452,7 +1458,7 @@ EXPORT_SYMBOL(__xa_cmpxchg); · 15 unmodified lines
14521458 *
14531459 * Context: Any context. Expects xa_lock to be held on entry. May
14541460 * release and reacquire xa_lock if @gfp flags permit.
1455− * Return: 0 if the store succeeded. -EEXIST if another entry was present.
1461+ * Return: 0 if the store succeeded. -EBUSY if another entry was present.
14561462 * -ENOMEM if memory could not be allocated.
14571463 */
14581464 int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
@@ -1472,7 +1478,7 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 13 unmodified lines
14721478 if (xa_track_free(xa))
14731479 xas_clear_mark(&xas, XA_FREE_MARK);
14741480 } else {
1475− xas_set_err(&xas, -EEXIST);
1481+ xas_set_err(&xas, -EBUSY);
14761482 }
14771483 } while (__xas_nomem(&xas, gfp));
14781484
@@ -1480,42 +1486,6 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 1 unmodified line
14801486 }
14811487 EXPORT_SYMBOL(__xa_insert);
14821488
1483−/**
1484− * __xa_reserve() - Reserve this index in the XArray.
1485− * @xa: XArray.
1486− * @index: Index into array.
1487− * @gfp: Memory allocation flags.
1488− *
1489− * Ensures there is somewhere to store an entry at @index in the array.
1490− * If there is already something stored at @index, this function does
1491− * nothing. If there was nothing there, the entry is marked as reserved.
1492− * Loading from a reserved entry returns a %NULL pointer.
1493− *
1494− * If you do not use the entry that you have reserved, call xa_release()
1495− * or xa_erase() to free any unnecessary memory.
1496− *
1497− * Context: Any context. Expects the xa_lock to be held on entry. May
1498− * release the lock, sleep and reacquire the lock if the @gfp flags permit.
1499− * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
1500− */
1501−int __xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)
1502−{
1503− XA_STATE(xas, xa, index);
1504− void *curr;
1505−
1506− do {
1507− curr = xas_load(&xas);
1508− if (!curr) {
1509− xas_store(&xas, XA_ZERO_ENTRY);
1510− if (xa_track_free(xa))
1511− xas_clear_mark(&xas, XA_FREE_MARK);
1512− }
1513− } while (__xas_nomem(&xas, gfp));
1514−
1515− return xas_error(&xas);
1516−}
1517−EXPORT_SYMBOL(__xa_reserve);
1518−
15191489 #ifdef CONFIG_XARRAY_MULTI
15201490 static void xas_set_range(struct xa_state *xas, unsigned long first,
15211491 unsigned long last)
@@ -1607,23 +1577,23 @@ EXPORT_SYMBOL(xa_store_range); · 85 unmodified lines
16071577 * __xa_alloc() - Find somewhere to store this entry in the XArray.
16081578 * @xa: XArray.
16091579 * @id: Pointer to ID.
1610− * @max: Maximum ID to allocate (inclusive).
1580+ * @limit: Range for allocated ID.
16111581 * @entry: New entry.
16121582 * @gfp: Memory allocation flags.
16131583 *
1614− * Allocates an unused ID in the range specified by @id and @max.
1615− * Updates the @id pointer with the index, then stores the entry at that
1616− * index. A concurrent lookup will not see an uninitialised @id.
1584+ * Finds an empty entry in @xa between @limit.min and @limit.max,
1585+ * stores the index into the @id pointer, then stores the entry at
1586+ * that index. A concurrent lookup will not see an uninitialised @id.
16171587 *
16181588 * Context: Any context. Expects xa_lock to be held on entry. May
16191589 * release and reacquire xa_lock if @gfp flags permit.
1620− * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
1621− * there is no more space in the XArray.
1590+ * Return: 0 on success, -ENOMEM if memory could not be allocated or
1591+ * -EBUSY if there are no free entries in @limit.
16221592 */
1623−int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp)
1593+int __xa_alloc(struct xarray *xa, u32 *id, void *entry,
1594+ struct xa_limit limit, gfp_t gfp)
16241595 {
16251596 XA_STATE(xas, xa, 0);
1626− int err;
16271597
16281598 if (WARN_ON_ONCE(xa_is_advanced(entry)))
16291599 return -EINVAL;
@@ -1634,21 +1604,70 @@ int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) · 4 unmodified lines
16341604 entry = XA_ZERO_ENTRY;
16351605
16361606 do {
1637− xas.xa_index = *id;
1638− xas_find_marked(&xas, max, XA_FREE_MARK);
1607+ xas.xa_index = limit.min;
1608+ xas_find_marked(&xas, limit.max, XA_FREE_MARK);
16391609 if (xas.xa_node == XAS_RESTART)
1640− xas_set_err(&xas, -ENOSPC);
1610+ xas_set_err(&xas, -EBUSY);
1611+ else
1612+ *id = xas.xa_index;
16411613 xas_store(&xas, entry);
16421614 xas_clear_mark(&xas, XA_FREE_MARK);
16431615 } while (__xas_nomem(&xas, gfp));
16441616
1645− err = xas_error(&xas);
1646− if (!err)
1647− *id = xas.xa_index;
1648− return err;
1617+ return xas_error(&xas);
16491618 }
16501619 EXPORT_SYMBOL(__xa_alloc);
16511620
1621+/**
1622+ * __xa_alloc_cyclic() - Find somewhere to store this entry in the XArray.
1623+ * @xa: XArray.
1624+ * @id: Pointer to ID.
1625+ * @entry: New entry.
1626+ * @limit: Range of allocated ID.
1627+ * @next: Pointer to next ID to allocate.
1628+ * @gfp: Memory allocation flags.
1629+ *
1630+ * Finds an empty entry in @xa between @limit.min and @limit.max,
1631+ * stores the index into the @id pointer, then stores the entry at
1632+ * that index. A concurrent lookup will not see an uninitialised @id.
1633+ * The search for an empty entry will start at @next and will wrap
1634+ * around if necessary.
1635+ *
1636+ * Context: Any context. Expects xa_lock to be held on entry. May
1637+ * release and reacquire xa_lock if @gfp flags permit.
1638+ * Return: 0 if the allocation succeeded without wrapping. 1 if the
1639+ * allocation succeeded after wrapping, -ENOMEM if memory could not be
1640+ * allocated or -EBUSY if there are no free entries in @limit.
1641+ */
1642+int __xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
1643+ struct xa_limit limit, u32 *next, gfp_t gfp)
1644+{
1645+ u32 min = limit.min;
1646+ int ret;
1647+
1648+ limit.min = max(min, *next);
1649+ ret = __xa_alloc(xa, id, entry, limit, gfp);
1650+ if ((xa->xa_flags & XA_FLAGS_ALLOC_WRAPPED) && ret == 0) {
1651+ xa->xa_flags &= ~XA_FLAGS_ALLOC_WRAPPED;
1652+ ret = 1;
1653+ }
1654+
1655+ if (ret < 0 && limit.min > min) {
1656+ limit.min = min;
1657+ ret = __xa_alloc(xa, id, entry, limit, gfp);
1658+ if (ret == 0)
1659+ ret = 1;
1660+ }
1661+
1662+ if (ret >= 0) {
1663+ *next = *id + 1;
1664+ if (*next == 0)
1665+ xa->xa_flags |= XA_FLAGS_ALLOC_WRAPPED;
1666+ }
1667+ return ret;
1668+}
1669+EXPORT_SYMBOL(__xa_alloc_cyclic);
1670+
16521671 /**
16531672 * __xa_set_mark() - Set this mark on this entry while locked.
16541673 * @xa: XArray.
@@ -1943,6 +1962,8 @@ void xa_destroy(struct xarray *xa) · 288 unmodified lines
19431962 entry = xa_head_locked(xa);
19441963 RCU_INIT_POINTER(xa->xa_head, NULL);
19451964 xas_init_marks(&xas);
1965+ if (xa_zero_busy(xa))
1966+ xa_mark_clear(xa, XA_FREE_MARK);
19461967 /* lockdep checks we're still holding the lock in xas_free_nodes() */
19471968 if (xa_is_node(entry))
19481969 xas_free_nodes(&xas, xa_to_node(entry));

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t