History

Merge tag 'xarray-5.1-rc1' of git://git.infradead.org/users/willy/linux-dax

Pull XArray updates from Matthew Wilcox: "This pull request changes the xa_alloc() API. I'm only aware of one subsystem that has started trying to use it, and we agree on the fixup as part of the merge. The xa_insert() error code also changed to match xa_alloc() (EEXIST to EBUSY), and I added xa_alloc_cyclic(). Beyond that, the usual bugfixes, optimisations and tweaking. I now have a git tree with all users of the radix tree and IDR converted over to the XArray that I'll be feeding to maintainers over the next few weeks" * tag 'xarray-5.1-rc1' of git://git.infradead.org/users/willy/linux-dax: XArray: Fix xa_reserve for 2-byte aligned entries XArray: Fix xa_erase of 2-byte aligned entries XArray: Use xa_cmpxchg to implement xa_reserve XArray: Fix xa_release in allocating arrays XArray: Mark xa_insert and xa_reserve as must_check XArray: Add cyclic allocation XArray: Redesign xa_alloc API XArray: Add support for 1s-based allocation XArray: Change xa_insert to return -EBUSY XArray: Update xa_erase family descriptions XArray tests: RCU lock prohibits GFP_KERNEL

Linus Torvalds committed 7 years ago

Commit ea295482 parents f3124cc 4a5c8d87 files · +561 −260Browse files

Unified · Split

7 files
  1. Documentation/core-api/xarray.rst +10 −5
  2. drivers/infiniband/core/device.c +12 −20
  3. drivers/infiniband/core/restrack.c +2 −23
  4. fs/nilfs2/btnode.c +1 −1
  5. include/linux/xarray.h +212 −84
  6. lib/test_xarray.c +232 −56
  7. lib/xarray.c +92 −71
Documentation/core-api/xarray.rst+10−5
@@ -85,7 +85,7 @@ which was at that index; if it returns the same entry which was passed as · 84 unmodified lines
85 85
86If you want to only store a new entry to an index if the current entry86If you want to only store a new entry to an index if the current entry
87at that index is ``NULL``, you can use :c:func:`xa_insert` which87at that index is ``NULL``, you can use :c:func:`xa_insert` which
88returns ``-EEXIST`` if the entry is not empty.88returns ``-EBUSY`` if the entry is not empty.
89 89
90You can enquire whether a mark is set on an entry by using90You can enquire whether a mark is set on an entry by using
91:c:func:`xa_get_mark`. If the entry is not ``NULL``, you can set a mark91:c:func:`xa_get_mark`. If the entry is not ``NULL``, you can set a mark
@@ -131,17 +131,23 @@ If you use :c:func:`DEFINE_XARRAY_ALLOC` to define the XArray, or · 39 unmodified lines
131initialise it by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`,131initialise it by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`,
132the XArray changes to track whether entries are in use or not.132the XArray changes to track whether entries are in use or not.
133 133
134You can call :c:func:`xa_alloc` to store the entry at any unused index134You can call :c:func:`xa_alloc` to store the entry at an unused index
135in the XArray. If you need to modify the array from interrupt context,135in the XArray. If you need to modify the array from interrupt context,
136you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable136you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable
137interrupts while allocating the ID.137interrupts while allocating the ID.
138 138
139Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert`139Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert` will
140will mark the entry as being allocated. Unlike a normal XArray, storing140also mark the entry as being allocated. Unlike a normal XArray, storing
141``NULL`` will mark the entry as being in use, like :c:func:`xa_reserve`.141``NULL`` will mark the entry as being in use, like :c:func:`xa_reserve`.
142To free an entry, use :c:func:`xa_erase` (or :c:func:`xa_release` if142To free an entry, use :c:func:`xa_erase` (or :c:func:`xa_release` if
143you only want to free the entry if it's ``NULL``).143you only want to free the entry if it's ``NULL``).
144 144
145By default, the lowest free entry is allocated starting from 0. If you
146want to allocate entries starting at 1, it is more efficient to use
147:c:func:`DEFINE_XARRAY_ALLOC1` or ``XA_FLAGS_ALLOC1``. If you want to
148allocate IDs up to a maximum, then wrap back around to the lowest free
149ID, you can use :c:func:`xa_alloc_cyclic`.
150
145You cannot use ``XA_MARK_0`` with an allocating XArray as this mark151You cannot use ``XA_MARK_0`` with an allocating XArray as this mark
146is used to track whether an entry is free or not. The other marks are152is used to track whether an entry is free or not. The other marks are
147available for your use.153available for your use.
@@ -209,7 +215,6 @@ Assumes xa_lock held on entry: · 61 unmodified lines
209 * :c:func:`__xa_erase`215 * :c:func:`__xa_erase`
210 * :c:func:`__xa_cmpxchg`216 * :c:func:`__xa_cmpxchg`
211 * :c:func:`__xa_alloc`217 * :c:func:`__xa_alloc`
212 * :c:func:`__xa_reserve`
213 * :c:func:`__xa_set_mark`218 * :c:func:`__xa_set_mark`
214 * :c:func:`__xa_clear_mark`219 * :c:func:`__xa_clear_mark`
215 220
drivers/infiniband/core/device.c+12−20
@@ -668,19 +668,10 @@ static int assign_name(struct ib_device *device, const char *name) · 667 unmodified lines
668 }668 }
669 strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX);669 strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX);
670 670
671 /* Cyclically allocate a user visible ID for the device */671 ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b,
672 device->index = last_id;672 &last_id, GFP_KERNEL);
673 ret = xa_alloc(&devices, &device->index, INT_MAX, device, GFP_KERNEL);673 if (ret > 0)
674 if (ret == -ENOSPC) {674 ret = 0;
675 device->index = 0;
676 ret = xa_alloc(&devices, &device->index, INT_MAX, device,
677 GFP_KERNEL);
678 }
679 if (ret)
680 goto out;
681 last_id = device->index + 1;
682
683 ret = 0;
684 675
685out:676out:
686 up_write(&devices_rwsem);677 up_write(&devices_rwsem);
@@ -1059,14 +1050,15 @@ static int assign_client_id(struct ib_client *client) · 372 unmodified lines
1059 * to get the LIFO order. The extra linked list can go away if xarray1050 * to get the LIFO order. The extra linked list can go away if xarray
1060 * learns to reverse iterate.1051 * learns to reverse iterate.
1061 */1052 */
1062 if (list_empty(&client_list))1053 if (list_empty(&client_list)) {
1063 client->client_id = 0;1054 client->client_id = 0;
1064 else1055 } else {
1065 client->client_id =1056 struct ib_client *last;
1066 list_last_entry(&client_list, struct ib_client, list)1057
1067 ->client_id;1058 last = list_last_entry(&client_list, struct ib_client, list);
1068 ret = xa_alloc(&clients, &client->client_id, INT_MAX, client,1059 client->client_id = last->client_id + 1;
1069 GFP_KERNEL);1060 }
1061 ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL);
1070 if (ret)1062 if (ret)
1071 goto out;1063 goto out;
1072 1064
drivers/infiniband/core/restrack.c+2−23
@@ -13,28 +13,6 @@ · 12 unmodified lines
13#include "cma_priv.h"13#include "cma_priv.h"
14#include "restrack.h"14#include "restrack.h"
15 15
16static int rt_xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
17 u32 *next)
18{
19 int err;
20
21 *id = *next;
22 if (*next == U32_MAX)
23 *id = 0;
24
25 xa_lock(xa);
26 err = __xa_alloc(xa, id, U32_MAX, entry, GFP_KERNEL);
27 if (err && *next != U32_MAX) {
28 *id = 0;
29 err = __xa_alloc(xa, id, *next, entry, GFP_KERNEL);
30 }
31
32 if (!err)
33 *next = *id + 1;
34 xa_unlock(xa);
35 return err;
36}
37
38/**16/**
39 * rdma_restrack_init() - initialize and allocate resource tracking17 * rdma_restrack_init() - initialize and allocate resource tracking
40 * @dev: IB device18 * @dev: IB device
@@ -226,7 +204,8 @@ static void rdma_restrack_add(struct rdma_restrack_entry *res) · 185 unmodified lines
226 kref_init(&res->kref);204 kref_init(&res->kref);
227 init_completion(&res->comp);205 init_completion(&res->comp);
228 if (res->type != RDMA_RESTRACK_QP)206 if (res->type != RDMA_RESTRACK_QP)
229 ret = rt_xa_alloc_cyclic(&rt->xa, &res->id, res, &rt->next_id);207 ret = xa_alloc_cyclic(&rt->xa, &res->id, res, xa_limit_32b,
208 &rt->next_id, GFP_KERNEL);
230 else {209 else {
231 /* Special case to ensure that LQPN points to right QP */210 /* Special case to ensure that LQPN points to right QP */
232 struct ib_qp *qp = container_of(res, struct ib_qp, res);211 struct ib_qp *qp = container_of(res, struct ib_qp, res);
fs/nilfs2/btnode.c+1−1
@@ -189,7 +189,7 @@ retry: · 188 unmodified lines
189 */189 */
190 if (!err)190 if (!err)
191 return 0;191 return 0;
192 else if (err != -EEXIST)192 else if (err != -EBUSY)
193 goto failed_unlock;193 goto failed_unlock;
194 194
195 err = invalidate_inode_pages2_range(btnc, newkey, newkey);195 err = invalidate_inode_pages2_range(btnc, newkey, newkey);
include/linux/xarray.h+212−84
@@ -131,6 +131,12 @@ static inline unsigned int xa_pointer_tag(void *entry) · 130 unmodified lines
131 * xa_mk_internal() - Create an internal entry.131 * xa_mk_internal() - Create an internal entry.
132 * @v: Value to turn into an internal entry.132 * @v: Value to turn into an internal entry.
133 *133 *
134 * Internal entries are used for a number of purposes. Entries 0-255 are
135 * used for sibling entries (only 0-62 are used by the current code). 256
136 * is used for the retry entry. 257 is used for the reserved / zero entry.
137 * Negative internal entries are used to represent errnos. Node pointers
138 * are also tagged as internal entries in some situations.
139 *
134 * Context: Any context.140 * Context: Any context.
135 * Return: An XArray internal entry corresponding to this value.141 * Return: An XArray internal entry corresponding to this value.
136 */142 */
@@ -163,6 +169,22 @@ static inline bool xa_is_internal(const void *entry) · 26 unmodified lines
163 return ((unsigned long)entry & 3) == 2;169 return ((unsigned long)entry & 3) == 2;
164}170}
165 171
172#define XA_ZERO_ENTRY xa_mk_internal(257)
173
174/**
175 * xa_is_zero() - Is the entry a zero entry?
176 * @entry: Entry retrieved from the XArray
177 *
178 * The normal API will return NULL as the contents of a slot containing
179 * a zero entry. You can only see zero entries by using the advanced API.
180 *
181 * Return: %true if the entry is a zero entry.
182 */
183static inline bool xa_is_zero(const void *entry)
184{
185 return unlikely(entry == XA_ZERO_ENTRY);
186}
187
166/**188/**
167 * xa_is_err() - Report whether an XArray operation returned an error189 * xa_is_err() - Report whether an XArray operation returned an error
168 * @entry: Result from calling an XArray function190 * @entry: Result from calling an XArray function
@@ -200,6 +222,27 @@ static inline int xa_err(void *entry) · 31 unmodified lines
200 return 0;222 return 0;
201}223}
202 224
225/**
226 * struct xa_limit - Represents a range of IDs.
227 * @min: The lowest ID to allocate (inclusive).
228 * @max: The maximum ID to allocate (inclusive).
229 *
230 * This structure is used either directly or via the XA_LIMIT() macro
231 * to communicate the range of IDs that are valid for allocation.
232 * Two common ranges are predefined for you:
233 * * xa_limit_32b - [0 - UINT_MAX]
234 * * xa_limit_31b - [0 - INT_MAX]
235 */
236struct xa_limit {
237 u32 max;
238 u32 min;
239};
240
241#define XA_LIMIT(_min, _max) (struct xa_limit) { .min = _min, .max = _max }
242
243#define xa_limit_32b XA_LIMIT(0, UINT_MAX)
244#define xa_limit_31b XA_LIMIT(0, INT_MAX)
245
203typedef unsigned __bitwise xa_mark_t;246typedef unsigned __bitwise xa_mark_t;
204#define XA_MARK_0 ((__force xa_mark_t)0U)247#define XA_MARK_0 ((__force xa_mark_t)0U)
205#define XA_MARK_1 ((__force xa_mark_t)1U)248#define XA_MARK_1 ((__force xa_mark_t)1U)
@@ -220,10 +263,14 @@ enum xa_lock_type { · 14 unmodified lines
220#define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ)263#define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ)
221#define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH)264#define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH)
222#define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U)265#define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U)
266#define XA_FLAGS_ZERO_BUSY ((__force gfp_t)8U)
267#define XA_FLAGS_ALLOC_WRAPPED ((__force gfp_t)16U)
223#define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \268#define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \
224 (__force unsigned)(mark)))269 (__force unsigned)(mark)))
225 270
271/* ALLOC is for a normal 0-based alloc. ALLOC1 is for an 1-based alloc */
226#define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK))272#define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK))
273#define XA_FLAGS_ALLOC1 (XA_FLAGS_TRACK_FREE | XA_FLAGS_ZERO_BUSY)
227 274
228/**275/**
229 * struct xarray - The anchor of the XArray.276 * struct xarray - The anchor of the XArray.
@@ -279,7 +326,7 @@ struct xarray { · 49 unmodified lines
279#define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0)326#define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0)
280 327
281/**328/**
282 * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs.329 * DEFINE_XARRAY_ALLOC() - Define an XArray which allocates IDs starting at 0.
283 * @name: A string that names your XArray.330 * @name: A string that names your XArray.
284 *331 *
285 * This is intended for file scope definitions of allocating XArrays.332 * This is intended for file scope definitions of allocating XArrays.
@@ -287,6 +334,15 @@ struct xarray { · 1 unmodified line
287 */334 */
288#define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC)335#define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC)
289 336
337/**
338 * DEFINE_XARRAY_ALLOC1() - Define an XArray which allocates IDs starting at 1.
339 * @name: A string that names your XArray.
340 *
341 * This is intended for file scope definitions of allocating XArrays.
342 * See also DEFINE_XARRAY().
343 */
344#define DEFINE_XARRAY_ALLOC1(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC1)
345
290void *xa_load(struct xarray *, unsigned long index);346void *xa_load(struct xarray *, unsigned long index);
291void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);347void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
292void *xa_erase(struct xarray *, unsigned long index);348void *xa_erase(struct xarray *, unsigned long index);
@@ -463,9 +519,12 @@ void *__xa_erase(struct xarray *, unsigned long index); · 170 unmodified lines
463void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);519void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t);
464void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old,520void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old,
465 void *entry, gfp_t);521 void *entry, gfp_t);
466int __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t);522int __must_check __xa_insert(struct xarray *, unsigned long index,
467int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t);523 void *entry, gfp_t);
468int __xa_reserve(struct xarray *, unsigned long index, gfp_t);524int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry,
525 struct xa_limit, gfp_t);
526int __must_check __xa_alloc_cyclic(struct xarray *, u32 *id, void *entry,
527 struct xa_limit, u32 *next, gfp_t);
469void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);528void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
470void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);529void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
471 530
@@ -526,9 +585,9 @@ static inline void *xa_store_irq(struct xarray *xa, unsigned long index, · 54 unmodified lines
526 * @xa: XArray.585 * @xa: XArray.
527 * @index: Index of entry.586 * @index: Index of entry.
528 *587 *
529 * This function is the equivalent of calling xa_store() with %NULL as588 * After this function returns, loading from @index will return %NULL.
530 * the third argument. The XArray does not need to allocate memory, so589 * If the index is part of a multi-index entry, all indices will be erased
531 * the user does not need to provide GFP flags.590 * and none of the entries will be part of a multi-index entry.
532 *591 *
533 * Context: Any context. Takes and releases the xa_lock while592 * Context: Any context. Takes and releases the xa_lock while
534 * disabling softirqs.593 * disabling softirqs.
@@ -550,9 +609,9 @@ static inline void *xa_erase_bh(struct xarray *xa, unsigned long index) · 15 unmodified lines
550 * @xa: XArray.609 * @xa: XArray.
551 * @index: Index of entry.610 * @index: Index of entry.
552 *611 *
553 * This function is the equivalent of calling xa_store() with %NULL as612 * After this function returns, loading from @index will return %NULL.
554 * the third argument. The XArray does not need to allocate memory, so613 * If the index is part of a multi-index entry, all indices will be erased
555 * the user does not need to provide GFP flags.614 * and none of the entries will be part of a multi-index entry.
556 *615 *
557 * Context: Process context. Takes and releases the xa_lock while616 * Context: Process context. Takes and releases the xa_lock while
558 * disabling interrupts.617 * disabling interrupts.
@@ -664,11 +723,11 @@ static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index, · 105 unmodified lines
664 *723 *
665 * Context: Any context. Takes and releases the xa_lock. May sleep if724 * Context: Any context. Takes and releases the xa_lock. May sleep if
666 * the @gfp flags permit.725 * the @gfp flags permit.
667 * Return: 0 if the store succeeded. -EEXIST if another entry was present.726 * Return: 0 if the store succeeded. -EBUSY if another entry was present.
668 * -ENOMEM if memory could not be allocated.727 * -ENOMEM if memory could not be allocated.
669 */728 */
670static inline int xa_insert(struct xarray *xa, unsigned long index,729static inline int __must_check xa_insert(struct xarray *xa,
671 void *entry, gfp_t gfp)730 unsigned long index, void *entry, gfp_t gfp)
672{731{
673 int err;732 int err;
674 733
@@ -693,11 +752,11 @@ static inline int xa_insert(struct xarray *xa, unsigned long index, · 18 unmodified lines
693 *752 *
694 * Context: Any context. Takes and releases the xa_lock while753 * Context: Any context. Takes and releases the xa_lock while
695 * disabling softirqs. May sleep if the @gfp flags permit.754 * disabling softirqs. May sleep if the @gfp flags permit.
696 * Return: 0 if the store succeeded. -EEXIST if another entry was present.755 * Return: 0 if the store succeeded. -EBUSY if another entry was present.
697 * -ENOMEM if memory could not be allocated.756 * -ENOMEM if memory could not be allocated.
698 */757 */
699static inline int xa_insert_bh(struct xarray *xa, unsigned long index,758static inline int __must_check xa_insert_bh(struct xarray *xa,
700 void *entry, gfp_t gfp)759 unsigned long index, void *entry, gfp_t gfp)
701{760{
702 int err;761 int err;
703 762
@@ -722,11 +781,11 @@ static inline int xa_insert_bh(struct xarray *xa, unsigned long index, · 18 unmodified lines
722 *781 *
723 * Context: Process context. Takes and releases the xa_lock while782 * Context: Process context. Takes and releases the xa_lock while
724 * disabling interrupts. May sleep if the @gfp flags permit.783 * disabling interrupts. May sleep if the @gfp flags permit.
725 * Return: 0 if the store succeeded. -EEXIST if another entry was present.784 * Return: 0 if the store succeeded. -EBUSY if another entry was present.
726 * -ENOMEM if memory could not be allocated.785 * -ENOMEM if memory could not be allocated.
727 */786 */
728static inline int xa_insert_irq(struct xarray *xa, unsigned long index,787static inline int __must_check xa_insert_irq(struct xarray *xa,
729 void *entry, gfp_t gfp)788 unsigned long index, void *entry, gfp_t gfp)
730{789{
731 int err;790 int err;
732 791
@@ -741,26 +800,26 @@ static inline int xa_insert_irq(struct xarray *xa, unsigned long index, · 8 unmodified lines
741 * xa_alloc() - Find somewhere to store this entry in the XArray.800 * xa_alloc() - Find somewhere to store this entry in the XArray.
742 * @xa: XArray.801 * @xa: XArray.
743 * @id: Pointer to ID.802 * @id: Pointer to ID.
744 * @max: Maximum ID to allocate (inclusive).
745 * @entry: New entry.803 * @entry: New entry.
804 * @limit: Range of ID to allocate.
746 * @gfp: Memory allocation flags.805 * @gfp: Memory allocation flags.
747 *806 *
748 * Allocates an unused ID in the range specified by @id and @max.807 * Finds an empty entry in @xa between @limit.min and @limit.max,
749 * Updates the @id pointer with the index, then stores the entry at that808 * stores the index into the @id pointer, then stores the entry at
750 * index. A concurrent lookup will not see an uninitialised @id.809 * that index. A concurrent lookup will not see an uninitialised @id.
751 *810 *
752 * Context: Process context. Takes and releases the xa_lock. May sleep if811 * Context: Any context. Takes and releases the xa_lock. May sleep if
753 * the @gfp flags permit.812 * the @gfp flags permit.
754 * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if813 * Return: 0 on success, -ENOMEM if memory could not be allocated or
755 * there is no more space in the XArray.814 * -EBUSY if there are no free entries in @limit.
756 */815 */
757static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry,816static inline __must_check int xa_alloc(struct xarray *xa, u32 *id,
758 gfp_t gfp)817 void *entry, struct xa_limit limit, gfp_t gfp)
759{818{
760 int err;819 int err;
761 820
762 xa_lock(xa);821 xa_lock(xa);
763 err = __xa_alloc(xa, id, max, entry, gfp);822 err = __xa_alloc(xa, id, entry, limit, gfp);
764 xa_unlock(xa);823 xa_unlock(xa);
765 824
766 return err;825 return err;
@@ -770,26 +829,26 @@ static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines
770 * xa_alloc_bh() - Find somewhere to store this entry in the XArray.829 * xa_alloc_bh() - Find somewhere to store this entry in the XArray.
771 * @xa: XArray.830 * @xa: XArray.
772 * @id: Pointer to ID.831 * @id: Pointer to ID.
773 * @max: Maximum ID to allocate (inclusive).
774 * @entry: New entry.832 * @entry: New entry.
833 * @limit: Range of ID to allocate.
775 * @gfp: Memory allocation flags.834 * @gfp: Memory allocation flags.
776 *835 *
777 * Allocates an unused ID in the range specified by @id and @max.836 * Finds an empty entry in @xa between @limit.min and @limit.max,
778 * Updates the @id pointer with the index, then stores the entry at that837 * stores the index into the @id pointer, then stores the entry at
779 * index. A concurrent lookup will not see an uninitialised @id.838 * that index. A concurrent lookup will not see an uninitialised @id.
780 *839 *
781 * Context: Any context. Takes and releases the xa_lock while840 * Context: Any context. Takes and releases the xa_lock while
782 * disabling softirqs. May sleep if the @gfp flags permit.841 * disabling softirqs. May sleep if the @gfp flags permit.
783 * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if842 * Return: 0 on success, -ENOMEM if memory could not be allocated or
784 * there is no more space in the XArray.843 * -EBUSY if there are no free entries in @limit.
785 */844 */
786static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry,845static inline int __must_check xa_alloc_bh(struct xarray *xa, u32 *id,
787 gfp_t gfp)846 void *entry, struct xa_limit limit, gfp_t gfp)
788{847{
789 int err;848 int err;
790 849
791 xa_lock_bh(xa);850 xa_lock_bh(xa);
792 err = __xa_alloc(xa, id, max, entry, gfp);851 err = __xa_alloc(xa, id, entry, limit, gfp);
793 xa_unlock_bh(xa);852 xa_unlock_bh(xa);
794 853
795 return err;854 return err;
@@ -799,26 +858,125 @@ static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines
799 * xa_alloc_irq() - Find somewhere to store this entry in the XArray.858 * xa_alloc_irq() - Find somewhere to store this entry in the XArray.
800 * @xa: XArray.859 * @xa: XArray.
801 * @id: Pointer to ID.860 * @id: Pointer to ID.
802 * @max: Maximum ID to allocate (inclusive).
803 * @entry: New entry.861 * @entry: New entry.
862 * @limit: Range of ID to allocate.
804 * @gfp: Memory allocation flags.863 * @gfp: Memory allocation flags.
805 *864 *
806 * Allocates an unused ID in the range specified by @id and @max.865 * Finds an empty entry in @xa between @limit.min and @limit.max,
807 * Updates the @id pointer with the index, then stores the entry at that866 * stores the index into the @id pointer, then stores the entry at
808 * index. A concurrent lookup will not see an uninitialised @id.867 * that index. A concurrent lookup will not see an uninitialised @id.
809 *868 *
810 * Context: Process context. Takes and releases the xa_lock while869 * Context: Process context. Takes and releases the xa_lock while
811 * disabling interrupts. May sleep if the @gfp flags permit.870 * disabling interrupts. May sleep if the @gfp flags permit.
812 * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if871 * Return: 0 on success, -ENOMEM if memory could not be allocated or
813 * there is no more space in the XArray.872 * -EBUSY if there are no free entries in @limit.
814 */873 */
815static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry,874static inline int __must_check xa_alloc_irq(struct xarray *xa, u32 *id,
816 gfp_t gfp)875 void *entry, struct xa_limit limit, gfp_t gfp)
817{876{
818 int err;877 int err;
819 878
820 xa_lock_irq(xa);879 xa_lock_irq(xa);
821 err = __xa_alloc(xa, id, max, entry, gfp);880 err = __xa_alloc(xa, id, entry, limit, gfp);
881 xa_unlock_irq(xa);
882
883 return err;
884}
885
886/**
887 * xa_alloc_cyclic() - Find somewhere to store this entry in the XArray.
888 * @xa: XArray.
889 * @id: Pointer to ID.
890 * @entry: New entry.
891 * @limit: Range of allocated ID.
892 * @next: Pointer to next ID to allocate.
893 * @gfp: Memory allocation flags.
894 *
895 * Finds an empty entry in @xa between @limit.min and @limit.max,
896 * stores the index into the @id pointer, then stores the entry at
897 * that index. A concurrent lookup will not see an uninitialised @id.
898 * The search for an empty entry will start at @next and will wrap
899 * around if necessary.
900 *
901 * Context: Any context. Takes and releases the xa_lock. May sleep if
902 * the @gfp flags permit.
903 * Return: 0 if the allocation succeeded without wrapping. 1 if the
904 * allocation succeeded after wrapping, -ENOMEM if memory could not be
905 * allocated or -EBUSY if there are no free entries in @limit.
906 */
907static inline int xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
908 struct xa_limit limit, u32 *next, gfp_t gfp)
909{
910 int err;
911
912 xa_lock(xa);
913 err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
914 xa_unlock(xa);
915
916 return err;
917}
918
919/**
920 * xa_alloc_cyclic_bh() - Find somewhere to store this entry in the XArray.
921 * @xa: XArray.
922 * @id: Pointer to ID.
923 * @entry: New entry.
924 * @limit: Range of allocated ID.
925 * @next: Pointer to next ID to allocate.
926 * @gfp: Memory allocation flags.
927 *
928 * Finds an empty entry in @xa between @limit.min and @limit.max,
929 * stores the index into the @id pointer, then stores the entry at
930 * that index. A concurrent lookup will not see an uninitialised @id.
931 * The search for an empty entry will start at @next and will wrap
932 * around if necessary.
933 *
934 * Context: Any context. Takes and releases the xa_lock while
935 * disabling softirqs. May sleep if the @gfp flags permit.
936 * Return: 0 if the allocation succeeded without wrapping. 1 if the
937 * allocation succeeded after wrapping, -ENOMEM if memory could not be
938 * allocated or -EBUSY if there are no free entries in @limit.
939 */
940static inline int xa_alloc_cyclic_bh(struct xarray *xa, u32 *id, void *entry,
941 struct xa_limit limit, u32 *next, gfp_t gfp)
942{
943 int err;
944
945 xa_lock_bh(xa);
946 err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
947 xa_unlock_bh(xa);
948
949 return err;
950}
951
952/**
953 * xa_alloc_cyclic_irq() - Find somewhere to store this entry in the XArray.
954 * @xa: XArray.
955 * @id: Pointer to ID.
956 * @entry: New entry.
957 * @limit: Range of allocated ID.
958 * @next: Pointer to next ID to allocate.
959 * @gfp: Memory allocation flags.
960 *
961 * Finds an empty entry in @xa between @limit.min and @limit.max,
962 * stores the index into the @id pointer, then stores the entry at
963 * that index. A concurrent lookup will not see an uninitialised @id.
964 * The search for an empty entry will start at @next and will wrap
965 * around if necessary.
966 *
967 * Context: Process context. Takes and releases the xa_lock while
968 * disabling interrupts. May sleep if the @gfp flags permit.
969 * Return: 0 if the allocation succeeded without wrapping. 1 if the
970 * allocation succeeded after wrapping, -ENOMEM if memory could not be
971 * allocated or -EBUSY if there are no free entries in @limit.
972 */
973static inline int xa_alloc_cyclic_irq(struct xarray *xa, u32 *id, void *entry,
974 struct xa_limit limit, u32 *next, gfp_t gfp)
975{
976 int err;
977
978 xa_lock_irq(xa);
979 err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp);
822 xa_unlock_irq(xa);980 xa_unlock_irq(xa);
823 981
824 return err;982 return err;
@@ -842,16 +1000,10 @@ static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, · 17 unmodified lines
842 * May sleep if the @gfp flags permit.1000 * May sleep if the @gfp flags permit.
843 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.1001 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
844 */1002 */
845static inline1003static inline __must_check
846int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)1004int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)
847{1005{
848 int ret;1006 return xa_err(xa_cmpxchg(xa, index, NULL, XA_ZERO_ENTRY, gfp));
849
850 xa_lock(xa);
851 ret = __xa_reserve(xa, index, gfp);
852 xa_unlock(xa);
853
854 return ret;
855}1007}
856 1008
857/**1009/**
@@ -866,16 +1018,10 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines
866 * disabling softirqs.1018 * disabling softirqs.
867 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.1019 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
868 */1020 */
869static inline1021static inline __must_check
870int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp)1022int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp)
871{1023{
872 int ret;1024 return xa_err(xa_cmpxchg_bh(xa, index, NULL, XA_ZERO_ENTRY, gfp));
873
874 xa_lock_bh(xa);
875 ret = __xa_reserve(xa, index, gfp);
876 xa_unlock_bh(xa);
877
878 return ret;
879}1025}
880 1026
881/**1027/**
@@ -890,16 +1036,10 @@ int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines
890 * disabling interrupts.1036 * disabling interrupts.
891 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.1037 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
892 */1038 */
893static inline1039static inline __must_check
894int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp)1040int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp)
895{1041{
896 int ret;1042 return xa_err(xa_cmpxchg_irq(xa, index, NULL, XA_ZERO_ENTRY, gfp));
897
898 xa_lock_irq(xa);
899 ret = __xa_reserve(xa, index, gfp);
900 xa_unlock_irq(xa);
901
902 return ret;
903}1043}
904 1044
905/**1045/**
@@ -913,7 +1053,7 @@ int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) · 7 unmodified lines
913 */1053 */
914static inline void xa_release(struct xarray *xa, unsigned long index)1054static inline void xa_release(struct xarray *xa, unsigned long index)
915{1055{
916 xa_cmpxchg(xa, index, NULL, NULL, 0);1056 xa_cmpxchg(xa, index, XA_ZERO_ENTRY, NULL, 0);
917}1057}
918 1058
919/* Everything below here is the Advanced API. Proceed with caution. */1059/* Everything below here is the Advanced API. Proceed with caution. */
@@ -1073,18 +1213,6 @@ static inline bool xa_is_sibling(const void *entry) · 153 unmodified lines
1073}1213}
1074 1214
1075#define XA_RETRY_ENTRY xa_mk_internal(256)1215#define XA_RETRY_ENTRY xa_mk_internal(256)
1076#define XA_ZERO_ENTRY xa_mk_internal(257)
1077
1078/**
1079 * xa_is_zero() - Is the entry a zero entry?
1080 * @entry: Entry retrieved from the XArray
1081 *
1082 * Return: %true if the entry is a zero entry.
1083 */
1084static inline bool xa_is_zero(const void *entry)
1085{
1086 return unlikely(entry == XA_ZERO_ENTRY);
1087}
1088 1216
1089/**1217/**
1090 * xa_is_retry() - Is the entry a retry entry?1218 * xa_is_retry() - Is the entry a retry entry?
lib/test_xarray.c+232−56
@@ -40,9 +40,9 @@ static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp) · 39 unmodified lines
40 40
41static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp)41static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp)
42{42{
43 u32 id = 0;43 u32 id;
44 44
45 XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, xa_mk_index(index),45 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(index), xa_limit_32b,
46 gfp) != 0);46 gfp) != 0);
47 XA_BUG_ON(xa, id != index);47 XA_BUG_ON(xa, id != index);
48}48}
@@ -107,8 +107,11 @@ static noinline void check_xas_retry(struct xarray *xa) · 58 unmodified lines
107 XA_BUG_ON(xa, xas.xa_node != XAS_RESTART);107 XA_BUG_ON(xa, xas.xa_node != XAS_RESTART);
108 XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));108 XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
109 XA_BUG_ON(xa, xas.xa_node != NULL);109 XA_BUG_ON(xa, xas.xa_node != NULL);
110 rcu_read_unlock();
110 111
111 XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);112 XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);
113
114 rcu_read_lock();
112 XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas)));115 XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas)));
113 xas.xa_node = XAS_RESTART;116 xas.xa_node = XAS_RESTART;
114 XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));117 XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
@@ -343,7 +346,7 @@ static noinline void check_cmpxchg(struct xarray *xa) · 228 unmodified lines
343 346
344 XA_BUG_ON(xa, !xa_empty(xa));347 XA_BUG_ON(xa, !xa_empty(xa));
345 XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL);348 XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL);
346 XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST);349 XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EBUSY);
347 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS);350 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS);
348 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS);351 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS);
349 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE);352 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE);
@@ -358,46 +361,65 @@ static noinline void check_reserve(struct xarray *xa) · 8 unmodified lines
358{361{
359 void *entry;362 void *entry;
360 unsigned long index;363 unsigned long index;
364 int count;
361 365
362 /* An array with a reserved entry is not empty */366 /* An array with a reserved entry is not empty */
363 XA_BUG_ON(xa, !xa_empty(xa));367 XA_BUG_ON(xa, !xa_empty(xa));
364 xa_reserve(xa, 12345678, GFP_KERNEL);368 XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
365 XA_BUG_ON(xa, xa_empty(xa));369 XA_BUG_ON(xa, xa_empty(xa));
366 XA_BUG_ON(xa, xa_load(xa, 12345678));370 XA_BUG_ON(xa, xa_load(xa, 12345678));
367 xa_release(xa, 12345678);371 xa_release(xa, 12345678);
368 XA_BUG_ON(xa, !xa_empty(xa));372 XA_BUG_ON(xa, !xa_empty(xa));
369 373
370 /* Releasing a used entry does nothing */374 /* Releasing a used entry does nothing */
371 xa_reserve(xa, 12345678, GFP_KERNEL);375 XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
372 XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_NOWAIT) != NULL);376 XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_NOWAIT) != NULL);
373 xa_release(xa, 12345678);377 xa_release(xa, 12345678);
374 xa_erase_index(xa, 12345678);378 xa_erase_index(xa, 12345678);
375 XA_BUG_ON(xa, !xa_empty(xa));379 XA_BUG_ON(xa, !xa_empty(xa));
376 380
377 /* cmpxchg sees a reserved entry as NULL */381 /* cmpxchg sees a reserved entry as ZERO */
378 xa_reserve(xa, 12345678, GFP_KERNEL);382 XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
379 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, NULL, xa_mk_value(12345678),383 XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, XA_ZERO_ENTRY,
380 GFP_NOWAIT) != NULL);384 xa_mk_value(12345678), GFP_NOWAIT) != NULL);
381 xa_release(xa, 12345678);385 xa_release(xa, 12345678);
382 xa_erase_index(xa, 12345678);386 xa_erase_index(xa, 12345678);
383 XA_BUG_ON(xa, !xa_empty(xa));387 XA_BUG_ON(xa, !xa_empty(xa));
384 388
385 /* But xa_insert does not */389 /* xa_insert treats it as busy */
386 xa_reserve(xa, 12345678, GFP_KERNEL);390 XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0);
387 XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) !=391 XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) !=
388 -EEXIST);392 -EBUSY);
389 XA_BUG_ON(xa, xa_empty(xa));393 XA_BUG_ON(xa, xa_empty(xa));
390 XA_BUG_ON(xa, xa_erase(xa, 12345678) != NULL);394 XA_BUG_ON(xa, xa_erase(xa, 12345678) != NULL);
391 XA_BUG_ON(xa, !xa_empty(xa));395 XA_BUG_ON(xa, !xa_empty(xa));
392 396
393 /* Can iterate through a reserved entry */397 /* Can iterate through a reserved entry */
394 xa_store_index(xa, 5, GFP_KERNEL);398 xa_store_index(xa, 5, GFP_KERNEL);
395 xa_reserve(xa, 6, GFP_KERNEL);399 XA_BUG_ON(xa, xa_reserve(xa, 6, GFP_KERNEL) != 0);
396 xa_store_index(xa, 7, GFP_KERNEL);400 xa_store_index(xa, 7, GFP_KERNEL);
397 401
402 count = 0;
398 xa_for_each(xa, index, entry) {403 xa_for_each(xa, index, entry) {
399 XA_BUG_ON(xa, index != 5 && index != 7);404 XA_BUG_ON(xa, index != 5 && index != 7);
405 count++;
406 }
407 XA_BUG_ON(xa, count != 2);
408
409 /* If we free a reserved entry, we should be able to allocate it */
410 if (xa->xa_flags & XA_FLAGS_ALLOC) {
411 u32 id;
412
413 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(8),
414 XA_LIMIT(5, 10), GFP_KERNEL) != 0);
415 XA_BUG_ON(xa, id != 8);
416
417 xa_release(xa, 6);
418 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(6),
419 XA_LIMIT(5, 10), GFP_KERNEL) != 0);
420 XA_BUG_ON(xa, id != 6);
400 }421 }
422
401 xa_destroy(xa);423 xa_destroy(xa);
402}424}
403 425
@@ -586,64 +608,194 @@ static noinline void check_multi_store(struct xarray *xa) · 182 unmodified lines
586#endif608#endif
587}609}
588 610
589static DEFINE_XARRAY_ALLOC(xa0);611static noinline void check_xa_alloc_1(struct xarray *xa, unsigned int base)
590
591static noinline void check_xa_alloc(void)
592{612{
593 int i;613 int i;
594 u32 id;614 u32 id;
595 615
596 /* An empty array should assign 0 to the first alloc */616 XA_BUG_ON(xa, !xa_empty(xa));
597 xa_alloc_index(&xa0, 0, GFP_KERNEL);617 /* An empty array should assign %base to the first alloc */
618 xa_alloc_index(xa, base, GFP_KERNEL);
598 619
599 /* Erasing it should make the array empty again */620 /* Erasing it should make the array empty again */
600 xa_erase_index(&xa0, 0);621 xa_erase_index(xa, base);
601 XA_BUG_ON(&xa0, !xa_empty(&xa0));622 XA_BUG_ON(xa, !xa_empty(xa));
623
624 /* And it should assign %base again */
625 xa_alloc_index(xa, base, GFP_KERNEL);
626
627 /* Allocating and then erasing a lot should not lose base */
628 for (i = base + 1; i < 2 * XA_CHUNK_SIZE; i++)
629 xa_alloc_index(xa, i, GFP_KERNEL);
630 for (i = base; i < 2 * XA_CHUNK_SIZE; i++)
631 xa_erase_index(xa, i);
632 xa_alloc_index(xa, base, GFP_KERNEL);
602 633
603 /* And it should assign 0 again */634 /* Destroying the array should do the same as erasing */
604 xa_alloc_index(&xa0, 0, GFP_KERNEL);635 xa_destroy(xa);
636
637 /* And it should assign %base again */
638 xa_alloc_index(xa, base, GFP_KERNEL);
605 639
606 /* The next assigned ID should be 1 */640 /* The next assigned ID should be base+1 */
607 xa_alloc_index(&xa0, 1, GFP_KERNEL);641 xa_alloc_index(xa, base + 1, GFP_KERNEL);
608 xa_erase_index(&xa0, 1);642 xa_erase_index(xa, base + 1);
609 643
610 /* Storing a value should mark it used */644 /* Storing a value should mark it used */
611 xa_store_index(&xa0, 1, GFP_KERNEL);645 xa_store_index(xa, base + 1, GFP_KERNEL);
612 xa_alloc_index(&xa0, 2, GFP_KERNEL);646 xa_alloc_index(xa, base + 2, GFP_KERNEL);
613 647
614 /* If we then erase 0, it should be free */648 /* If we then erase base, it should be free */
615 xa_erase_index(&xa0, 0);649 xa_erase_index(xa, base);
616 xa_alloc_index(&xa0, 0, GFP_KERNEL);650 xa_alloc_index(xa, base, GFP_KERNEL);
617 651
618 xa_erase_index(&xa0, 1);652 xa_erase_index(xa, base + 1);
619 xa_erase_index(&xa0, 2);653 xa_erase_index(xa, base + 2);
620 654
621 for (i = 1; i < 5000; i++) {655 for (i = 1; i < 5000; i++) {
622 xa_alloc_index(&xa0, i, GFP_KERNEL);656 xa_alloc_index(xa, base + i, GFP_KERNEL);
623 }657 }
624 658
625 xa_destroy(&xa0);659 xa_destroy(xa);
626 660
627 id = 0xfffffffeU;661 /* Check that we fail properly at the limit of allocation */
628 XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id),662 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX - 1),
663 XA_LIMIT(UINT_MAX - 1, UINT_MAX),
629 GFP_KERNEL) != 0);664 GFP_KERNEL) != 0);
630 XA_BUG_ON(&xa0, id != 0xfffffffeU);665 XA_BUG_ON(xa, id != 0xfffffffeU);
631 XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id),666 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX),
667 XA_LIMIT(UINT_MAX - 1, UINT_MAX),
632 GFP_KERNEL) != 0);668 GFP_KERNEL) != 0);
633 XA_BUG_ON(&xa0, id != 0xffffffffU);669 XA_BUG_ON(xa, id != 0xffffffffU);
634 XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id),670 id = 3;
635 GFP_KERNEL) != -ENOSPC);671 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(0),
636 XA_BUG_ON(&xa0, id != 0xffffffffU);672 XA_LIMIT(UINT_MAX - 1, UINT_MAX),
637 xa_destroy(&xa0);673 GFP_KERNEL) != -EBUSY);
638 674 XA_BUG_ON(xa, id != 3);
639 id = 10;675 xa_destroy(xa);
640 XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id),676
641 GFP_KERNEL) != -ENOSPC);677 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5),
642 XA_BUG_ON(&xa0, xa_store_index(&xa0, 3, GFP_KERNEL) != 0);678 GFP_KERNEL) != -EBUSY);
643 XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id),679 XA_BUG_ON(xa, xa_store_index(xa, 3, GFP_KERNEL) != 0);
644 GFP_KERNEL) != -ENOSPC);680 XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5),
645 xa_erase_index(&xa0, 3);681 GFP_KERNEL) != -EBUSY);
646 XA_BUG_ON(&xa0, !xa_empty(&xa0));682 xa_erase_index(xa, 3);
683 XA_BUG_ON(xa, !xa_empty(xa));
684}
685
686static noinline void check_xa_alloc_2(struct xarray *xa, unsigned int base)
687{
688 unsigned int i, id;
689 unsigned long index;
690 void *entry;
691
692 /* Allocate and free a NULL and check xa_empty() behaves */
693 XA_BUG_ON(xa, !xa_empty(xa));
694 XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0);
695 XA_BUG_ON(xa, id != base);
696 XA_BUG_ON(xa, xa_empty(xa));
697 XA_BUG_ON(xa, xa_erase(xa, id) != NULL);
698 XA_BUG_ON(xa, !xa_empty(xa));
699
700 /* Ditto, but check destroy instead of erase */
701 XA_BUG_ON(xa, !xa_empty(xa));
702 XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0);
703 XA_BUG_ON(xa, id != base);
704 XA_BUG_ON(xa, xa_empty(xa));
705 xa_destroy(xa);
706 XA_BUG_ON(xa, !xa_empty(xa));
707
708 for (i = base; i < base + 10; i++) {
709 XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b,
710 GFP_KERNEL) != 0);
711 XA_BUG_ON(xa, id != i);
712 }
713
714 XA_BUG_ON(xa, xa_store(xa, 3, xa_mk_index(3), GFP_KERNEL) != NULL);
715 XA_BUG_ON(xa, xa_store(xa, 4, xa_mk_index(4), GFP_KERNEL) != NULL);
716 XA_BUG_ON(xa, xa_store(xa, 4, NULL, GFP_KERNEL) != xa_mk_index(4));
717 XA_BUG_ON(xa, xa_erase(xa, 5) != NULL);
718 XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0);
719 XA_BUG_ON(xa, id != 5);
720
721 xa_for_each(xa, index, entry) {
722 xa_erase_index(xa, index);
723 }
724
725 for (i = base; i < base + 9; i++) {
726 XA_BUG_ON(xa, xa_erase(xa, i) != NULL);
727 XA_BUG_ON(xa, xa_empty(xa));
728 }
729 XA_BUG_ON(xa, xa_erase(xa, 8) != NULL);
730 XA_BUG_ON(xa, xa_empty(xa));
731 XA_BUG_ON(xa, xa_erase(xa, base + 9) != NULL);
732 XA_BUG_ON(xa, !xa_empty(xa));
733
734 xa_destroy(xa);
735}
736
737static noinline void check_xa_alloc_3(struct xarray *xa, unsigned int base)
738{
739 struct xa_limit limit = XA_LIMIT(1, 0x3fff);
740 u32 next = 0;
741 unsigned int i, id;
742 unsigned long index;
743 void *entry;
744
745 XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(1), limit,
746 &next, GFP_KERNEL) != 0);
747 XA_BUG_ON(xa, id != 1);
748
749 next = 0x3ffd;
750 XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(0x3ffd), limit,
751 &next, GFP_KERNEL) != 0);
752 XA_BUG_ON(xa, id != 0x3ffd);
753 xa_erase_index(xa, 0x3ffd);
754 xa_erase_index(xa, 1);
755 XA_BUG_ON(xa, !xa_empty(xa));
756
757 for (i = 0x3ffe; i < 0x4003; i++) {
758 if (i < 0x4000)
759 entry = xa_mk_index(i);
760 else
761 entry = xa_mk_index(i - 0x3fff);
762 XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, entry, limit,
763 &next, GFP_KERNEL) != (id == 1));
764 XA_BUG_ON(xa, xa_mk_index(id) != entry);
765 }
766
767 /* Check wrap-around is handled correctly */
768 if (base != 0)
769 xa_erase_index(xa, base);
770 xa_erase_index(xa, base + 1);
771 next = UINT_MAX;
772 XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(UINT_MAX),
773 xa_limit_32b, &next, GFP_KERNEL) != 0);
774 XA_BUG_ON(xa, id != UINT_MAX);
775 XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base),
776 xa_limit_32b, &next, GFP_KERNEL) != 1);
777 XA_BUG_ON(xa, id != base);
778 XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base + 1),
779 xa_limit_32b, &next, GFP_KERNEL) != 0);
780 XA_BUG_ON(xa, id != base + 1);
781
782 xa_for_each(xa, index, entry)
783 xa_erase_index(xa, index);
784
785 XA_BUG_ON(xa, !xa_empty(xa));
786}
787
788static DEFINE_XARRAY_ALLOC(xa0);
789static DEFINE_XARRAY_ALLOC1(xa1);
790
791static noinline void check_xa_alloc(void)
792{
793 check_xa_alloc_1(&xa0, 0);
794 check_xa_alloc_1(&xa1, 1);
795 check_xa_alloc_2(&xa0, 0);
796 check_xa_alloc_2(&xa1, 1);
797 check_xa_alloc_3(&xa0, 0);
798 check_xa_alloc_3(&xa1, 1);
647}799}
648 800
649static noinline void __check_store_iter(struct xarray *xa, unsigned long start,801static noinline void __check_store_iter(struct xarray *xa, unsigned long start,
@@ -1194,9 +1346,8 @@ static void check_align_1(struct xarray *xa, char *name) · 544 unmodified lines
1194 void *entry;1346 void *entry;
1195 1347
1196 for (i = 0; i < 8; i++) {1348 for (i = 0; i < 8; i++) {
1197 id = 0;1349 XA_BUG_ON(xa, xa_alloc(xa, &id, name + i, xa_limit_32b,
1198 XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, name + i, GFP_KERNEL)1350 GFP_KERNEL) != 0);
1199 != 0);
1200 XA_BUG_ON(xa, id != i);1351 XA_BUG_ON(xa, id != i);
1201 }1352 }
1202 xa_for_each(xa, index, entry)1353 xa_for_each(xa, index, entry)
@@ -1204,6 +1355,30 @@ static void check_align_1(struct xarray *xa, char *name) · 1 unmodified line
1204 xa_destroy(xa);1355 xa_destroy(xa);
1205}1356}
1206 1357
1358/*
1359 * We should always be able to store without allocating memory after
1360 * reserving a slot.
1361 */
1362static void check_align_2(struct xarray *xa, char *name)
1363{
1364 int i;
1365
1366 XA_BUG_ON(xa, !xa_empty(xa));
1367
1368 for (i = 0; i < 8; i++) {
1369 XA_BUG_ON(xa, xa_store(xa, 0, name + i, GFP_KERNEL) != NULL);
1370 xa_erase(xa, 0);
1371 }
1372
1373 for (i = 0; i < 8; i++) {
1374 XA_BUG_ON(xa, xa_reserve(xa, 0, GFP_KERNEL) != 0);
1375 XA_BUG_ON(xa, xa_store(xa, 0, name + i, 0) != NULL);
1376 xa_erase(xa, 0);
1377 }
1378
1379 XA_BUG_ON(xa, !xa_empty(xa));
1380}
1381
1207static noinline void check_align(struct xarray *xa)1382static noinline void check_align(struct xarray *xa)
1208{1383{
1209 char name[] = "Motorola 68000";1384 char name[] = "Motorola 68000";
@@ -1212,7 +1387,7 @@ static noinline void check_align(struct xarray *xa) · 2 unmodified lines
1212 check_align_1(xa, name + 1);1387 check_align_1(xa, name + 1);
1213 check_align_1(xa, name + 2);1388 check_align_1(xa, name + 2);
1214 check_align_1(xa, name + 3);1389 check_align_1(xa, name + 3);
1215// check_align_2(xa, name);1390 check_align_2(xa, name);
1216}1391}
1217 1392
1218static LIST_HEAD(shadow_nodes);1393static LIST_HEAD(shadow_nodes);
@@ -1354,6 +1529,7 @@ static int xarray_checks(void) · 135 unmodified lines
1354 check_xas_erase(&array);1529 check_xas_erase(&array);
1355 check_cmpxchg(&array);1530 check_cmpxchg(&array);
1356 check_reserve(&array);1531 check_reserve(&array);
1532 check_reserve(&xa0);
1357 check_multi_store(&array);1533 check_multi_store(&array);
1358 check_xa_alloc();1534 check_xa_alloc();
1359 check_find(&array);1535 check_find(&array);
lib/xarray.c+92−71
@@ -57,6 +57,11 @@ static inline bool xa_track_free(const struct xarray *xa) · 56 unmodified lines
57 return xa->xa_flags & XA_FLAGS_TRACK_FREE;57 return xa->xa_flags & XA_FLAGS_TRACK_FREE;
58}58}
59 59
60static inline bool xa_zero_busy(const struct xarray *xa)
61{
62 return xa->xa_flags & XA_FLAGS_ZERO_BUSY;
63}
64
60static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)65static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)
61{66{
62 if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))67 if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))
@@ -432,6 +437,8 @@ static void xas_shrink(struct xa_state *xas) · 369 unmodified lines
432 break;437 break;
433 if (!xa_is_node(entry) && node->shift)438 if (!xa_is_node(entry) && node->shift)
434 break;439 break;
440 if (xa_is_zero(entry) && xa_zero_busy(xa))
441 entry = NULL;
435 xas->xa_node = XAS_BOUNDS;442 xas->xa_node = XAS_BOUNDS;
436 443
437 RCU_INIT_POINTER(xa->xa_head, entry);444 RCU_INIT_POINTER(xa->xa_head, entry);
@@ -628,6 +635,8 @@ static void *xas_create(struct xa_state *xas, bool allow_root) · 190 unmodified lines
628 if (xas_top(node)) {635 if (xas_top(node)) {
629 entry = xa_head_locked(xa);636 entry = xa_head_locked(xa);
630 xas->xa_node = NULL;637 xas->xa_node = NULL;
638 if (!entry && xa_zero_busy(xa))
639 entry = XA_ZERO_ENTRY;
631 shift = xas_expand(xas, entry);640 shift = xas_expand(xas, entry);
632 if (shift < 0)641 if (shift < 0)
633 return NULL;642 return NULL;
@@ -758,10 +767,12 @@ void *xas_store(struct xa_state *xas, void *entry) · 124 unmodified lines
758 void *first, *next;767 void *first, *next;
759 bool value = xa_is_value(entry);768 bool value = xa_is_value(entry);
760 769
761 if (entry)770 if (entry) {
762 first = xas_create(xas, !xa_is_node(entry));771 bool allow_root = !xa_is_node(entry) && !xa_is_zero(entry);
763 else772 first = xas_create(xas, allow_root);
773 } else {
764 first = xas_load(xas);774 first = xas_load(xas);
775 }
765 776
766 if (xas_invalid(xas))777 if (xas_invalid(xas))
767 return first;778 return first;
@@ -791,7 +802,7 @@ void *xas_store(struct xa_state *xas, void *entry) · 23 unmodified lines
791 * entry is set to NULL.802 * entry is set to NULL.
792 */803 */
793 rcu_assign_pointer(*slot, entry);804 rcu_assign_pointer(*slot, entry);
794 if (xa_is_node(next))805 if (xa_is_node(next) && (!node || node->shift))
795 xas_free_nodes(xas, xa_to_node(next));806 xas_free_nodes(xas, xa_to_node(next));
796 if (!node)807 if (!node)
797 break;808 break;
@@ -1294,13 +1305,12 @@ static void *xas_result(struct xa_state *xas, void *curr) · 496 unmodified lines
1294 * @xa: XArray.1305 * @xa: XArray.
1295 * @index: Index into array.1306 * @index: Index into array.
1296 *1307 *
1297 * If the entry at this index is a multi-index entry then all indices will1308 * After this function returns, loading from @index will return %NULL.
1298 * be erased, and the entry will no longer be a multi-index entry.1309 * If the index is part of a multi-index entry, all indices will be erased
1299 * This function expects the xa_lock to be held on entry.1310 * and none of the entries will be part of a multi-index entry.
1300 *1311 *
1301 * Context: Any context. Expects xa_lock to be held on entry. May1312 * Context: Any context. Expects xa_lock to be held on entry.
1302 * release and reacquire xa_lock if @gfp flags permit.1313 * Return: The entry which used to be at this index.
1303 * Return: The old entry at this index.
1304 */1314 */
1305void *__xa_erase(struct xarray *xa, unsigned long index)1315void *__xa_erase(struct xarray *xa, unsigned long index)
1306{1316{
@@ -1314,9 +1324,9 @@ EXPORT_SYMBOL(__xa_erase); · 7 unmodified lines
1314 * @xa: XArray.1324 * @xa: XArray.
1315 * @index: Index of entry.1325 * @index: Index of entry.
1316 *1326 *
1317 * This function is the equivalent of calling xa_store() with %NULL as1327 * After this function returns, loading from @index will return %NULL.
1318 * the third argument. The XArray does not need to allocate memory, so1328 * If the index is part of a multi-index entry, all indices will be erased
1319 * the user does not need to provide GFP flags.1329 * and none of the entries will be part of a multi-index entry.
1320 *1330 *
1321 * Context: Any context. Takes and releases the xa_lock.1331 * Context: Any context. Takes and releases the xa_lock.
1322 * Return: The entry which used to be at this index.1332 * Return: The entry which used to be at this index.
@@ -1421,16 +1431,12 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, · 98 unmodified lines
1421 1431
1422 if (WARN_ON_ONCE(xa_is_advanced(entry)))1432 if (WARN_ON_ONCE(xa_is_advanced(entry)))
1423 return XA_ERROR(-EINVAL);1433 return XA_ERROR(-EINVAL);
1424 if (xa_track_free(xa) && !entry)
1425 entry = XA_ZERO_ENTRY;
1426 1434
1427 do {1435 do {
1428 curr = xas_load(&xas);1436 curr = xas_load(&xas);
1429 if (curr == XA_ZERO_ENTRY)
1430 curr = NULL;
1431 if (curr == old) {1437 if (curr == old) {
1432 xas_store(&xas, entry);1438 xas_store(&xas, entry);
1433 if (xa_track_free(xa))1439 if (xa_track_free(xa) && entry && !curr)
1434 xas_clear_mark(&xas, XA_FREE_MARK);1440 xas_clear_mark(&xas, XA_FREE_MARK);
1435 }1441 }
1436 } while (__xas_nomem(&xas, gfp));1442 } while (__xas_nomem(&xas, gfp));
@@ -1452,7 +1458,7 @@ EXPORT_SYMBOL(__xa_cmpxchg); · 15 unmodified lines
1452 *1458 *
1453 * Context: Any context. Expects xa_lock to be held on entry. May1459 * Context: Any context. Expects xa_lock to be held on entry. May
1454 * release and reacquire xa_lock if @gfp flags permit.1460 * release and reacquire xa_lock if @gfp flags permit.
1455 * Return: 0 if the store succeeded. -EEXIST if another entry was present.1461 * Return: 0 if the store succeeded. -EBUSY if another entry was present.
1456 * -ENOMEM if memory could not be allocated.1462 * -ENOMEM if memory could not be allocated.
1457 */1463 */
1458int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)1464int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
@@ -1472,7 +1478,7 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 13 unmodified lines
1472 if (xa_track_free(xa))1478 if (xa_track_free(xa))
1473 xas_clear_mark(&xas, XA_FREE_MARK);1479 xas_clear_mark(&xas, XA_FREE_MARK);
1474 } else {1480 } else {
1475 xas_set_err(&xas, -EEXIST);1481 xas_set_err(&xas, -EBUSY);
1476 }1482 }
1477 } while (__xas_nomem(&xas, gfp));1483 } while (__xas_nomem(&xas, gfp));
1478 1484
@@ -1480,42 +1486,6 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 1 unmodified line
1480}1486}
1481EXPORT_SYMBOL(__xa_insert);1487EXPORT_SYMBOL(__xa_insert);
1482 1488
1483/**
1484 * __xa_reserve() - Reserve this index in the XArray.
1485 * @xa: XArray.
1486 * @index: Index into array.
1487 * @gfp: Memory allocation flags.
1488 *
1489 * Ensures there is somewhere to store an entry at @index in the array.
1490 * If there is already something stored at @index, this function does
1491 * nothing. If there was nothing there, the entry is marked as reserved.
1492 * Loading from a reserved entry returns a %NULL pointer.
1493 *
1494 * If you do not use the entry that you have reserved, call xa_release()
1495 * or xa_erase() to free any unnecessary memory.
1496 *
1497 * Context: Any context. Expects the xa_lock to be held on entry. May
1498 * release the lock, sleep and reacquire the lock if the @gfp flags permit.
1499 * Return: 0 if the reservation succeeded or -ENOMEM if it failed.
1500 */
1501int __xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp)
1502{
1503 XA_STATE(xas, xa, index);
1504 void *curr;
1505
1506 do {
1507 curr = xas_load(&xas);
1508 if (!curr) {
1509 xas_store(&xas, XA_ZERO_ENTRY);
1510 if (xa_track_free(xa))
1511 xas_clear_mark(&xas, XA_FREE_MARK);
1512 }
1513 } while (__xas_nomem(&xas, gfp));
1514
1515 return xas_error(&xas);
1516}
1517EXPORT_SYMBOL(__xa_reserve);
1518
1519#ifdef CONFIG_XARRAY_MULTI1489#ifdef CONFIG_XARRAY_MULTI
1520static void xas_set_range(struct xa_state *xas, unsigned long first,1490static void xas_set_range(struct xa_state *xas, unsigned long first,
1521 unsigned long last)1491 unsigned long last)
@@ -1607,23 +1577,23 @@ EXPORT_SYMBOL(xa_store_range); · 85 unmodified lines
1607 * __xa_alloc() - Find somewhere to store this entry in the XArray.1577 * __xa_alloc() - Find somewhere to store this entry in the XArray.
1608 * @xa: XArray.1578 * @xa: XArray.
1609 * @id: Pointer to ID.1579 * @id: Pointer to ID.
1610 * @max: Maximum ID to allocate (inclusive).1580 * @limit: Range for allocated ID.
1611 * @entry: New entry.1581 * @entry: New entry.
1612 * @gfp: Memory allocation flags.1582 * @gfp: Memory allocation flags.
1613 *1583 *
1614 * Allocates an unused ID in the range specified by @id and @max.1584 * Finds an empty entry in @xa between @limit.min and @limit.max,
1615 * Updates the @id pointer with the index, then stores the entry at that1585 * stores the index into the @id pointer, then stores the entry at
1616 * index. A concurrent lookup will not see an uninitialised @id.1586 * that index. A concurrent lookup will not see an uninitialised @id.
1617 *1587 *
1618 * Context: Any context. Expects xa_lock to be held on entry. May1588 * Context: Any context. Expects xa_lock to be held on entry. May
1619 * release and reacquire xa_lock if @gfp flags permit.1589 * release and reacquire xa_lock if @gfp flags permit.
1620 * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if1590 * Return: 0 on success, -ENOMEM if memory could not be allocated or
1621 * there is no more space in the XArray.1591 * -EBUSY if there are no free entries in @limit.
1622 */1592 */
1623int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp)1593int __xa_alloc(struct xarray *xa, u32 *id, void *entry,
1594 struct xa_limit limit, gfp_t gfp)
1624{1595{
1625 XA_STATE(xas, xa, 0);1596 XA_STATE(xas, xa, 0);
1626 int err;
1627 1597
1628 if (WARN_ON_ONCE(xa_is_advanced(entry)))1598 if (WARN_ON_ONCE(xa_is_advanced(entry)))
1629 return -EINVAL;1599 return -EINVAL;
@@ -1634,21 +1604,70 @@ int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) · 4 unmodified lines
1634 entry = XA_ZERO_ENTRY;1604 entry = XA_ZERO_ENTRY;
1635 1605
1636 do {1606 do {
1637 xas.xa_index = *id;1607 xas.xa_index = limit.min;
1638 xas_find_marked(&xas, max, XA_FREE_MARK);1608 xas_find_marked(&xas, limit.max, XA_FREE_MARK);
1639 if (xas.xa_node == XAS_RESTART)1609 if (xas.xa_node == XAS_RESTART)
1640 xas_set_err(&xas, -ENOSPC);1610 xas_set_err(&xas, -EBUSY);
1611 else
1612 *id = xas.xa_index;
1641 xas_store(&xas, entry);1613 xas_store(&xas, entry);
1642 xas_clear_mark(&xas, XA_FREE_MARK);1614 xas_clear_mark(&xas, XA_FREE_MARK);
1643 } while (__xas_nomem(&xas, gfp));1615 } while (__xas_nomem(&xas, gfp));
1644 1616
1645 err = xas_error(&xas);1617 return xas_error(&xas);
1646 if (!err)
1647 *id = xas.xa_index;
1648 return err;
1649}1618}
1650EXPORT_SYMBOL(__xa_alloc);1619EXPORT_SYMBOL(__xa_alloc);
1651 1620
1621/**
1622 * __xa_alloc_cyclic() - Find somewhere to store this entry in the XArray.
1623 * @xa: XArray.
1624 * @id: Pointer to ID.
1625 * @entry: New entry.
1626 * @limit: Range of allocated ID.
1627 * @next: Pointer to next ID to allocate.
1628 * @gfp: Memory allocation flags.
1629 *
1630 * Finds an empty entry in @xa between @limit.min and @limit.max,
1631 * stores the index into the @id pointer, then stores the entry at
1632 * that index. A concurrent lookup will not see an uninitialised @id.
1633 * The search for an empty entry will start at @next and will wrap
1634 * around if necessary.
1635 *
1636 * Context: Any context. Expects xa_lock to be held on entry. May
1637 * release and reacquire xa_lock if @gfp flags permit.
1638 * Return: 0 if the allocation succeeded without wrapping. 1 if the
1639 * allocation succeeded after wrapping, -ENOMEM if memory could not be
1640 * allocated or -EBUSY if there are no free entries in @limit.
1641 */
1642int __xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
1643 struct xa_limit limit, u32 *next, gfp_t gfp)
1644{
1645 u32 min = limit.min;
1646 int ret;
1647
1648 limit.min = max(min, *next);
1649 ret = __xa_alloc(xa, id, entry, limit, gfp);
1650 if ((xa->xa_flags & XA_FLAGS_ALLOC_WRAPPED) && ret == 0) {
1651 xa->xa_flags &= ~XA_FLAGS_ALLOC_WRAPPED;
1652 ret = 1;
1653 }
1654
1655 if (ret < 0 && limit.min > min) {
1656 limit.min = min;
1657 ret = __xa_alloc(xa, id, entry, limit, gfp);
1658 if (ret == 0)
1659 ret = 1;
1660 }
1661
1662 if (ret >= 0) {
1663 *next = *id + 1;
1664 if (*next == 0)
1665 xa->xa_flags |= XA_FLAGS_ALLOC_WRAPPED;
1666 }
1667 return ret;
1668}
1669EXPORT_SYMBOL(__xa_alloc_cyclic);
1670
1652/**1671/**
1653 * __xa_set_mark() - Set this mark on this entry while locked.1672 * __xa_set_mark() - Set this mark on this entry while locked.
1654 * @xa: XArray.1673 * @xa: XArray.
@@ -1943,6 +1962,8 @@ void xa_destroy(struct xarray *xa) · 288 unmodified lines
1943 entry = xa_head_locked(xa);1962 entry = xa_head_locked(xa);
1944 RCU_INIT_POINTER(xa->xa_head, NULL);1963 RCU_INIT_POINTER(xa->xa_head, NULL);
1945 xas_init_marks(&xas);1964 xas_init_marks(&xas);
1965 if (xa_zero_busy(xa))
1966 xa_mark_clear(xa, XA_FREE_MARK);
1946 /* lockdep checks we're still holding the lock in xas_free_nodes() */1967 /* lockdep checks we're still holding the lock in xas_free_nodes() */
1947 if (xa_is_node(entry))1968 if (xa_is_node(entry))
1948 xas_free_nodes(&xas, xa_to_node(entry));1969 xas_free_nodes(&xas, xa_to_node(entry));

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t