Pull XArray updates from Matthew Wilcox: "This pull request changes the xa_alloc() API. I'm only aware of one subsystem that has started trying to use it, and we agree on the fixup as part of the merge. The xa_insert() error code also changed to match xa_alloc() (EEXIST to EBUSY), and I added xa_alloc_cyclic(). Beyond that, the usual bugfixes, optimisations and tweaking. I now have a git tree with all users of the radix tree and IDR converted over to the XArray that I'll be feeding to maintainers over the next few weeks" * tag 'xarray-5.1-rc1' of git://git.infradead.org/users/willy/linux-dax: XArray: Fix xa_reserve for 2-byte aligned entries XArray: Fix xa_erase of 2-byte aligned entries XArray: Use xa_cmpxchg to implement xa_reserve XArray: Fix xa_release in allocating arrays XArray: Mark xa_insert and xa_reserve as must_check XArray: Add cyclic allocation XArray: Redesign xa_alloc API XArray: Add support for 1s-based allocation XArray: Change xa_insert to return -EBUSY XArray: Update xa_erase family descriptions XArray tests: RCU lock prohibits GFP_KERNEL
Unified · Split
| @@ -85,7 +85,7 @@ which was at that index; if it returns the same entry which was passed as · 84 unmodified lines | ||
| 85 | 85 | |
| 86 | 86 | If you want to only store a new entry to an index if the current entry |
| 87 | 87 | at that index is ``NULL``, you can use :c:func:`xa_insert` which |
| 88 | −returns ``-EEXIST`` if the entry is not empty. | |
| 88 | +returns ``-EBUSY`` if the entry is not empty. | |
| 89 | 89 | |
| 90 | 90 | You can enquire whether a mark is set on an entry by using |
| 91 | 91 | :c:func:`xa_get_mark`. If the entry is not ``NULL``, you can set a mark |
| @@ -131,17 +131,23 @@ If you use :c:func:`DEFINE_XARRAY_ALLOC` to define the XArray, or · 39 unmodified lines | ||
| 131 | 131 | initialise it by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`, |
| 132 | 132 | the XArray changes to track whether entries are in use or not. |
| 133 | 133 | |
| 134 | −You can call :c:func:`xa_alloc` to store the entry at any unused index | |
| 134 | +You can call :c:func:`xa_alloc` to store the entry at an unused index | |
| 135 | 135 | in the XArray. If you need to modify the array from interrupt context, |
| 136 | 136 | you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable |
| 137 | 137 | interrupts while allocating the ID. |
| 138 | 138 | |
| 139 | −Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert` | |
| 140 | −will mark the entry as being allocated. Unlike a normal XArray, storing | |
| 139 | +Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert` will | |
| 140 | +also mark the entry as being allocated. Unlike a normal XArray, storing | |
| 141 | 141 | ``NULL`` will mark the entry as being in use, like :c:func:`xa_reserve`. |
| 142 | 142 | To free an entry, use :c:func:`xa_erase` (or :c:func:`xa_release` if |
| 143 | 143 | you only want to free the entry if it's ``NULL``). |
| 144 | 144 | |
| 145 | +By default, the lowest free entry is allocated starting from 0. If you | |
| 146 | +want to allocate entries starting at 1, it is more efficient to use | |
| 147 | +:c:func:`DEFINE_XARRAY_ALLOC1` or ``XA_FLAGS_ALLOC1``. If you want to | |
| 148 | +allocate IDs up to a maximum, then wrap back around to the lowest free | |
| 149 | +ID, you can use :c:func:`xa_alloc_cyclic`. | |
| 150 | + | |
| 145 | 151 | You cannot use ``XA_MARK_0`` with an allocating XArray as this mark |
| 146 | 152 | is used to track whether an entry is free or not. The other marks are |
| 147 | 153 | available for your use. |
| @@ -209,7 +215,6 @@ Assumes xa_lock held on entry: · 61 unmodified lines | ||
| 209 | 215 | * :c:func:`__xa_erase` |
| 210 | 216 | * :c:func:`__xa_cmpxchg` |
| 211 | 217 | * :c:func:`__xa_alloc` |
| 212 | − * :c:func:`__xa_reserve` | |
| 213 | 218 | * :c:func:`__xa_set_mark` |
| 214 | 219 | * :c:func:`__xa_clear_mark` |
| 215 | 220 | |
| @@ -668,19 +668,10 @@ static int assign_name(struct ib_device *device, const char *name) · 667 unmodified lines | ||
| 668 | 668 | } |
| 669 | 669 | strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); |
| 670 | 670 | |
| 671 | − /* Cyclically allocate a user visible ID for the device */ | |
| 672 | − device->index = last_id; | |
| 673 | − ret = xa_alloc(&devices, &device->index, INT_MAX, device, GFP_KERNEL); | |
| 674 | − if (ret == -ENOSPC) { | |
| 675 | − device->index = 0; | |
| 676 | − ret = xa_alloc(&devices, &device->index, INT_MAX, device, | |
| 677 | − GFP_KERNEL); | |
| 678 | − } | |
| 679 | − if (ret) | |
| 680 | − goto out; | |
| 681 | − last_id = device->index + 1; | |
| 682 | − | |
| 683 | − ret = 0; | |
| 671 | + ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b, | |
| 672 | + &last_id, GFP_KERNEL); | |
| 673 | + if (ret > 0) | |
| 674 | + ret = 0; | |
| 684 | 675 | |
| 685 | 676 | out: |
| 686 | 677 | up_write(&devices_rwsem); |
| @@ -1059,14 +1050,15 @@ static int assign_client_id(struct ib_client *client) · 372 unmodified lines | ||
| 1059 | 1050 | * to get the LIFO order. The extra linked list can go away if xarray |
| 1060 | 1051 | * learns to reverse iterate. |
| 1061 | 1052 | */ |
| 1062 | − if (list_empty(&client_list)) | |
| 1053 | + if (list_empty(&client_list)) { | |
| 1063 | 1054 | client->client_id = 0; |
| 1064 | − else | |
| 1065 | − client->client_id = | |
| 1066 | − list_last_entry(&client_list, struct ib_client, list) | |
| 1067 | − ->client_id; | |
| 1068 | − ret = xa_alloc(&clients, &client->client_id, INT_MAX, client, | |
| 1069 | − GFP_KERNEL); | |
| 1055 | + } else { | |
| 1056 | + struct ib_client *last; | |
| 1057 | + | |
| 1058 | + last = list_last_entry(&client_list, struct ib_client, list); | |
| 1059 | + client->client_id = last->client_id + 1; | |
| 1060 | + } | |
| 1061 | + ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL); | |
| 1070 | 1062 | if (ret) |
| 1071 | 1063 | goto out; |
| 1072 | 1064 | |
| @@ -13,28 +13,6 @@ · 12 unmodified lines | ||
| 13 | 13 | #include "cma_priv.h" |
| 14 | 14 | #include "restrack.h" |
| 15 | 15 | |
| 16 | −static int rt_xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, | |
| 17 | − u32 *next) | |
| 18 | −{ | |
| 19 | − int err; | |
| 20 | − | |
| 21 | − *id = *next; | |
| 22 | − if (*next == U32_MAX) | |
| 23 | − *id = 0; | |
| 24 | − | |
| 25 | − xa_lock(xa); | |
| 26 | − err = __xa_alloc(xa, id, U32_MAX, entry, GFP_KERNEL); | |
| 27 | − if (err && *next != U32_MAX) { | |
| 28 | − *id = 0; | |
| 29 | − err = __xa_alloc(xa, id, *next, entry, GFP_KERNEL); | |
| 30 | − } | |
| 31 | − | |
| 32 | − if (!err) | |
| 33 | − *next = *id + 1; | |
| 34 | − xa_unlock(xa); | |
| 35 | − return err; | |
| 36 | −} | |
| 37 | − | |
| 38 | 16 | /** |
| 39 | 17 | * rdma_restrack_init() - initialize and allocate resource tracking |
| 40 | 18 | * @dev: IB device |
| @@ -226,7 +204,8 @@ static void rdma_restrack_add(struct rdma_restrack_entry *res) · 185 unmodified lines | ||
| 226 | 204 | kref_init(&res->kref); |
| 227 | 205 | init_completion(&res->comp); |
| 228 | 206 | if (res->type != RDMA_RESTRACK_QP) |
| 229 | − ret = rt_xa_alloc_cyclic(&rt->xa, &res->id, res, &rt->next_id); | |
| 207 | + ret = xa_alloc_cyclic(&rt->xa, &res->id, res, xa_limit_32b, | |
| 208 | + &rt->next_id, GFP_KERNEL); | |
| 230 | 209 | else { |
| 231 | 210 | /* Special case to ensure that LQPN points to right QP */ |
| 232 | 211 | struct ib_qp *qp = container_of(res, struct ib_qp, res); |
| @@ -189,7 +189,7 @@ retry: · 188 unmodified lines | ||
| 189 | 189 | */ |
| 190 | 190 | if (!err) |
| 191 | 191 | return 0; |
| 192 | − else if (err != -EEXIST) | |
| 192 | + else if (err != -EBUSY) | |
| 193 | 193 | goto failed_unlock; |
| 194 | 194 | |
| 195 | 195 | err = invalidate_inode_pages2_range(btnc, newkey, newkey); |
| @@ -131,6 +131,12 @@ static inline unsigned int xa_pointer_tag(void *entry) · 130 unmodified lines | ||
| 131 | 131 | * xa_mk_internal() - Create an internal entry. |
| 132 | 132 | * @v: Value to turn into an internal entry. |
| 133 | 133 | * |
| 134 | + * Internal entries are used for a number of purposes. Entries 0-255 are | |
| 135 | + * used for sibling entries (only 0-62 are used by the current code). 256 | |
| 136 | + * is used for the retry entry. 257 is used for the reserved / zero entry. | |
| 137 | + * Negative internal entries are used to represent errnos. Node pointers | |
| 138 | + * are also tagged as internal entries in some situations. | |
| 139 | + * | |
| 134 | 140 | * Context: Any context. |
| 135 | 141 | * Return: An XArray internal entry corresponding to this value. |
| 136 | 142 | */ |
| @@ -163,6 +169,22 @@ static inline bool xa_is_internal(const void *entry) · 26 unmodified lines | ||
| 163 | 169 | return ((unsigned long)entry & 3) == 2; |
| 164 | 170 | } |
| 165 | 171 | |
| 172 | +#define XA_ZERO_ENTRY xa_mk_internal(257) | |
| 173 | + | |
| 174 | +/** | |
| 175 | + * xa_is_zero() - Is the entry a zero entry? | |
| 176 | + * @entry: Entry retrieved from the XArray | |
| 177 | + * | |
| 178 | + * The normal API will return NULL as the contents of a slot containing | |
| 179 | + * a zero entry. You can only see zero entries by using the advanced API. | |
| 180 | + * | |
| 181 | + * Return: %true if the entry is a zero entry. | |
| 182 | + */ | |
| 183 | +static inline bool xa_is_zero(const void *entry) | |
| 184 | +{ | |
| 185 | + return unlikely(entry == XA_ZERO_ENTRY); | |
| 186 | +} | |
| 187 | + | |
| 166 | 188 | /** |
| 167 | 189 | * xa_is_err() - Report whether an XArray operation returned an error |
| 168 | 190 | * @entry: Result from calling an XArray function |
| @@ -200,6 +222,27 @@ static inline int xa_err(void *entry) · 31 unmodified lines | ||
| 200 | 222 | return 0; |
| 201 | 223 | } |
| 202 | 224 | |
| 225 | +/** | |
| 226 | + * struct xa_limit - Represents a range of IDs. | |
| 227 | + * @min: The lowest ID to allocate (inclusive). | |
| 228 | + * @max: The maximum ID to allocate (inclusive). | |
| 229 | + * | |
| 230 | + * This structure is used either directly or via the XA_LIMIT() macro | |
| 231 | + * to communicate the range of IDs that are valid for allocation. | |
| 232 | + * Two common ranges are predefined for you: | |
| 233 | + * * xa_limit_32b - [0 - UINT_MAX] | |
| 234 | + * * xa_limit_31b - [0 - INT_MAX] | |
| 235 | + */ | |
| 236 | +struct xa_limit { | |
| 237 | + u32 max; | |
| 238 | + u32 min; | |
| 239 | +}; | |
| 240 | + | |
| 241 | +#define XA_LIMIT(_min, _max) (struct xa_limit) { .min = _min, .max = _max } | |
| 242 | + | |
| 243 | +#define xa_limit_32b XA_LIMIT(0, UINT_MAX) | |
| 244 | +#define xa_limit_31b XA_LIMIT(0, INT_MAX) | |
| 245 | + | |
| 203 | 246 | typedef unsigned __bitwise xa_mark_t; |
| 204 | 247 | #define XA_MARK_0 ((__force xa_mark_t)0U) |
| 205 | 248 | #define XA_MARK_1 ((__force xa_mark_t)1U) |
| @@ -220,10 +263,14 @@ enum xa_lock_type { · 14 unmodified lines | ||
| 220 | 263 | #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) |
| 221 | 264 | #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) |
| 222 | 265 | #define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U) |
| 266 | +#define XA_FLAGS_ZERO_BUSY ((__force gfp_t)8U) | |
| 267 | +#define XA_FLAGS_ALLOC_WRAPPED ((__force gfp_t)16U) | |
| 223 | 268 | #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ |
| 224 | 269 | (__force unsigned)(mark))) |
| 225 | 270 | |
| 271 | +/* ALLOC is for a normal 0-based alloc. ALLOC1 is for an 1-based alloc */ | |
| 226 | 272 | #define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK)) |
| 273 | +#define XA_FLAGS_ALLOC1 (XA_FLAGS_TRACK_FREE | XA_FLAGS_ZERO_BUSY) | |
| 227 | 274 | |
| 228 | 275 | /** |
| 229 | 276 | * struct xarray - The anchor of the XArray. |
| @@ -279,7 +326,7 @@ struct xarray { · 49 unmodified lines | ||
| 279 | 326 | #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) |
| 280 | 327 | |
| 281 | 328 | /** |
| 282 | − * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs. | |
| 329 | + * DEFINE_XARRAY_ALLOC() - Define an XArray which allocates IDs starting at 0. | |
| 283 | 330 | * @name: A string that names your XArray. |
| 284 | 331 | * |
| 285 | 332 | * This is intended for file scope definitions of allocating XArrays. |
| @@ -287,6 +334,15 @@ struct xarray { · 1 unmodified line | ||
| 287 | 334 | */ |
| 288 | 335 | #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC) |
| 289 | 336 | |
| 337 | +/** | |
| 338 | + * DEFINE_XARRAY_ALLOC1() - Define an XArray which allocates IDs starting at 1. | |
| 339 | + * @name: A string that names your XArray. | |
| 340 | + * | |
| 341 | + * This is intended for file scope definitions of allocating XArrays. | |
| 342 | + * See also DEFINE_XARRAY(). | |
| 343 | + */ | |
| 344 | +#define DEFINE_XARRAY_ALLOC1(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC1) | |
| 345 | + | |
| 290 | 346 | void *xa_load(struct xarray *, unsigned long index); |
| 291 | 347 | void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| 292 | 348 | void *xa_erase(struct xarray *, unsigned long index); |
| @@ -463,9 +519,12 @@ void *__xa_erase(struct xarray *, unsigned long index); · 170 unmodified lines | ||
| 463 | 519 | void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| 464 | 520 | void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, |
| 465 | 521 | void *entry, gfp_t); |
| 466 | −int __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t); | |
| 467 | −int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t); | |
| 468 | −int __xa_reserve(struct xarray *, unsigned long index, gfp_t); | |
| 522 | +int __must_check __xa_insert(struct xarray *, unsigned long index, | |
| 523 | + void *entry, gfp_t); | |
| 524 | +int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry, | |
| 525 | + struct xa_limit, gfp_t); | |
| 526 | +int __must_check __xa_alloc_cyclic(struct xarray *, u32 *id, void *entry, | |
| 527 | + struct xa_limit, u32 *next, gfp_t); | |
| 469 | 528 | void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 470 | 529 | void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 471 | 530 | |
| @@ -526,9 +585,9 @@ static inline void *xa_store_irq(struct xarray *xa, unsigned long index, · 54 unmodified lines | ||
| 526 | 585 | * @xa: XArray. |
| 527 | 586 | * @index: Index of entry. |
| 528 | 587 | * |
| 529 | − * This function is the equivalent of calling xa_store() with %NULL as | |
| 530 | − * the third argument. The XArray does not need to allocate memory, so | |
| 531 | − * the user does not need to provide GFP flags. | |
| 588 | + * After this function returns, loading from @index will return %NULL. | |
| 589 | + * If the index is part of a multi-index entry, all indices will be erased | |
| 590 | + * and none of the entries will be part of a multi-index entry. | |
| 532 | 591 | * |
| 533 | 592 | * Context: Any context. Takes and releases the xa_lock while |
| 534 | 593 | * disabling softirqs. |
| @@ -550,9 +609,9 @@ static inline void *xa_erase_bh(struct xarray *xa, unsigned long index) · 15 unmodified lines | ||
| 550 | 609 | * @xa: XArray. |
| 551 | 610 | * @index: Index of entry. |
| 552 | 611 | * |
| 553 | − * This function is the equivalent of calling xa_store() with %NULL as | |
| 554 | − * the third argument. The XArray does not need to allocate memory, so | |
| 555 | − * the user does not need to provide GFP flags. | |
| 612 | + * After this function returns, loading from @index will return %NULL. | |
| 613 | + * If the index is part of a multi-index entry, all indices will be erased | |
| 614 | + * and none of the entries will be part of a multi-index entry. | |
| 556 | 615 | * |
| 557 | 616 | * Context: Process context. Takes and releases the xa_lock while |
| 558 | 617 | * disabling interrupts. |
| @@ -664,11 +723,11 @@ static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index, · 105 unmodified lines | ||
| 664 | 723 | * |
| 665 | 724 | * Context: Any context. Takes and releases the xa_lock. May sleep if |
| 666 | 725 | * the @gfp flags permit. |
| 667 | − * Return: 0 if the store succeeded. -EEXIST if another entry was present. | |
| 726 | + * Return: 0 if the store succeeded. -EBUSY if another entry was present. | |
| 668 | 727 | * -ENOMEM if memory could not be allocated. |
| 669 | 728 | */ |
| 670 | −static inline int xa_insert(struct xarray *xa, unsigned long index, | |
| 671 | − void *entry, gfp_t gfp) | |
| 729 | +static inline int __must_check xa_insert(struct xarray *xa, | |
| 730 | + unsigned long index, void *entry, gfp_t gfp) | |
| 672 | 731 | { |
| 673 | 732 | int err; |
| 674 | 733 | |
| @@ -693,11 +752,11 @@ static inline int xa_insert(struct xarray *xa, unsigned long index, · 18 unmodified lines | ||
| 693 | 752 | * |
| 694 | 753 | * Context: Any context. Takes and releases the xa_lock while |
| 695 | 754 | * disabling softirqs. May sleep if the @gfp flags permit. |
| 696 | − * Return: 0 if the store succeeded. -EEXIST if another entry was present. | |
| 755 | + * Return: 0 if the store succeeded. -EBUSY if another entry was present. | |
| 697 | 756 | * -ENOMEM if memory could not be allocated. |
| 698 | 757 | */ |
| 699 | −static inline int xa_insert_bh(struct xarray *xa, unsigned long index, | |
| 700 | − void *entry, gfp_t gfp) | |
| 758 | +static inline int __must_check xa_insert_bh(struct xarray *xa, | |
| 759 | + unsigned long index, void *entry, gfp_t gfp) | |
| 701 | 760 | { |
| 702 | 761 | int err; |
| 703 | 762 | |
| @@ -722,11 +781,11 @@ static inline int xa_insert_bh(struct xarray *xa, unsigned long index, · 18 unmodified lines | ||
| 722 | 781 | * |
| 723 | 782 | * Context: Process context. Takes and releases the xa_lock while |
| 724 | 783 | * disabling interrupts. May sleep if the @gfp flags permit. |
| 725 | − * Return: 0 if the store succeeded. -EEXIST if another entry was present. | |
| 784 | + * Return: 0 if the store succeeded. -EBUSY if another entry was present. | |
| 726 | 785 | * -ENOMEM if memory could not be allocated. |
| 727 | 786 | */ |
| 728 | −static inline int xa_insert_irq(struct xarray *xa, unsigned long index, | |
| 729 | − void *entry, gfp_t gfp) | |
| 787 | +static inline int __must_check xa_insert_irq(struct xarray *xa, | |
| 788 | + unsigned long index, void *entry, gfp_t gfp) | |
| 730 | 789 | { |
| 731 | 790 | int err; |
| 732 | 791 | |
| @@ -741,26 +800,26 @@ static inline int xa_insert_irq(struct xarray *xa, unsigned long index, · 8 unmodified lines | ||
| 741 | 800 | * xa_alloc() - Find somewhere to store this entry in the XArray. |
| 742 | 801 | * @xa: XArray. |
| 743 | 802 | * @id: Pointer to ID. |
| 744 | − * @max: Maximum ID to allocate (inclusive). | |
| 745 | 803 | * @entry: New entry. |
| 804 | + * @limit: Range of ID to allocate. | |
| 746 | 805 | * @gfp: Memory allocation flags. |
| 747 | 806 | * |
| 748 | − * Allocates an unused ID in the range specified by @id and @max. | |
| 749 | − * Updates the @id pointer with the index, then stores the entry at that | |
| 750 | − * index. A concurrent lookup will not see an uninitialised @id. | |
| 807 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 808 | + * stores the index into the @id pointer, then stores the entry at | |
| 809 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 751 | 810 | * |
| 752 | − * Context: Process context. Takes and releases the xa_lock. May sleep if | |
| 811 | + * Context: Any context. Takes and releases the xa_lock. May sleep if | |
| 753 | 812 | * the @gfp flags permit. |
| 754 | − * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | |
| 755 | − * there is no more space in the XArray. | |
| 813 | + * Return: 0 on success, -ENOMEM if memory could not be allocated or | |
| 814 | + * -EBUSY if there are no free entries in @limit. | |
| 756 | 815 | */ |
| 757 | −static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, | |
| 758 | − gfp_t gfp) | |
| 816 | +static inline __must_check int xa_alloc(struct xarray *xa, u32 *id, | |
| 817 | + void *entry, struct xa_limit limit, gfp_t gfp) | |
| 759 | 818 | { |
| 760 | 819 | int err; |
| 761 | 820 | |
| 762 | 821 | xa_lock(xa); |
| 763 | − err = __xa_alloc(xa, id, max, entry, gfp); | |
| 822 | + err = __xa_alloc(xa, id, entry, limit, gfp); | |
| 764 | 823 | xa_unlock(xa); |
| 765 | 824 | |
| 766 | 825 | return err; |
| @@ -770,26 +829,26 @@ static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines | ||
| 770 | 829 | * xa_alloc_bh() - Find somewhere to store this entry in the XArray. |
| 771 | 830 | * @xa: XArray. |
| 772 | 831 | * @id: Pointer to ID. |
| 773 | − * @max: Maximum ID to allocate (inclusive). | |
| 774 | 832 | * @entry: New entry. |
| 833 | + * @limit: Range of ID to allocate. | |
| 775 | 834 | * @gfp: Memory allocation flags. |
| 776 | 835 | * |
| 777 | − * Allocates an unused ID in the range specified by @id and @max. | |
| 778 | − * Updates the @id pointer with the index, then stores the entry at that | |
| 779 | − * index. A concurrent lookup will not see an uninitialised @id. | |
| 836 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 837 | + * stores the index into the @id pointer, then stores the entry at | |
| 838 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 780 | 839 | * |
| 781 | 840 | * Context: Any context. Takes and releases the xa_lock while |
| 782 | 841 | * disabling softirqs. May sleep if the @gfp flags permit. |
| 783 | − * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | |
| 784 | − * there is no more space in the XArray. | |
| 842 | + * Return: 0 on success, -ENOMEM if memory could not be allocated or | |
| 843 | + * -EBUSY if there are no free entries in @limit. | |
| 785 | 844 | */ |
| 786 | −static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, | |
| 787 | − gfp_t gfp) | |
| 845 | +static inline int __must_check xa_alloc_bh(struct xarray *xa, u32 *id, | |
| 846 | + void *entry, struct xa_limit limit, gfp_t gfp) | |
| 788 | 847 | { |
| 789 | 848 | int err; |
| 790 | 849 | |
| 791 | 850 | xa_lock_bh(xa); |
| 792 | − err = __xa_alloc(xa, id, max, entry, gfp); | |
| 851 | + err = __xa_alloc(xa, id, entry, limit, gfp); | |
| 793 | 852 | xa_unlock_bh(xa); |
| 794 | 853 | |
| 795 | 854 | return err; |
| @@ -799,26 +858,125 @@ static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines | ||
| 799 | 858 | * xa_alloc_irq() - Find somewhere to store this entry in the XArray. |
| 800 | 859 | * @xa: XArray. |
| 801 | 860 | * @id: Pointer to ID. |
| 802 | − * @max: Maximum ID to allocate (inclusive). | |
| 803 | 861 | * @entry: New entry. |
| 862 | + * @limit: Range of ID to allocate. | |
| 804 | 863 | * @gfp: Memory allocation flags. |
| 805 | 864 | * |
| 806 | − * Allocates an unused ID in the range specified by @id and @max. | |
| 807 | − * Updates the @id pointer with the index, then stores the entry at that | |
| 808 | − * index. A concurrent lookup will not see an uninitialised @id. | |
| 865 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 866 | + * stores the index into the @id pointer, then stores the entry at | |
| 867 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 809 | 868 | * |
| 810 | 869 | * Context: Process context. Takes and releases the xa_lock while |
| 811 | 870 | * disabling interrupts. May sleep if the @gfp flags permit. |
| 812 | − * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | |
| 813 | − * there is no more space in the XArray. | |
| 871 | + * Return: 0 on success, -ENOMEM if memory could not be allocated or | |
| 872 | + * -EBUSY if there are no free entries in @limit. | |
| 814 | 873 | */ |
| 815 | −static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, | |
| 816 | − gfp_t gfp) | |
| 874 | +static inline int __must_check xa_alloc_irq(struct xarray *xa, u32 *id, | |
| 875 | + void *entry, struct xa_limit limit, gfp_t gfp) | |
| 817 | 876 | { |
| 818 | 877 | int err; |
| 819 | 878 | |
| 820 | 879 | xa_lock_irq(xa); |
| 821 | − err = __xa_alloc(xa, id, max, entry, gfp); | |
| 880 | + err = __xa_alloc(xa, id, entry, limit, gfp); | |
| 881 | + xa_unlock_irq(xa); | |
| 882 | + | |
| 883 | + return err; | |
| 884 | +} | |
| 885 | + | |
| 886 | +/** | |
| 887 | + * xa_alloc_cyclic() - Find somewhere to store this entry in the XArray. | |
| 888 | + * @xa: XArray. | |
| 889 | + * @id: Pointer to ID. | |
| 890 | + * @entry: New entry. | |
| 891 | + * @limit: Range of allocated ID. | |
| 892 | + * @next: Pointer to next ID to allocate. | |
| 893 | + * @gfp: Memory allocation flags. | |
| 894 | + * | |
| 895 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 896 | + * stores the index into the @id pointer, then stores the entry at | |
| 897 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 898 | + * The search for an empty entry will start at @next and will wrap | |
| 899 | + * around if necessary. | |
| 900 | + * | |
| 901 | + * Context: Any context. Takes and releases the xa_lock. May sleep if | |
| 902 | + * the @gfp flags permit. | |
| 903 | + * Return: 0 if the allocation succeeded without wrapping. 1 if the | |
| 904 | + * allocation succeeded after wrapping, -ENOMEM if memory could not be | |
| 905 | + * allocated or -EBUSY if there are no free entries in @limit. | |
| 906 | + */ | |
| 907 | +static inline int xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, | |
| 908 | + struct xa_limit limit, u32 *next, gfp_t gfp) | |
| 909 | +{ | |
| 910 | + int err; | |
| 911 | + | |
| 912 | + xa_lock(xa); | |
| 913 | + err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | |
| 914 | + xa_unlock(xa); | |
| 915 | + | |
| 916 | + return err; | |
| 917 | +} | |
| 918 | + | |
| 919 | +/** | |
| 920 | + * xa_alloc_cyclic_bh() - Find somewhere to store this entry in the XArray. | |
| 921 | + * @xa: XArray. | |
| 922 | + * @id: Pointer to ID. | |
| 923 | + * @entry: New entry. | |
| 924 | + * @limit: Range of allocated ID. | |
| 925 | + * @next: Pointer to next ID to allocate. | |
| 926 | + * @gfp: Memory allocation flags. | |
| 927 | + * | |
| 928 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 929 | + * stores the index into the @id pointer, then stores the entry at | |
| 930 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 931 | + * The search for an empty entry will start at @next and will wrap | |
| 932 | + * around if necessary. | |
| 933 | + * | |
| 934 | + * Context: Any context. Takes and releases the xa_lock while | |
| 935 | + * disabling softirqs. May sleep if the @gfp flags permit. | |
| 936 | + * Return: 0 if the allocation succeeded without wrapping. 1 if the | |
| 937 | + * allocation succeeded after wrapping, -ENOMEM if memory could not be | |
| 938 | + * allocated or -EBUSY if there are no free entries in @limit. | |
| 939 | + */ | |
| 940 | +static inline int xa_alloc_cyclic_bh(struct xarray *xa, u32 *id, void *entry, | |
| 941 | + struct xa_limit limit, u32 *next, gfp_t gfp) | |
| 942 | +{ | |
| 943 | + int err; | |
| 944 | + | |
| 945 | + xa_lock_bh(xa); | |
| 946 | + err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | |
| 947 | + xa_unlock_bh(xa); | |
| 948 | + | |
| 949 | + return err; | |
| 950 | +} | |
| 951 | + | |
| 952 | +/** | |
| 953 | + * xa_alloc_cyclic_irq() - Find somewhere to store this entry in the XArray. | |
| 954 | + * @xa: XArray. | |
| 955 | + * @id: Pointer to ID. | |
| 956 | + * @entry: New entry. | |
| 957 | + * @limit: Range of allocated ID. | |
| 958 | + * @next: Pointer to next ID to allocate. | |
| 959 | + * @gfp: Memory allocation flags. | |
| 960 | + * | |
| 961 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 962 | + * stores the index into the @id pointer, then stores the entry at | |
| 963 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 964 | + * The search for an empty entry will start at @next and will wrap | |
| 965 | + * around if necessary. | |
| 966 | + * | |
| 967 | + * Context: Process context. Takes and releases the xa_lock while | |
| 968 | + * disabling interrupts. May sleep if the @gfp flags permit. | |
| 969 | + * Return: 0 if the allocation succeeded without wrapping. 1 if the | |
| 970 | + * allocation succeeded after wrapping, -ENOMEM if memory could not be | |
| 971 | + * allocated or -EBUSY if there are no free entries in @limit. | |
| 972 | + */ | |
| 973 | +static inline int xa_alloc_cyclic_irq(struct xarray *xa, u32 *id, void *entry, | |
| 974 | + struct xa_limit limit, u32 *next, gfp_t gfp) | |
| 975 | +{ | |
| 976 | + int err; | |
| 977 | + | |
| 978 | + xa_lock_irq(xa); | |
| 979 | + err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | |
| 822 | 980 | xa_unlock_irq(xa); |
| 823 | 981 | |
| 824 | 982 | return err; |
| @@ -842,16 +1000,10 @@ static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, · 17 unmodified lines | ||
| 842 | 1000 | * May sleep if the @gfp flags permit. |
| 843 | 1001 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 844 | 1002 | */ |
| 845 | −static inline | |
| 1003 | +static inline __must_check | |
| 846 | 1004 | int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 847 | 1005 | { |
| 848 | − int ret; | |
| 849 | − | |
| 850 | − xa_lock(xa); | |
| 851 | − ret = __xa_reserve(xa, index, gfp); | |
| 852 | − xa_unlock(xa); | |
| 853 | − | |
| 854 | − return ret; | |
| 1006 | + return xa_err(xa_cmpxchg(xa, index, NULL, XA_ZERO_ENTRY, gfp)); | |
| 855 | 1007 | } |
| 856 | 1008 | |
| 857 | 1009 | /** |
| @@ -866,16 +1018,10 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines | ||
| 866 | 1018 | * disabling softirqs. |
| 867 | 1019 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 868 | 1020 | */ |
| 869 | −static inline | |
| 1021 | +static inline __must_check | |
| 870 | 1022 | int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 871 | 1023 | { |
| 872 | − int ret; | |
| 873 | − | |
| 874 | − xa_lock_bh(xa); | |
| 875 | − ret = __xa_reserve(xa, index, gfp); | |
| 876 | − xa_unlock_bh(xa); | |
| 877 | − | |
| 878 | − return ret; | |
| 1024 | + return xa_err(xa_cmpxchg_bh(xa, index, NULL, XA_ZERO_ENTRY, gfp)); | |
| 879 | 1025 | } |
| 880 | 1026 | |
| 881 | 1027 | /** |
| @@ -890,16 +1036,10 @@ int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines | ||
| 890 | 1036 | * disabling interrupts. |
| 891 | 1037 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 892 | 1038 | */ |
| 893 | −static inline | |
| 1039 | +static inline __must_check | |
| 894 | 1040 | int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 895 | 1041 | { |
| 896 | − int ret; | |
| 897 | − | |
| 898 | − xa_lock_irq(xa); | |
| 899 | − ret = __xa_reserve(xa, index, gfp); | |
| 900 | − xa_unlock_irq(xa); | |
| 901 | − | |
| 902 | − return ret; | |
| 1042 | + return xa_err(xa_cmpxchg_irq(xa, index, NULL, XA_ZERO_ENTRY, gfp)); | |
| 903 | 1043 | } |
| 904 | 1044 | |
| 905 | 1045 | /** |
| @@ -913,7 +1053,7 @@ int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) · 7 unmodified lines | ||
| 913 | 1053 | */ |
| 914 | 1054 | static inline void xa_release(struct xarray *xa, unsigned long index) |
| 915 | 1055 | { |
| 916 | − xa_cmpxchg(xa, index, NULL, NULL, 0); | |
| 1056 | + xa_cmpxchg(xa, index, XA_ZERO_ENTRY, NULL, 0); | |
| 917 | 1057 | } |
| 918 | 1058 | |
| 919 | 1059 | /* Everything below here is the Advanced API. Proceed with caution. */ |
| @@ -1073,18 +1213,6 @@ static inline bool xa_is_sibling(const void *entry) · 153 unmodified lines | ||
| 1073 | 1213 | } |
| 1074 | 1214 | |
| 1075 | 1215 | #define XA_RETRY_ENTRY xa_mk_internal(256) |
| 1076 | −#define XA_ZERO_ENTRY xa_mk_internal(257) | |
| 1077 | − | |
| 1078 | −/** | |
| 1079 | − * xa_is_zero() - Is the entry a zero entry? | |
| 1080 | − * @entry: Entry retrieved from the XArray | |
| 1081 | − * | |
| 1082 | − * Return: %true if the entry is a zero entry. | |
| 1083 | − */ | |
| 1084 | −static inline bool xa_is_zero(const void *entry) | |
| 1085 | −{ | |
| 1086 | − return unlikely(entry == XA_ZERO_ENTRY); | |
| 1087 | −} | |
| 1088 | 1216 | |
| 1089 | 1217 | /** |
| 1090 | 1218 | * xa_is_retry() - Is the entry a retry entry? |
| @@ -40,9 +40,9 @@ static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp) · 39 unmodified lines | ||
| 40 | 40 | |
| 41 | 41 | static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 42 | 42 | { |
| 43 | − u32 id = 0; | |
| 43 | + u32 id; | |
| 44 | 44 | |
| 45 | − XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, xa_mk_index(index), | |
| 45 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(index), xa_limit_32b, | |
| 46 | 46 | gfp) != 0); |
| 47 | 47 | XA_BUG_ON(xa, id != index); |
| 48 | 48 | } |
| @@ -107,8 +107,11 @@ static noinline void check_xas_retry(struct xarray *xa) · 58 unmodified lines | ||
| 107 | 107 | XA_BUG_ON(xa, xas.xa_node != XAS_RESTART); |
| 108 | 108 | XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0)); |
| 109 | 109 | XA_BUG_ON(xa, xas.xa_node != NULL); |
| 110 | + rcu_read_unlock(); | |
| 110 | 111 | |
| 111 | 112 | XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL); |
| 113 | + | |
| 114 | + rcu_read_lock(); | |
| 112 | 115 | XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas))); |
| 113 | 116 | xas.xa_node = XAS_RESTART; |
| 114 | 117 | XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0)); |
| @@ -343,7 +346,7 @@ static noinline void check_cmpxchg(struct xarray *xa) · 228 unmodified lines | ||
| 343 | 346 | |
| 344 | 347 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 345 | 348 | XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL); |
| 346 | − XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST); | |
| 349 | + XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EBUSY); | |
| 347 | 350 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS); |
| 348 | 351 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS); |
| 349 | 352 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE); |
| @@ -358,46 +361,65 @@ static noinline void check_reserve(struct xarray *xa) · 8 unmodified lines | ||
| 358 | 361 | { |
| 359 | 362 | void *entry; |
| 360 | 363 | unsigned long index; |
| 364 | + int count; | |
| 361 | 365 | |
| 362 | 366 | /* An array with a reserved entry is not empty */ |
| 363 | 367 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 364 | − xa_reserve(xa, 12345678, GFP_KERNEL); | |
| 368 | + XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); | |
| 365 | 369 | XA_BUG_ON(xa, xa_empty(xa)); |
| 366 | 370 | XA_BUG_ON(xa, xa_load(xa, 12345678)); |
| 367 | 371 | xa_release(xa, 12345678); |
| 368 | 372 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 369 | 373 | |
| 370 | 374 | /* Releasing a used entry does nothing */ |
| 371 | − xa_reserve(xa, 12345678, GFP_KERNEL); | |
| 375 | + XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); | |
| 372 | 376 | XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_NOWAIT) != NULL); |
| 373 | 377 | xa_release(xa, 12345678); |
| 374 | 378 | xa_erase_index(xa, 12345678); |
| 375 | 379 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 376 | 380 | |
| 377 | − /* cmpxchg sees a reserved entry as NULL */ | |
| 378 | − xa_reserve(xa, 12345678, GFP_KERNEL); | |
| 379 | − XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, NULL, xa_mk_value(12345678), | |
| 380 | − GFP_NOWAIT) != NULL); | |
| 381 | + /* cmpxchg sees a reserved entry as ZERO */ | |
| 382 | + XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); | |
| 383 | + XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, XA_ZERO_ENTRY, | |
| 384 | + xa_mk_value(12345678), GFP_NOWAIT) != NULL); | |
| 381 | 385 | xa_release(xa, 12345678); |
| 382 | 386 | xa_erase_index(xa, 12345678); |
| 383 | 387 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 384 | 388 | |
| 385 | − /* But xa_insert does not */ | |
| 386 | − xa_reserve(xa, 12345678, GFP_KERNEL); | |
| 389 | + /* xa_insert treats it as busy */ | |
| 390 | + XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); | |
| 387 | 391 | XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) != |
| 388 | − -EEXIST); | |
| 392 | + -EBUSY); | |
| 389 | 393 | XA_BUG_ON(xa, xa_empty(xa)); |
| 390 | 394 | XA_BUG_ON(xa, xa_erase(xa, 12345678) != NULL); |
| 391 | 395 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 392 | 396 | |
| 393 | 397 | /* Can iterate through a reserved entry */ |
| 394 | 398 | xa_store_index(xa, 5, GFP_KERNEL); |
| 395 | − xa_reserve(xa, 6, GFP_KERNEL); | |
| 399 | + XA_BUG_ON(xa, xa_reserve(xa, 6, GFP_KERNEL) != 0); | |
| 396 | 400 | xa_store_index(xa, 7, GFP_KERNEL); |
| 397 | 401 | |
| 402 | + count = 0; | |
| 398 | 403 | xa_for_each(xa, index, entry) { |
| 399 | 404 | XA_BUG_ON(xa, index != 5 && index != 7); |
| 405 | + count++; | |
| 406 | + } | |
| 407 | + XA_BUG_ON(xa, count != 2); | |
| 408 | + | |
| 409 | + /* If we free a reserved entry, we should be able to allocate it */ | |
| 410 | + if (xa->xa_flags & XA_FLAGS_ALLOC) { | |
| 411 | + u32 id; | |
| 412 | + | |
| 413 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(8), | |
| 414 | + XA_LIMIT(5, 10), GFP_KERNEL) != 0); | |
| 415 | + XA_BUG_ON(xa, id != 8); | |
| 416 | + | |
| 417 | + xa_release(xa, 6); | |
| 418 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(6), | |
| 419 | + XA_LIMIT(5, 10), GFP_KERNEL) != 0); | |
| 420 | + XA_BUG_ON(xa, id != 6); | |
| 400 | 421 | } |
| 422 | + | |
| 401 | 423 | xa_destroy(xa); |
| 402 | 424 | } |
| 403 | 425 | |
| @@ -586,64 +608,194 @@ static noinline void check_multi_store(struct xarray *xa) · 182 unmodified lines | ||
| 586 | 608 | #endif |
| 587 | 609 | } |
| 588 | 610 | |
| 589 | −static DEFINE_XARRAY_ALLOC(xa0); | |
| 590 | − | |
| 591 | −static noinline void check_xa_alloc(void) | |
| 611 | +static noinline void check_xa_alloc_1(struct xarray *xa, unsigned int base) | |
| 592 | 612 | { |
| 593 | 613 | int i; |
| 594 | 614 | u32 id; |
| 595 | 615 | |
| 596 | − /* An empty array should assign 0 to the first alloc */ | |
| 597 | − xa_alloc_index(&xa0, 0, GFP_KERNEL); | |
| 616 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 617 | + /* An empty array should assign %base to the first alloc */ | |
| 618 | + xa_alloc_index(xa, base, GFP_KERNEL); | |
| 598 | 619 | |
| 599 | 620 | /* Erasing it should make the array empty again */ |
| 600 | − xa_erase_index(&xa0, 0); | |
| 601 | − XA_BUG_ON(&xa0, !xa_empty(&xa0)); | |
| 621 | + xa_erase_index(xa, base); | |
| 622 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 623 | + | |
| 624 | + /* And it should assign %base again */ | |
| 625 | + xa_alloc_index(xa, base, GFP_KERNEL); | |
| 626 | + | |
| 627 | + /* Allocating and then erasing a lot should not lose base */ | |
| 628 | + for (i = base + 1; i < 2 * XA_CHUNK_SIZE; i++) | |
| 629 | + xa_alloc_index(xa, i, GFP_KERNEL); | |
| 630 | + for (i = base; i < 2 * XA_CHUNK_SIZE; i++) | |
| 631 | + xa_erase_index(xa, i); | |
| 632 | + xa_alloc_index(xa, base, GFP_KERNEL); | |
| 602 | 633 | |
| 603 | − /* And it should assign 0 again */ | |
| 604 | − xa_alloc_index(&xa0, 0, GFP_KERNEL); | |
| 634 | + /* Destroying the array should do the same as erasing */ | |
| 635 | + xa_destroy(xa); | |
| 636 | + | |
| 637 | + /* And it should assign %base again */ | |
| 638 | + xa_alloc_index(xa, base, GFP_KERNEL); | |
| 605 | 639 | |
| 606 | − /* The next assigned ID should be 1 */ | |
| 607 | − xa_alloc_index(&xa0, 1, GFP_KERNEL); | |
| 608 | − xa_erase_index(&xa0, 1); | |
| 640 | + /* The next assigned ID should be base+1 */ | |
| 641 | + xa_alloc_index(xa, base + 1, GFP_KERNEL); | |
| 642 | + xa_erase_index(xa, base + 1); | |
| 609 | 643 | |
| 610 | 644 | /* Storing a value should mark it used */ |
| 611 | − xa_store_index(&xa0, 1, GFP_KERNEL); | |
| 612 | − xa_alloc_index(&xa0, 2, GFP_KERNEL); | |
| 645 | + xa_store_index(xa, base + 1, GFP_KERNEL); | |
| 646 | + xa_alloc_index(xa, base + 2, GFP_KERNEL); | |
| 613 | 647 | |
| 614 | − /* If we then erase 0, it should be free */ | |
| 615 | − xa_erase_index(&xa0, 0); | |
| 616 | − xa_alloc_index(&xa0, 0, GFP_KERNEL); | |
| 648 | + /* If we then erase base, it should be free */ | |
| 649 | + xa_erase_index(xa, base); | |
| 650 | + xa_alloc_index(xa, base, GFP_KERNEL); | |
| 617 | 651 | |
| 618 | − xa_erase_index(&xa0, 1); | |
| 619 | − xa_erase_index(&xa0, 2); | |
| 652 | + xa_erase_index(xa, base + 1); | |
| 653 | + xa_erase_index(xa, base + 2); | |
| 620 | 654 | |
| 621 | 655 | for (i = 1; i < 5000; i++) { |
| 622 | − xa_alloc_index(&xa0, i, GFP_KERNEL); | |
| 656 | + xa_alloc_index(xa, base + i, GFP_KERNEL); | |
| 623 | 657 | } |
| 624 | 658 | |
| 625 | − xa_destroy(&xa0); | |
| 659 | + xa_destroy(xa); | |
| 626 | 660 | |
| 627 | − id = 0xfffffffeU; | |
| 628 | − XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id), | |
| 661 | + /* Check that we fail properly at the limit of allocation */ | |
| 662 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX - 1), | |
| 663 | + XA_LIMIT(UINT_MAX - 1, UINT_MAX), | |
| 629 | 664 | GFP_KERNEL) != 0); |
| 630 | − XA_BUG_ON(&xa0, id != 0xfffffffeU); | |
| 631 | − XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id), | |
| 665 | + XA_BUG_ON(xa, id != 0xfffffffeU); | |
| 666 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX), | |
| 667 | + XA_LIMIT(UINT_MAX - 1, UINT_MAX), | |
| 632 | 668 | GFP_KERNEL) != 0); |
| 633 | − XA_BUG_ON(&xa0, id != 0xffffffffU); | |
| 634 | − XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id), | |
| 635 | − GFP_KERNEL) != -ENOSPC); | |
| 636 | − XA_BUG_ON(&xa0, id != 0xffffffffU); | |
| 637 | − xa_destroy(&xa0); | |
| 638 | − | |
| 639 | − id = 10; | |
| 640 | − XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id), | |
| 641 | − GFP_KERNEL) != -ENOSPC); | |
| 642 | − XA_BUG_ON(&xa0, xa_store_index(&xa0, 3, GFP_KERNEL) != 0); | |
| 643 | − XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id), | |
| 644 | − GFP_KERNEL) != -ENOSPC); | |
| 645 | − xa_erase_index(&xa0, 3); | |
| 646 | − XA_BUG_ON(&xa0, !xa_empty(&xa0)); | |
| 669 | + XA_BUG_ON(xa, id != 0xffffffffU); | |
| 670 | + id = 3; | |
| 671 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(0), | |
| 672 | + XA_LIMIT(UINT_MAX - 1, UINT_MAX), | |
| 673 | + GFP_KERNEL) != -EBUSY); | |
| 674 | + XA_BUG_ON(xa, id != 3); | |
| 675 | + xa_destroy(xa); | |
| 676 | + | |
| 677 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5), | |
| 678 | + GFP_KERNEL) != -EBUSY); | |
| 679 | + XA_BUG_ON(xa, xa_store_index(xa, 3, GFP_KERNEL) != 0); | |
| 680 | + XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5), | |
| 681 | + GFP_KERNEL) != -EBUSY); | |
| 682 | + xa_erase_index(xa, 3); | |
| 683 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 684 | +} | |
| 685 | + | |
| 686 | +static noinline void check_xa_alloc_2(struct xarray *xa, unsigned int base) | |
| 687 | +{ | |
| 688 | + unsigned int i, id; | |
| 689 | + unsigned long index; | |
| 690 | + void *entry; | |
| 691 | + | |
| 692 | + /* Allocate and free a NULL and check xa_empty() behaves */ | |
| 693 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 694 | + XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0); | |
| 695 | + XA_BUG_ON(xa, id != base); | |
| 696 | + XA_BUG_ON(xa, xa_empty(xa)); | |
| 697 | + XA_BUG_ON(xa, xa_erase(xa, id) != NULL); | |
| 698 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 699 | + | |
| 700 | + /* Ditto, but check destroy instead of erase */ | |
| 701 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 702 | + XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0); | |
| 703 | + XA_BUG_ON(xa, id != base); | |
| 704 | + XA_BUG_ON(xa, xa_empty(xa)); | |
| 705 | + xa_destroy(xa); | |
| 706 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 707 | + | |
| 708 | + for (i = base; i < base + 10; i++) { | |
| 709 | + XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, | |
| 710 | + GFP_KERNEL) != 0); | |
| 711 | + XA_BUG_ON(xa, id != i); | |
| 712 | + } | |
| 713 | + | |
| 714 | + XA_BUG_ON(xa, xa_store(xa, 3, xa_mk_index(3), GFP_KERNEL) != NULL); | |
| 715 | + XA_BUG_ON(xa, xa_store(xa, 4, xa_mk_index(4), GFP_KERNEL) != NULL); | |
| 716 | + XA_BUG_ON(xa, xa_store(xa, 4, NULL, GFP_KERNEL) != xa_mk_index(4)); | |
| 717 | + XA_BUG_ON(xa, xa_erase(xa, 5) != NULL); | |
| 718 | + XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0); | |
| 719 | + XA_BUG_ON(xa, id != 5); | |
| 720 | + | |
| 721 | + xa_for_each(xa, index, entry) { | |
| 722 | + xa_erase_index(xa, index); | |
| 723 | + } | |
| 724 | + | |
| 725 | + for (i = base; i < base + 9; i++) { | |
| 726 | + XA_BUG_ON(xa, xa_erase(xa, i) != NULL); | |
| 727 | + XA_BUG_ON(xa, xa_empty(xa)); | |
| 728 | + } | |
| 729 | + XA_BUG_ON(xa, xa_erase(xa, 8) != NULL); | |
| 730 | + XA_BUG_ON(xa, xa_empty(xa)); | |
| 731 | + XA_BUG_ON(xa, xa_erase(xa, base + 9) != NULL); | |
| 732 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 733 | + | |
| 734 | + xa_destroy(xa); | |
| 735 | +} | |
| 736 | + | |
| 737 | +static noinline void check_xa_alloc_3(struct xarray *xa, unsigned int base) | |
| 738 | +{ | |
| 739 | + struct xa_limit limit = XA_LIMIT(1, 0x3fff); | |
| 740 | + u32 next = 0; | |
| 741 | + unsigned int i, id; | |
| 742 | + unsigned long index; | |
| 743 | + void *entry; | |
| 744 | + | |
| 745 | + XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(1), limit, | |
| 746 | + &next, GFP_KERNEL) != 0); | |
| 747 | + XA_BUG_ON(xa, id != 1); | |
| 748 | + | |
| 749 | + next = 0x3ffd; | |
| 750 | + XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(0x3ffd), limit, | |
| 751 | + &next, GFP_KERNEL) != 0); | |
| 752 | + XA_BUG_ON(xa, id != 0x3ffd); | |
| 753 | + xa_erase_index(xa, 0x3ffd); | |
| 754 | + xa_erase_index(xa, 1); | |
| 755 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 756 | + | |
| 757 | + for (i = 0x3ffe; i < 0x4003; i++) { | |
| 758 | + if (i < 0x4000) | |
| 759 | + entry = xa_mk_index(i); | |
| 760 | + else | |
| 761 | + entry = xa_mk_index(i - 0x3fff); | |
| 762 | + XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, entry, limit, | |
| 763 | + &next, GFP_KERNEL) != (id == 1)); | |
| 764 | + XA_BUG_ON(xa, xa_mk_index(id) != entry); | |
| 765 | + } | |
| 766 | + | |
| 767 | + /* Check wrap-around is handled correctly */ | |
| 768 | + if (base != 0) | |
| 769 | + xa_erase_index(xa, base); | |
| 770 | + xa_erase_index(xa, base + 1); | |
| 771 | + next = UINT_MAX; | |
| 772 | + XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(UINT_MAX), | |
| 773 | + xa_limit_32b, &next, GFP_KERNEL) != 0); | |
| 774 | + XA_BUG_ON(xa, id != UINT_MAX); | |
| 775 | + XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base), | |
| 776 | + xa_limit_32b, &next, GFP_KERNEL) != 1); | |
| 777 | + XA_BUG_ON(xa, id != base); | |
| 778 | + XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base + 1), | |
| 779 | + xa_limit_32b, &next, GFP_KERNEL) != 0); | |
| 780 | + XA_BUG_ON(xa, id != base + 1); | |
| 781 | + | |
| 782 | + xa_for_each(xa, index, entry) | |
| 783 | + xa_erase_index(xa, index); | |
| 784 | + | |
| 785 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 786 | +} | |
| 787 | + | |
| 788 | +static DEFINE_XARRAY_ALLOC(xa0); | |
| 789 | +static DEFINE_XARRAY_ALLOC1(xa1); | |
| 790 | + | |
| 791 | +static noinline void check_xa_alloc(void) | |
| 792 | +{ | |
| 793 | + check_xa_alloc_1(&xa0, 0); | |
| 794 | + check_xa_alloc_1(&xa1, 1); | |
| 795 | + check_xa_alloc_2(&xa0, 0); | |
| 796 | + check_xa_alloc_2(&xa1, 1); | |
| 797 | + check_xa_alloc_3(&xa0, 0); | |
| 798 | + check_xa_alloc_3(&xa1, 1); | |
| 647 | 799 | } |
| 648 | 800 | |
| 649 | 801 | static noinline void __check_store_iter(struct xarray *xa, unsigned long start, |
| @@ -1194,9 +1346,8 @@ static void check_align_1(struct xarray *xa, char *name) · 544 unmodified lines | ||
| 1194 | 1346 | void *entry; |
| 1195 | 1347 | |
| 1196 | 1348 | for (i = 0; i < 8; i++) { |
| 1197 | − id = 0; | |
| 1198 | − XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, name + i, GFP_KERNEL) | |
| 1199 | − != 0); | |
| 1349 | + XA_BUG_ON(xa, xa_alloc(xa, &id, name + i, xa_limit_32b, | |
| 1350 | + GFP_KERNEL) != 0); | |
| 1200 | 1351 | XA_BUG_ON(xa, id != i); |
| 1201 | 1352 | } |
| 1202 | 1353 | xa_for_each(xa, index, entry) |
| @@ -1204,6 +1355,30 @@ static void check_align_1(struct xarray *xa, char *name) · 1 unmodified line | ||
| 1204 | 1355 | xa_destroy(xa); |
| 1205 | 1356 | } |
| 1206 | 1357 | |
| 1358 | +/* | |
| 1359 | + * We should always be able to store without allocating memory after | |
| 1360 | + * reserving a slot. | |
| 1361 | + */ | |
| 1362 | +static void check_align_2(struct xarray *xa, char *name) | |
| 1363 | +{ | |
| 1364 | + int i; | |
| 1365 | + | |
| 1366 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 1367 | + | |
| 1368 | + for (i = 0; i < 8; i++) { | |
| 1369 | + XA_BUG_ON(xa, xa_store(xa, 0, name + i, GFP_KERNEL) != NULL); | |
| 1370 | + xa_erase(xa, 0); | |
| 1371 | + } | |
| 1372 | + | |
| 1373 | + for (i = 0; i < 8; i++) { | |
| 1374 | + XA_BUG_ON(xa, xa_reserve(xa, 0, GFP_KERNEL) != 0); | |
| 1375 | + XA_BUG_ON(xa, xa_store(xa, 0, name + i, 0) != NULL); | |
| 1376 | + xa_erase(xa, 0); | |
| 1377 | + } | |
| 1378 | + | |
| 1379 | + XA_BUG_ON(xa, !xa_empty(xa)); | |
| 1380 | +} | |
| 1381 | + | |
| 1207 | 1382 | static noinline void check_align(struct xarray *xa) |
| 1208 | 1383 | { |
| 1209 | 1384 | char name[] = "Motorola 68000"; |
| @@ -1212,7 +1387,7 @@ static noinline void check_align(struct xarray *xa) · 2 unmodified lines | ||
| 1212 | 1387 | check_align_1(xa, name + 1); |
| 1213 | 1388 | check_align_1(xa, name + 2); |
| 1214 | 1389 | check_align_1(xa, name + 3); |
| 1215 | −// check_align_2(xa, name); | |
| 1390 | + check_align_2(xa, name); | |
| 1216 | 1391 | } |
| 1217 | 1392 | |
| 1218 | 1393 | static LIST_HEAD(shadow_nodes); |
| @@ -1354,6 +1529,7 @@ static int xarray_checks(void) · 135 unmodified lines | ||
| 1354 | 1529 | check_xas_erase(&array); |
| 1355 | 1530 | check_cmpxchg(&array); |
| 1356 | 1531 | check_reserve(&array); |
| 1532 | + check_reserve(&xa0); | |
| 1357 | 1533 | check_multi_store(&array); |
| 1358 | 1534 | check_xa_alloc(); |
| 1359 | 1535 | check_find(&array); |
| @@ -57,6 +57,11 @@ static inline bool xa_track_free(const struct xarray *xa) · 56 unmodified lines | ||
| 57 | 57 | return xa->xa_flags & XA_FLAGS_TRACK_FREE; |
| 58 | 58 | } |
| 59 | 59 | |
| 60 | +static inline bool xa_zero_busy(const struct xarray *xa) | |
| 61 | +{ | |
| 62 | + return xa->xa_flags & XA_FLAGS_ZERO_BUSY; | |
| 63 | +} | |
| 64 | + | |
| 60 | 65 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) |
| 61 | 66 | { |
| 62 | 67 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) |
| @@ -432,6 +437,8 @@ static void xas_shrink(struct xa_state *xas) · 369 unmodified lines | ||
| 432 | 437 | break; |
| 433 | 438 | if (!xa_is_node(entry) && node->shift) |
| 434 | 439 | break; |
| 440 | + if (xa_is_zero(entry) && xa_zero_busy(xa)) | |
| 441 | + entry = NULL; | |
| 435 | 442 | xas->xa_node = XAS_BOUNDS; |
| 436 | 443 | |
| 437 | 444 | RCU_INIT_POINTER(xa->xa_head, entry); |
| @@ -628,6 +635,8 @@ static void *xas_create(struct xa_state *xas, bool allow_root) · 190 unmodified lines | ||
| 628 | 635 | if (xas_top(node)) { |
| 629 | 636 | entry = xa_head_locked(xa); |
| 630 | 637 | xas->xa_node = NULL; |
| 638 | + if (!entry && xa_zero_busy(xa)) | |
| 639 | + entry = XA_ZERO_ENTRY; | |
| 631 | 640 | shift = xas_expand(xas, entry); |
| 632 | 641 | if (shift < 0) |
| 633 | 642 | return NULL; |
| @@ -758,10 +767,12 @@ void *xas_store(struct xa_state *xas, void *entry) · 124 unmodified lines | ||
| 758 | 767 | void *first, *next; |
| 759 | 768 | bool value = xa_is_value(entry); |
| 760 | 769 | |
| 761 | − if (entry) | |
| 762 | − first = xas_create(xas, !xa_is_node(entry)); | |
| 763 | − else | |
| 770 | + if (entry) { | |
| 771 | + bool allow_root = !xa_is_node(entry) && !xa_is_zero(entry); | |
| 772 | + first = xas_create(xas, allow_root); | |
| 773 | + } else { | |
| 764 | 774 | first = xas_load(xas); |
| 775 | + } | |
| 765 | 776 | |
| 766 | 777 | if (xas_invalid(xas)) |
| 767 | 778 | return first; |
| @@ -791,7 +802,7 @@ void *xas_store(struct xa_state *xas, void *entry) · 23 unmodified lines | ||
| 791 | 802 | * entry is set to NULL. |
| 792 | 803 | */ |
| 793 | 804 | rcu_assign_pointer(*slot, entry); |
| 794 | − if (xa_is_node(next)) | |
| 805 | + if (xa_is_node(next) && (!node || node->shift)) | |
| 795 | 806 | xas_free_nodes(xas, xa_to_node(next)); |
| 796 | 807 | if (!node) |
| 797 | 808 | break; |
| @@ -1294,13 +1305,12 @@ static void *xas_result(struct xa_state *xas, void *curr) · 496 unmodified lines | ||
| 1294 | 1305 | * @xa: XArray. |
| 1295 | 1306 | * @index: Index into array. |
| 1296 | 1307 | * |
| 1297 | − * If the entry at this index is a multi-index entry then all indices will | |
| 1298 | − * be erased, and the entry will no longer be a multi-index entry. | |
| 1299 | − * This function expects the xa_lock to be held on entry. | |
| 1308 | + * After this function returns, loading from @index will return %NULL. | |
| 1309 | + * If the index is part of a multi-index entry, all indices will be erased | |
| 1310 | + * and none of the entries will be part of a multi-index entry. | |
| 1300 | 1311 | * |
| 1301 | − * Context: Any context. Expects xa_lock to be held on entry. May | |
| 1302 | − * release and reacquire xa_lock if @gfp flags permit. | |
| 1303 | − * Return: The old entry at this index. | |
| 1312 | + * Context: Any context. Expects xa_lock to be held on entry. | |
| 1313 | + * Return: The entry which used to be at this index. | |
| 1304 | 1314 | */ |
| 1305 | 1315 | void *__xa_erase(struct xarray *xa, unsigned long index) |
| 1306 | 1316 | { |
| @@ -1314,9 +1324,9 @@ EXPORT_SYMBOL(__xa_erase); · 7 unmodified lines | ||
| 1314 | 1324 | * @xa: XArray. |
| 1315 | 1325 | * @index: Index of entry. |
| 1316 | 1326 | * |
| 1317 | − * This function is the equivalent of calling xa_store() with %NULL as | |
| 1318 | − * the third argument. The XArray does not need to allocate memory, so | |
| 1319 | − * the user does not need to provide GFP flags. | |
| 1327 | + * After this function returns, loading from @index will return %NULL. | |
| 1328 | + * If the index is part of a multi-index entry, all indices will be erased | |
| 1329 | + * and none of the entries will be part of a multi-index entry. | |
| 1320 | 1330 | * |
| 1321 | 1331 | * Context: Any context. Takes and releases the xa_lock. |
| 1322 | 1332 | * Return: The entry which used to be at this index. |
| @@ -1421,16 +1431,12 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, · 98 unmodified lines | ||
| 1421 | 1431 | |
| 1422 | 1432 | if (WARN_ON_ONCE(xa_is_advanced(entry))) |
| 1423 | 1433 | return XA_ERROR(-EINVAL); |
| 1424 | − if (xa_track_free(xa) && !entry) | |
| 1425 | − entry = XA_ZERO_ENTRY; | |
| 1426 | 1434 | |
| 1427 | 1435 | do { |
| 1428 | 1436 | curr = xas_load(&xas); |
| 1429 | − if (curr == XA_ZERO_ENTRY) | |
| 1430 | − curr = NULL; | |
| 1431 | 1437 | if (curr == old) { |
| 1432 | 1438 | xas_store(&xas, entry); |
| 1433 | − if (xa_track_free(xa)) | |
| 1439 | + if (xa_track_free(xa) && entry && !curr) | |
| 1434 | 1440 | xas_clear_mark(&xas, XA_FREE_MARK); |
| 1435 | 1441 | } |
| 1436 | 1442 | } while (__xas_nomem(&xas, gfp)); |
| @@ -1452,7 +1458,7 @@ EXPORT_SYMBOL(__xa_cmpxchg); · 15 unmodified lines | ||
| 1452 | 1458 | * |
| 1453 | 1459 | * Context: Any context. Expects xa_lock to be held on entry. May |
| 1454 | 1460 | * release and reacquire xa_lock if @gfp flags permit. |
| 1455 | − * Return: 0 if the store succeeded. -EEXIST if another entry was present. | |
| 1461 | + * Return: 0 if the store succeeded. -EBUSY if another entry was present. | |
| 1456 | 1462 | * -ENOMEM if memory could not be allocated. |
| 1457 | 1463 | */ |
| 1458 | 1464 | int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) |
| @@ -1472,7 +1478,7 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 13 unmodified lines | ||
| 1472 | 1478 | if (xa_track_free(xa)) |
| 1473 | 1479 | xas_clear_mark(&xas, XA_FREE_MARK); |
| 1474 | 1480 | } else { |
| 1475 | − xas_set_err(&xas, -EEXIST); | |
| 1481 | + xas_set_err(&xas, -EBUSY); | |
| 1476 | 1482 | } |
| 1477 | 1483 | } while (__xas_nomem(&xas, gfp)); |
| 1478 | 1484 | |
| @@ -1480,42 +1486,6 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 1 unmodified line | ||
| 1480 | 1486 | } |
| 1481 | 1487 | EXPORT_SYMBOL(__xa_insert); |
| 1482 | 1488 | |
| 1483 | −/** | |
| 1484 | − * __xa_reserve() - Reserve this index in the XArray. | |
| 1485 | − * @xa: XArray. | |
| 1486 | − * @index: Index into array. | |
| 1487 | − * @gfp: Memory allocation flags. | |
| 1488 | − * | |
| 1489 | − * Ensures there is somewhere to store an entry at @index in the array. | |
| 1490 | − * If there is already something stored at @index, this function does | |
| 1491 | − * nothing. If there was nothing there, the entry is marked as reserved. | |
| 1492 | − * Loading from a reserved entry returns a %NULL pointer. | |
| 1493 | − * | |
| 1494 | − * If you do not use the entry that you have reserved, call xa_release() | |
| 1495 | − * or xa_erase() to free any unnecessary memory. | |
| 1496 | − * | |
| 1497 | − * Context: Any context. Expects the xa_lock to be held on entry. May | |
| 1498 | − * release the lock, sleep and reacquire the lock if the @gfp flags permit. | |
| 1499 | − * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | |
| 1500 | − */ | |
| 1501 | −int __xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) | |
| 1502 | −{ | |
| 1503 | − XA_STATE(xas, xa, index); | |
| 1504 | − void *curr; | |
| 1505 | − | |
| 1506 | − do { | |
| 1507 | − curr = xas_load(&xas); | |
| 1508 | − if (!curr) { | |
| 1509 | − xas_store(&xas, XA_ZERO_ENTRY); | |
| 1510 | − if (xa_track_free(xa)) | |
| 1511 | − xas_clear_mark(&xas, XA_FREE_MARK); | |
| 1512 | − } | |
| 1513 | − } while (__xas_nomem(&xas, gfp)); | |
| 1514 | − | |
| 1515 | − return xas_error(&xas); | |
| 1516 | −} | |
| 1517 | −EXPORT_SYMBOL(__xa_reserve); | |
| 1518 | − | |
| 1519 | 1489 | #ifdef CONFIG_XARRAY_MULTI |
| 1520 | 1490 | static void xas_set_range(struct xa_state *xas, unsigned long first, |
| 1521 | 1491 | unsigned long last) |
| @@ -1607,23 +1577,23 @@ EXPORT_SYMBOL(xa_store_range); · 85 unmodified lines | ||
| 1607 | 1577 | * __xa_alloc() - Find somewhere to store this entry in the XArray. |
| 1608 | 1578 | * @xa: XArray. |
| 1609 | 1579 | * @id: Pointer to ID. |
| 1610 | − * @max: Maximum ID to allocate (inclusive). | |
| 1580 | + * @limit: Range for allocated ID. | |
| 1611 | 1581 | * @entry: New entry. |
| 1612 | 1582 | * @gfp: Memory allocation flags. |
| 1613 | 1583 | * |
| 1614 | − * Allocates an unused ID in the range specified by @id and @max. | |
| 1615 | − * Updates the @id pointer with the index, then stores the entry at that | |
| 1616 | − * index. A concurrent lookup will not see an uninitialised @id. | |
| 1584 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 1585 | + * stores the index into the @id pointer, then stores the entry at | |
| 1586 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 1617 | 1587 | * |
| 1618 | 1588 | * Context: Any context. Expects xa_lock to be held on entry. May |
| 1619 | 1589 | * release and reacquire xa_lock if @gfp flags permit. |
| 1620 | − * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | |
| 1621 | − * there is no more space in the XArray. | |
| 1590 | + * Return: 0 on success, -ENOMEM if memory could not be allocated or | |
| 1591 | + * -EBUSY if there are no free entries in @limit. | |
| 1622 | 1592 | */ |
| 1623 | −int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) | |
| 1593 | +int __xa_alloc(struct xarray *xa, u32 *id, void *entry, | |
| 1594 | + struct xa_limit limit, gfp_t gfp) | |
| 1624 | 1595 | { |
| 1625 | 1596 | XA_STATE(xas, xa, 0); |
| 1626 | − int err; | |
| 1627 | 1597 | |
| 1628 | 1598 | if (WARN_ON_ONCE(xa_is_advanced(entry))) |
| 1629 | 1599 | return -EINVAL; |
| @@ -1634,21 +1604,70 @@ int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) · 4 unmodified lines | ||
| 1634 | 1604 | entry = XA_ZERO_ENTRY; |
| 1635 | 1605 | |
| 1636 | 1606 | do { |
| 1637 | − xas.xa_index = *id; | |
| 1638 | − xas_find_marked(&xas, max, XA_FREE_MARK); | |
| 1607 | + xas.xa_index = limit.min; | |
| 1608 | + xas_find_marked(&xas, limit.max, XA_FREE_MARK); | |
| 1639 | 1609 | if (xas.xa_node == XAS_RESTART) |
| 1640 | − xas_set_err(&xas, -ENOSPC); | |
| 1610 | + xas_set_err(&xas, -EBUSY); | |
| 1611 | + else | |
| 1612 | + *id = xas.xa_index; | |
| 1641 | 1613 | xas_store(&xas, entry); |
| 1642 | 1614 | xas_clear_mark(&xas, XA_FREE_MARK); |
| 1643 | 1615 | } while (__xas_nomem(&xas, gfp)); |
| 1644 | 1616 | |
| 1645 | − err = xas_error(&xas); | |
| 1646 | − if (!err) | |
| 1647 | − *id = xas.xa_index; | |
| 1648 | − return err; | |
| 1617 | + return xas_error(&xas); | |
| 1649 | 1618 | } |
| 1650 | 1619 | EXPORT_SYMBOL(__xa_alloc); |
| 1651 | 1620 | |
| 1621 | +/** | |
| 1622 | + * __xa_alloc_cyclic() - Find somewhere to store this entry in the XArray. | |
| 1623 | + * @xa: XArray. | |
| 1624 | + * @id: Pointer to ID. | |
| 1625 | + * @entry: New entry. | |
| 1626 | + * @limit: Range of allocated ID. | |
| 1627 | + * @next: Pointer to next ID to allocate. | |
| 1628 | + * @gfp: Memory allocation flags. | |
| 1629 | + * | |
| 1630 | + * Finds an empty entry in @xa between @limit.min and @limit.max, | |
| 1631 | + * stores the index into the @id pointer, then stores the entry at | |
| 1632 | + * that index. A concurrent lookup will not see an uninitialised @id. | |
| 1633 | + * The search for an empty entry will start at @next and will wrap | |
| 1634 | + * around if necessary. | |
| 1635 | + * | |
| 1636 | + * Context: Any context. Expects xa_lock to be held on entry. May | |
| 1637 | + * release and reacquire xa_lock if @gfp flags permit. | |
| 1638 | + * Return: 0 if the allocation succeeded without wrapping. 1 if the | |
| 1639 | + * allocation succeeded after wrapping, -ENOMEM if memory could not be | |
| 1640 | + * allocated or -EBUSY if there are no free entries in @limit. | |
| 1641 | + */ | |
| 1642 | +int __xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, | |
| 1643 | + struct xa_limit limit, u32 *next, gfp_t gfp) | |
| 1644 | +{ | |
| 1645 | + u32 min = limit.min; | |
| 1646 | + int ret; | |
| 1647 | + | |
| 1648 | + limit.min = max(min, *next); | |
| 1649 | + ret = __xa_alloc(xa, id, entry, limit, gfp); | |
| 1650 | + if ((xa->xa_flags & XA_FLAGS_ALLOC_WRAPPED) && ret == 0) { | |
| 1651 | + xa->xa_flags &= ~XA_FLAGS_ALLOC_WRAPPED; | |
| 1652 | + ret = 1; | |
| 1653 | + } | |
| 1654 | + | |
| 1655 | + if (ret < 0 && limit.min > min) { | |
| 1656 | + limit.min = min; | |
| 1657 | + ret = __xa_alloc(xa, id, entry, limit, gfp); | |
| 1658 | + if (ret == 0) | |
| 1659 | + ret = 1; | |
| 1660 | + } | |
| 1661 | + | |
| 1662 | + if (ret >= 0) { | |
| 1663 | + *next = *id + 1; | |
| 1664 | + if (*next == 0) | |
| 1665 | + xa->xa_flags |= XA_FLAGS_ALLOC_WRAPPED; | |
| 1666 | + } | |
| 1667 | + return ret; | |
| 1668 | +} | |
| 1669 | +EXPORT_SYMBOL(__xa_alloc_cyclic); | |
| 1670 | + | |
| 1652 | 1671 | /** |
| 1653 | 1672 | * __xa_set_mark() - Set this mark on this entry while locked. |
| 1654 | 1673 | * @xa: XArray. |
| @@ -1943,6 +1962,8 @@ void xa_destroy(struct xarray *xa) · 288 unmodified lines | ||
| 1943 | 1962 | entry = xa_head_locked(xa); |
| 1944 | 1963 | RCU_INIT_POINTER(xa->xa_head, NULL); |
| 1945 | 1964 | xas_init_marks(&xas); |
| 1965 | + if (xa_zero_busy(xa)) | |
| 1966 | + xa_mark_clear(xa, XA_FREE_MARK); | |
| 1946 | 1967 | /* lockdep checks we're still holding the lock in xas_free_nodes() */ |
| 1947 | 1968 | if (xa_is_node(entry)) |
| 1948 | 1969 | xas_free_nodes(&xas, xa_to_node(entry)); |