Pull XArray updates from Matthew Wilcox: "This pull request changes the xa_alloc() API. I'm only aware of one subsystem that has started trying to use it, and we agree on the fixup as part of the merge. The xa_insert() error code also changed to match xa_alloc() (EEXIST to EBUSY), and I added xa_alloc_cyclic(). Beyond that, the usual bugfixes, optimisations and tweaking. I now have a git tree with all users of the radix tree and IDR converted over to the XArray that I'll be feeding to maintainers over the next few weeks" * tag 'xarray-5.1-rc1' of git://git.infradead.org/users/willy/linux-dax: XArray: Fix xa_reserve for 2-byte aligned entries XArray: Fix xa_erase of 2-byte aligned entries XArray: Use xa_cmpxchg to implement xa_reserve XArray: Fix xa_release in allocating arrays XArray: Mark xa_insert and xa_reserve as must_check XArray: Add cyclic allocation XArray: Redesign xa_alloc API XArray: Add support for 1s-based allocation XArray: Change xa_insert to return -EBUSY XArray: Update xa_erase family descriptions XArray tests: RCU lock prohibits GFP_KERNEL
Unified · Split
| @@ -85,7 +85,7 @@ which was at that index; if it returns the same entry which was passed as · 84 unmodified lines | |||
| 85 | 85 | ||
| 86 | If you want to only store a new entry to an index if the current entry | 86 | If you want to only store a new entry to an index if the current entry |
| 87 | at that index is ``NULL``, you can use :c:func:`xa_insert` which | 87 | at that index is ``NULL``, you can use :c:func:`xa_insert` which |
| 88 | returns ``-EEXIST`` if the entry is not empty. | 88 | returns ``-EBUSY`` if the entry is not empty. |
| 89 | 89 | ||
| 90 | You can enquire whether a mark is set on an entry by using | 90 | You can enquire whether a mark is set on an entry by using |
| 91 | :c:func:`xa_get_mark`. If the entry is not ``NULL``, you can set a mark | 91 | :c:func:`xa_get_mark`. If the entry is not ``NULL``, you can set a mark |
| @@ -131,17 +131,23 @@ If you use :c:func:`DEFINE_XARRAY_ALLOC` to define the XArray, or · 39 unmodified lines | |||
| 131 | initialise it by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`, | 131 | initialise it by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`, |
| 132 | the XArray changes to track whether entries are in use or not. | 132 | the XArray changes to track whether entries are in use or not. |
| 133 | 133 | ||
| 134 | You can call :c:func:`xa_alloc` to store the entry at any unused index | 134 | You can call :c:func:`xa_alloc` to store the entry at an unused index |
| 135 | in the XArray. If you need to modify the array from interrupt context, | 135 | in the XArray. If you need to modify the array from interrupt context, |
| 136 | you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable | 136 | you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable |
| 137 | interrupts while allocating the ID. | 137 | interrupts while allocating the ID. |
| 138 | 138 | ||
| 139 | Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert` | 139 | Using :c:func:`xa_store`, :c:func:`xa_cmpxchg` or :c:func:`xa_insert` will |
| 140 | will mark the entry as being allocated. Unlike a normal XArray, storing | 140 | also mark the entry as being allocated. Unlike a normal XArray, storing |
| 141 | ``NULL`` will mark the entry as being in use, like :c:func:`xa_reserve`. | 141 | ``NULL`` will mark the entry as being in use, like :c:func:`xa_reserve`. |
| 142 | To free an entry, use :c:func:`xa_erase` (or :c:func:`xa_release` if | 142 | To free an entry, use :c:func:`xa_erase` (or :c:func:`xa_release` if |
| 143 | you only want to free the entry if it's ``NULL``). | 143 | you only want to free the entry if it's ``NULL``). |
| 144 | 144 | ||
| 145 | By default, the lowest free entry is allocated starting from 0. If you | ||
| 146 | want to allocate entries starting at 1, it is more efficient to use | ||
| 147 | :c:func:`DEFINE_XARRAY_ALLOC1` or ``XA_FLAGS_ALLOC1``. If you want to | ||
| 148 | allocate IDs up to a maximum, then wrap back around to the lowest free | ||
| 149 | ID, you can use :c:func:`xa_alloc_cyclic`. | ||
| 150 | |||
| 145 | You cannot use ``XA_MARK_0`` with an allocating XArray as this mark | 151 | You cannot use ``XA_MARK_0`` with an allocating XArray as this mark |
| 146 | is used to track whether an entry is free or not. The other marks are | 152 | is used to track whether an entry is free or not. The other marks are |
| 147 | available for your use. | 153 | available for your use. |
| @@ -209,7 +215,6 @@ Assumes xa_lock held on entry: · 61 unmodified lines | |||
| 209 | * :c:func:`__xa_erase` | 215 | * :c:func:`__xa_erase` |
| 210 | * :c:func:`__xa_cmpxchg` | 216 | * :c:func:`__xa_cmpxchg` |
| 211 | * :c:func:`__xa_alloc` | 217 | * :c:func:`__xa_alloc` |
| 212 | * :c:func:`__xa_reserve` | ||
| 213 | * :c:func:`__xa_set_mark` | 218 | * :c:func:`__xa_set_mark` |
| 214 | * :c:func:`__xa_clear_mark` | 219 | * :c:func:`__xa_clear_mark` |
| 215 | 220 | ||
| @@ -668,19 +668,10 @@ static int assign_name(struct ib_device *device, const char *name) · 667 unmodified lines | |||
| 668 | } | 668 | } |
| 669 | strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); | 669 | strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); |
| 670 | 670 | ||
| 671 | /* Cyclically allocate a user visible ID for the device */ | 671 | ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b, |
| 672 | device->index = last_id; | 672 | &last_id, GFP_KERNEL); |
| 673 | ret = xa_alloc(&devices, &device->index, INT_MAX, device, GFP_KERNEL); | 673 | if (ret > 0) |
| 674 | if (ret == -ENOSPC) { | 674 | ret = 0; |
| 675 | device->index = 0; | ||
| 676 | ret = xa_alloc(&devices, &device->index, INT_MAX, device, | ||
| 677 | GFP_KERNEL); | ||
| 678 | } | ||
| 679 | if (ret) | ||
| 680 | goto out; | ||
| 681 | last_id = device->index + 1; | ||
| 682 | |||
| 683 | ret = 0; | ||
| 684 | 675 | ||
| 685 | out: | 676 | out: |
| 686 | up_write(&devices_rwsem); | 677 | up_write(&devices_rwsem); |
| @@ -1059,14 +1050,15 @@ static int assign_client_id(struct ib_client *client) · 372 unmodified lines | |||
| 1059 | * to get the LIFO order. The extra linked list can go away if xarray | 1050 | * to get the LIFO order. The extra linked list can go away if xarray |
| 1060 | * learns to reverse iterate. | 1051 | * learns to reverse iterate. |
| 1061 | */ | 1052 | */ |
| 1062 | if (list_empty(&client_list)) | 1053 | if (list_empty(&client_list)) { |
| 1063 | client->client_id = 0; | 1054 | client->client_id = 0; |
| 1064 | else | 1055 | } else { |
| 1065 | client->client_id = | 1056 | struct ib_client *last; |
| 1066 | list_last_entry(&client_list, struct ib_client, list) | 1057 | |
| 1067 | ->client_id; | 1058 | last = list_last_entry(&client_list, struct ib_client, list); |
| 1068 | ret = xa_alloc(&clients, &client->client_id, INT_MAX, client, | 1059 | client->client_id = last->client_id + 1; |
| 1069 | GFP_KERNEL); | 1060 | } |
| 1061 | ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL); | ||
| 1070 | if (ret) | 1062 | if (ret) |
| 1071 | goto out; | 1063 | goto out; |
| 1072 | 1064 | ||
| @@ -13,28 +13,6 @@ · 12 unmodified lines | |||
| 13 | #include "cma_priv.h" | 13 | #include "cma_priv.h" |
| 14 | #include "restrack.h" | 14 | #include "restrack.h" |
| 15 | 15 | ||
| 16 | static int rt_xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, | ||
| 17 | u32 *next) | ||
| 18 | { | ||
| 19 | int err; | ||
| 20 | |||
| 21 | *id = *next; | ||
| 22 | if (*next == U32_MAX) | ||
| 23 | *id = 0; | ||
| 24 | |||
| 25 | xa_lock(xa); | ||
| 26 | err = __xa_alloc(xa, id, U32_MAX, entry, GFP_KERNEL); | ||
| 27 | if (err && *next != U32_MAX) { | ||
| 28 | *id = 0; | ||
| 29 | err = __xa_alloc(xa, id, *next, entry, GFP_KERNEL); | ||
| 30 | } | ||
| 31 | |||
| 32 | if (!err) | ||
| 33 | *next = *id + 1; | ||
| 34 | xa_unlock(xa); | ||
| 35 | return err; | ||
| 36 | } | ||
| 37 | |||
| 38 | /** | 16 | /** |
| 39 | * rdma_restrack_init() - initialize and allocate resource tracking | 17 | * rdma_restrack_init() - initialize and allocate resource tracking |
| 40 | * @dev: IB device | 18 | * @dev: IB device |
| @@ -226,7 +204,8 @@ static void rdma_restrack_add(struct rdma_restrack_entry *res) · 185 unmodified lines | |||
| 226 | kref_init(&res->kref); | 204 | kref_init(&res->kref); |
| 227 | init_completion(&res->comp); | 205 | init_completion(&res->comp); |
| 228 | if (res->type != RDMA_RESTRACK_QP) | 206 | if (res->type != RDMA_RESTRACK_QP) |
| 229 | ret = rt_xa_alloc_cyclic(&rt->xa, &res->id, res, &rt->next_id); | 207 | ret = xa_alloc_cyclic(&rt->xa, &res->id, res, xa_limit_32b, |
| 208 | &rt->next_id, GFP_KERNEL); | ||
| 230 | else { | 209 | else { |
| 231 | /* Special case to ensure that LQPN points to right QP */ | 210 | /* Special case to ensure that LQPN points to right QP */ |
| 232 | struct ib_qp *qp = container_of(res, struct ib_qp, res); | 211 | struct ib_qp *qp = container_of(res, struct ib_qp, res); |
| @@ -189,7 +189,7 @@ retry: · 188 unmodified lines | |||
| 189 | */ | 189 | */ |
| 190 | if (!err) | 190 | if (!err) |
| 191 | return 0; | 191 | return 0; |
| 192 | else if (err != -EEXIST) | 192 | else if (err != -EBUSY) |
| 193 | goto failed_unlock; | 193 | goto failed_unlock; |
| 194 | 194 | ||
| 195 | err = invalidate_inode_pages2_range(btnc, newkey, newkey); | 195 | err = invalidate_inode_pages2_range(btnc, newkey, newkey); |
| @@ -131,6 +131,12 @@ static inline unsigned int xa_pointer_tag(void *entry) · 130 unmodified lines | |||
| 131 | * xa_mk_internal() - Create an internal entry. | 131 | * xa_mk_internal() - Create an internal entry. |
| 132 | * @v: Value to turn into an internal entry. | 132 | * @v: Value to turn into an internal entry. |
| 133 | * | 133 | * |
| 134 | * Internal entries are used for a number of purposes. Entries 0-255 are | ||
| 135 | * used for sibling entries (only 0-62 are used by the current code). 256 | ||
| 136 | * is used for the retry entry. 257 is used for the reserved / zero entry. | ||
| 137 | * Negative internal entries are used to represent errnos. Node pointers | ||
| 138 | * are also tagged as internal entries in some situations. | ||
| 139 | * | ||
| 134 | * Context: Any context. | 140 | * Context: Any context. |
| 135 | * Return: An XArray internal entry corresponding to this value. | 141 | * Return: An XArray internal entry corresponding to this value. |
| 136 | */ | 142 | */ |
| @@ -163,6 +169,22 @@ static inline bool xa_is_internal(const void *entry) · 26 unmodified lines | |||
| 163 | return ((unsigned long)entry & 3) == 2; | 169 | return ((unsigned long)entry & 3) == 2; |
| 164 | } | 170 | } |
| 165 | 171 | ||
| 172 | #define XA_ZERO_ENTRY xa_mk_internal(257) | ||
| 173 | |||
| 174 | /** | ||
| 175 | * xa_is_zero() - Is the entry a zero entry? | ||
| 176 | * @entry: Entry retrieved from the XArray | ||
| 177 | * | ||
| 178 | * The normal API will return NULL as the contents of a slot containing | ||
| 179 | * a zero entry. You can only see zero entries by using the advanced API. | ||
| 180 | * | ||
| 181 | * Return: %true if the entry is a zero entry. | ||
| 182 | */ | ||
| 183 | static inline bool xa_is_zero(const void *entry) | ||
| 184 | { | ||
| 185 | return unlikely(entry == XA_ZERO_ENTRY); | ||
| 186 | } | ||
| 187 | |||
| 166 | /** | 188 | /** |
| 167 | * xa_is_err() - Report whether an XArray operation returned an error | 189 | * xa_is_err() - Report whether an XArray operation returned an error |
| 168 | * @entry: Result from calling an XArray function | 190 | * @entry: Result from calling an XArray function |
| @@ -200,6 +222,27 @@ static inline int xa_err(void *entry) · 31 unmodified lines | |||
| 200 | return 0; | 222 | return 0; |
| 201 | } | 223 | } |
| 202 | 224 | ||
| 225 | /** | ||
| 226 | * struct xa_limit - Represents a range of IDs. | ||
| 227 | * @min: The lowest ID to allocate (inclusive). | ||
| 228 | * @max: The maximum ID to allocate (inclusive). | ||
| 229 | * | ||
| 230 | * This structure is used either directly or via the XA_LIMIT() macro | ||
| 231 | * to communicate the range of IDs that are valid for allocation. | ||
| 232 | * Two common ranges are predefined for you: | ||
| 233 | * * xa_limit_32b - [0 - UINT_MAX] | ||
| 234 | * * xa_limit_31b - [0 - INT_MAX] | ||
| 235 | */ | ||
| 236 | struct xa_limit { | ||
| 237 | u32 max; | ||
| 238 | u32 min; | ||
| 239 | }; | ||
| 240 | |||
| 241 | #define XA_LIMIT(_min, _max) (struct xa_limit) { .min = _min, .max = _max } | ||
| 242 | |||
| 243 | #define xa_limit_32b XA_LIMIT(0, UINT_MAX) | ||
| 244 | #define xa_limit_31b XA_LIMIT(0, INT_MAX) | ||
| 245 | |||
| 203 | typedef unsigned __bitwise xa_mark_t; | 246 | typedef unsigned __bitwise xa_mark_t; |
| 204 | #define XA_MARK_0 ((__force xa_mark_t)0U) | 247 | #define XA_MARK_0 ((__force xa_mark_t)0U) |
| 205 | #define XA_MARK_1 ((__force xa_mark_t)1U) | 248 | #define XA_MARK_1 ((__force xa_mark_t)1U) |
| @@ -220,10 +263,14 @@ enum xa_lock_type { · 14 unmodified lines | |||
| 220 | #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) | 263 | #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) |
| 221 | #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) | 264 | #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) |
| 222 | #define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U) | 265 | #define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U) |
| 266 | #define XA_FLAGS_ZERO_BUSY ((__force gfp_t)8U) | ||
| 267 | #define XA_FLAGS_ALLOC_WRAPPED ((__force gfp_t)16U) | ||
| 223 | #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ | 268 | #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ |
| 224 | (__force unsigned)(mark))) | 269 | (__force unsigned)(mark))) |
| 225 | 270 | ||
| 271 | /* ALLOC is for a normal 0-based alloc. ALLOC1 is for an 1-based alloc */ | ||
| 226 | #define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK)) | 272 | #define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK)) |
| 273 | #define XA_FLAGS_ALLOC1 (XA_FLAGS_TRACK_FREE | XA_FLAGS_ZERO_BUSY) | ||
| 227 | 274 | ||
| 228 | /** | 275 | /** |
| 229 | * struct xarray - The anchor of the XArray. | 276 | * struct xarray - The anchor of the XArray. |
| @@ -279,7 +326,7 @@ struct xarray { · 49 unmodified lines | |||
| 279 | #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) | 326 | #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) |
| 280 | 327 | ||
| 281 | /** | 328 | /** |
| 282 | * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs. | 329 | * DEFINE_XARRAY_ALLOC() - Define an XArray which allocates IDs starting at 0. |
| 283 | * @name: A string that names your XArray. | 330 | * @name: A string that names your XArray. |
| 284 | * | 331 | * |
| 285 | * This is intended for file scope definitions of allocating XArrays. | 332 | * This is intended for file scope definitions of allocating XArrays. |
| @@ -287,6 +334,15 @@ struct xarray { · 1 unmodified line | |||
| 287 | */ | 334 | */ |
| 288 | #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC) | 335 | #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC) |
| 289 | 336 | ||
| 337 | /** | ||
| 338 | * DEFINE_XARRAY_ALLOC1() - Define an XArray which allocates IDs starting at 1. | ||
| 339 | * @name: A string that names your XArray. | ||
| 340 | * | ||
| 341 | * This is intended for file scope definitions of allocating XArrays. | ||
| 342 | * See also DEFINE_XARRAY(). | ||
| 343 | */ | ||
| 344 | #define DEFINE_XARRAY_ALLOC1(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC1) | ||
| 345 | |||
| 290 | void *xa_load(struct xarray *, unsigned long index); | 346 | void *xa_load(struct xarray *, unsigned long index); |
| 291 | void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); | 347 | void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| 292 | void *xa_erase(struct xarray *, unsigned long index); | 348 | void *xa_erase(struct xarray *, unsigned long index); |
| @@ -463,9 +519,12 @@ void *__xa_erase(struct xarray *, unsigned long index); · 170 unmodified lines | |||
| 463 | void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); | 519 | void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| 464 | void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, | 520 | void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, |
| 465 | void *entry, gfp_t); | 521 | void *entry, gfp_t); |
| 466 | int __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t); | 522 | int __must_check __xa_insert(struct xarray *, unsigned long index, |
| 467 | int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t); | 523 | void *entry, gfp_t); |
| 468 | int __xa_reserve(struct xarray *, unsigned long index, gfp_t); | 524 | int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry, |
| 525 | struct xa_limit, gfp_t); | ||
| 526 | int __must_check __xa_alloc_cyclic(struct xarray *, u32 *id, void *entry, | ||
| 527 | struct xa_limit, u32 *next, gfp_t); | ||
| 469 | void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); | 528 | void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 470 | void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); | 529 | void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 471 | 530 | ||
| @@ -526,9 +585,9 @@ static inline void *xa_store_irq(struct xarray *xa, unsigned long index, · 54 unmodified lines | |||
| 526 | * @xa: XArray. | 585 | * @xa: XArray. |
| 527 | * @index: Index of entry. | 586 | * @index: Index of entry. |
| 528 | * | 587 | * |
| 529 | * This function is the equivalent of calling xa_store() with %NULL as | 588 | * After this function returns, loading from @index will return %NULL. |
| 530 | * the third argument. The XArray does not need to allocate memory, so | 589 | * If the index is part of a multi-index entry, all indices will be erased |
| 531 | * the user does not need to provide GFP flags. | 590 | * and none of the entries will be part of a multi-index entry. |
| 532 | * | 591 | * |
| 533 | * Context: Any context. Takes and releases the xa_lock while | 592 | * Context: Any context. Takes and releases the xa_lock while |
| 534 | * disabling softirqs. | 593 | * disabling softirqs. |
| @@ -550,9 +609,9 @@ static inline void *xa_erase_bh(struct xarray *xa, unsigned long index) · 15 unmodified lines | |||
| 550 | * @xa: XArray. | 609 | * @xa: XArray. |
| 551 | * @index: Index of entry. | 610 | * @index: Index of entry. |
| 552 | * | 611 | * |
| 553 | * This function is the equivalent of calling xa_store() with %NULL as | 612 | * After this function returns, loading from @index will return %NULL. |
| 554 | * the third argument. The XArray does not need to allocate memory, so | 613 | * If the index is part of a multi-index entry, all indices will be erased |
| 555 | * the user does not need to provide GFP flags. | 614 | * and none of the entries will be part of a multi-index entry. |
| 556 | * | 615 | * |
| 557 | * Context: Process context. Takes and releases the xa_lock while | 616 | * Context: Process context. Takes and releases the xa_lock while |
| 558 | * disabling interrupts. | 617 | * disabling interrupts. |
| @@ -664,11 +723,11 @@ static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index, · 105 unmodified lines | |||
| 664 | * | 723 | * |
| 665 | * Context: Any context. Takes and releases the xa_lock. May sleep if | 724 | * Context: Any context. Takes and releases the xa_lock. May sleep if |
| 666 | * the @gfp flags permit. | 725 | * the @gfp flags permit. |
| 667 | * Return: 0 if the store succeeded. -EEXIST if another entry was present. | 726 | * Return: 0 if the store succeeded. -EBUSY if another entry was present. |
| 668 | * -ENOMEM if memory could not be allocated. | 727 | * -ENOMEM if memory could not be allocated. |
| 669 | */ | 728 | */ |
| 670 | static inline int xa_insert(struct xarray *xa, unsigned long index, | 729 | static inline int __must_check xa_insert(struct xarray *xa, |
| 671 | void *entry, gfp_t gfp) | 730 | unsigned long index, void *entry, gfp_t gfp) |
| 672 | { | 731 | { |
| 673 | int err; | 732 | int err; |
| 674 | 733 | ||
| @@ -693,11 +752,11 @@ static inline int xa_insert(struct xarray *xa, unsigned long index, · 18 unmodified lines | |||
| 693 | * | 752 | * |
| 694 | * Context: Any context. Takes and releases the xa_lock while | 753 | * Context: Any context. Takes and releases the xa_lock while |
| 695 | * disabling softirqs. May sleep if the @gfp flags permit. | 754 | * disabling softirqs. May sleep if the @gfp flags permit. |
| 696 | * Return: 0 if the store succeeded. -EEXIST if another entry was present. | 755 | * Return: 0 if the store succeeded. -EBUSY if another entry was present. |
| 697 | * -ENOMEM if memory could not be allocated. | 756 | * -ENOMEM if memory could not be allocated. |
| 698 | */ | 757 | */ |
| 699 | static inline int xa_insert_bh(struct xarray *xa, unsigned long index, | 758 | static inline int __must_check xa_insert_bh(struct xarray *xa, |
| 700 | void *entry, gfp_t gfp) | 759 | unsigned long index, void *entry, gfp_t gfp) |
| 701 | { | 760 | { |
| 702 | int err; | 761 | int err; |
| 703 | 762 | ||
| @@ -722,11 +781,11 @@ static inline int xa_insert_bh(struct xarray *xa, unsigned long index, · 18 unmodified lines | |||
| 722 | * | 781 | * |
| 723 | * Context: Process context. Takes and releases the xa_lock while | 782 | * Context: Process context. Takes and releases the xa_lock while |
| 724 | * disabling interrupts. May sleep if the @gfp flags permit. | 783 | * disabling interrupts. May sleep if the @gfp flags permit. |
| 725 | * Return: 0 if the store succeeded. -EEXIST if another entry was present. | 784 | * Return: 0 if the store succeeded. -EBUSY if another entry was present. |
| 726 | * -ENOMEM if memory could not be allocated. | 785 | * -ENOMEM if memory could not be allocated. |
| 727 | */ | 786 | */ |
| 728 | static inline int xa_insert_irq(struct xarray *xa, unsigned long index, | 787 | static inline int __must_check xa_insert_irq(struct xarray *xa, |
| 729 | void *entry, gfp_t gfp) | 788 | unsigned long index, void *entry, gfp_t gfp) |
| 730 | { | 789 | { |
| 731 | int err; | 790 | int err; |
| 732 | 791 | ||
| @@ -741,26 +800,26 @@ static inline int xa_insert_irq(struct xarray *xa, unsigned long index, · 8 unmodified lines | |||
| 741 | * xa_alloc() - Find somewhere to store this entry in the XArray. | 800 | * xa_alloc() - Find somewhere to store this entry in the XArray. |
| 742 | * @xa: XArray. | 801 | * @xa: XArray. |
| 743 | * @id: Pointer to ID. | 802 | * @id: Pointer to ID. |
| 744 | * @max: Maximum ID to allocate (inclusive). | ||
| 745 | * @entry: New entry. | 803 | * @entry: New entry. |
| 804 | * @limit: Range of ID to allocate. | ||
| 746 | * @gfp: Memory allocation flags. | 805 | * @gfp: Memory allocation flags. |
| 747 | * | 806 | * |
| 748 | * Allocates an unused ID in the range specified by @id and @max. | 807 | * Finds an empty entry in @xa between @limit.min and @limit.max, |
| 749 | * Updates the @id pointer with the index, then stores the entry at that | 808 | * stores the index into the @id pointer, then stores the entry at |
| 750 | * index. A concurrent lookup will not see an uninitialised @id. | 809 | * that index. A concurrent lookup will not see an uninitialised @id. |
| 751 | * | 810 | * |
| 752 | * Context: Process context. Takes and releases the xa_lock. May sleep if | 811 | * Context: Any context. Takes and releases the xa_lock. May sleep if |
| 753 | * the @gfp flags permit. | 812 | * the @gfp flags permit. |
| 754 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | 813 | * Return: 0 on success, -ENOMEM if memory could not be allocated or |
| 755 | * there is no more space in the XArray. | 814 | * -EBUSY if there are no free entries in @limit. |
| 756 | */ | 815 | */ |
| 757 | static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, | 816 | static inline __must_check int xa_alloc(struct xarray *xa, u32 *id, |
| 758 | gfp_t gfp) | 817 | void *entry, struct xa_limit limit, gfp_t gfp) |
| 759 | { | 818 | { |
| 760 | int err; | 819 | int err; |
| 761 | 820 | ||
| 762 | xa_lock(xa); | 821 | xa_lock(xa); |
| 763 | err = __xa_alloc(xa, id, max, entry, gfp); | 822 | err = __xa_alloc(xa, id, entry, limit, gfp); |
| 764 | xa_unlock(xa); | 823 | xa_unlock(xa); |
| 765 | 824 | ||
| 766 | return err; | 825 | return err; |
| @@ -770,26 +829,26 @@ static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines | |||
| 770 | * xa_alloc_bh() - Find somewhere to store this entry in the XArray. | 829 | * xa_alloc_bh() - Find somewhere to store this entry in the XArray. |
| 771 | * @xa: XArray. | 830 | * @xa: XArray. |
| 772 | * @id: Pointer to ID. | 831 | * @id: Pointer to ID. |
| 773 | * @max: Maximum ID to allocate (inclusive). | ||
| 774 | * @entry: New entry. | 832 | * @entry: New entry. |
| 833 | * @limit: Range of ID to allocate. | ||
| 775 | * @gfp: Memory allocation flags. | 834 | * @gfp: Memory allocation flags. |
| 776 | * | 835 | * |
| 777 | * Allocates an unused ID in the range specified by @id and @max. | 836 | * Finds an empty entry in @xa between @limit.min and @limit.max, |
| 778 | * Updates the @id pointer with the index, then stores the entry at that | 837 | * stores the index into the @id pointer, then stores the entry at |
| 779 | * index. A concurrent lookup will not see an uninitialised @id. | 838 | * that index. A concurrent lookup will not see an uninitialised @id. |
| 780 | * | 839 | * |
| 781 | * Context: Any context. Takes and releases the xa_lock while | 840 | * Context: Any context. Takes and releases the xa_lock while |
| 782 | * disabling softirqs. May sleep if the @gfp flags permit. | 841 | * disabling softirqs. May sleep if the @gfp flags permit. |
| 783 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | 842 | * Return: 0 on success, -ENOMEM if memory could not be allocated or |
| 784 | * there is no more space in the XArray. | 843 | * -EBUSY if there are no free entries in @limit. |
| 785 | */ | 844 | */ |
| 786 | static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, | 845 | static inline int __must_check xa_alloc_bh(struct xarray *xa, u32 *id, |
| 787 | gfp_t gfp) | 846 | void *entry, struct xa_limit limit, gfp_t gfp) |
| 788 | { | 847 | { |
| 789 | int err; | 848 | int err; |
| 790 | 849 | ||
| 791 | xa_lock_bh(xa); | 850 | xa_lock_bh(xa); |
| 792 | err = __xa_alloc(xa, id, max, entry, gfp); | 851 | err = __xa_alloc(xa, id, entry, limit, gfp); |
| 793 | xa_unlock_bh(xa); | 852 | xa_unlock_bh(xa); |
| 794 | 853 | ||
| 795 | return err; | 854 | return err; |
| @@ -799,26 +858,125 @@ static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines | |||
| 799 | * xa_alloc_irq() - Find somewhere to store this entry in the XArray. | 858 | * xa_alloc_irq() - Find somewhere to store this entry in the XArray. |
| 800 | * @xa: XArray. | 859 | * @xa: XArray. |
| 801 | * @id: Pointer to ID. | 860 | * @id: Pointer to ID. |
| 802 | * @max: Maximum ID to allocate (inclusive). | ||
| 803 | * @entry: New entry. | 861 | * @entry: New entry. |
| 862 | * @limit: Range of ID to allocate. | ||
| 804 | * @gfp: Memory allocation flags. | 863 | * @gfp: Memory allocation flags. |
| 805 | * | 864 | * |
| 806 | * Allocates an unused ID in the range specified by @id and @max. | 865 | * Finds an empty entry in @xa between @limit.min and @limit.max, |
| 807 | * Updates the @id pointer with the index, then stores the entry at that | 866 | * stores the index into the @id pointer, then stores the entry at |
| 808 | * index. A concurrent lookup will not see an uninitialised @id. | 867 | * that index. A concurrent lookup will not see an uninitialised @id. |
| 809 | * | 868 | * |
| 810 | * Context: Process context. Takes and releases the xa_lock while | 869 | * Context: Process context. Takes and releases the xa_lock while |
| 811 | * disabling interrupts. May sleep if the @gfp flags permit. | 870 | * disabling interrupts. May sleep if the @gfp flags permit. |
| 812 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | 871 | * Return: 0 on success, -ENOMEM if memory could not be allocated or |
| 813 | * there is no more space in the XArray. | 872 | * -EBUSY if there are no free entries in @limit. |
| 814 | */ | 873 | */ |
| 815 | static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, | 874 | static inline int __must_check xa_alloc_irq(struct xarray *xa, u32 *id, |
| 816 | gfp_t gfp) | 875 | void *entry, struct xa_limit limit, gfp_t gfp) |
| 817 | { | 876 | { |
| 818 | int err; | 877 | int err; |
| 819 | 878 | ||
| 820 | xa_lock_irq(xa); | 879 | xa_lock_irq(xa); |
| 821 | err = __xa_alloc(xa, id, max, entry, gfp); | 880 | err = __xa_alloc(xa, id, entry, limit, gfp); |
| 881 | xa_unlock_irq(xa); | ||
| 882 | |||
| 883 | return err; | ||
| 884 | } | ||
| 885 | |||
| 886 | /** | ||
| 887 | * xa_alloc_cyclic() - Find somewhere to store this entry in the XArray. | ||
| 888 | * @xa: XArray. | ||
| 889 | * @id: Pointer to ID. | ||
| 890 | * @entry: New entry. | ||
| 891 | * @limit: Range of allocated ID. | ||
| 892 | * @next: Pointer to next ID to allocate. | ||
| 893 | * @gfp: Memory allocation flags. | ||
| 894 | * | ||
| 895 | * Finds an empty entry in @xa between @limit.min and @limit.max, | ||
| 896 | * stores the index into the @id pointer, then stores the entry at | ||
| 897 | * that index. A concurrent lookup will not see an uninitialised @id. | ||
| 898 | * The search for an empty entry will start at @next and will wrap | ||
| 899 | * around if necessary. | ||
| 900 | * | ||
| 901 | * Context: Any context. Takes and releases the xa_lock. May sleep if | ||
| 902 | * the @gfp flags permit. | ||
| 903 | * Return: 0 if the allocation succeeded without wrapping. 1 if the | ||
| 904 | * allocation succeeded after wrapping, -ENOMEM if memory could not be | ||
| 905 | * allocated or -EBUSY if there are no free entries in @limit. | ||
| 906 | */ | ||
| 907 | static inline int xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, | ||
| 908 | struct xa_limit limit, u32 *next, gfp_t gfp) | ||
| 909 | { | ||
| 910 | int err; | ||
| 911 | |||
| 912 | xa_lock(xa); | ||
| 913 | err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | ||
| 914 | xa_unlock(xa); | ||
| 915 | |||
| 916 | return err; | ||
| 917 | } | ||
| 918 | |||
| 919 | /** | ||
| 920 | * xa_alloc_cyclic_bh() - Find somewhere to store this entry in the XArray. | ||
| 921 | * @xa: XArray. | ||
| 922 | * @id: Pointer to ID. | ||
| 923 | * @entry: New entry. | ||
| 924 | * @limit: Range of allocated ID. | ||
| 925 | * @next: Pointer to next ID to allocate. | ||
| 926 | * @gfp: Memory allocation flags. | ||
| 927 | * | ||
| 928 | * Finds an empty entry in @xa between @limit.min and @limit.max, | ||
| 929 | * stores the index into the @id pointer, then stores the entry at | ||
| 930 | * that index. A concurrent lookup will not see an uninitialised @id. | ||
| 931 | * The search for an empty entry will start at @next and will wrap | ||
| 932 | * around if necessary. | ||
| 933 | * | ||
| 934 | * Context: Any context. Takes and releases the xa_lock while | ||
| 935 | * disabling softirqs. May sleep if the @gfp flags permit. | ||
| 936 | * Return: 0 if the allocation succeeded without wrapping. 1 if the | ||
| 937 | * allocation succeeded after wrapping, -ENOMEM if memory could not be | ||
| 938 | * allocated or -EBUSY if there are no free entries in @limit. | ||
| 939 | */ | ||
| 940 | static inline int xa_alloc_cyclic_bh(struct xarray *xa, u32 *id, void *entry, | ||
| 941 | struct xa_limit limit, u32 *next, gfp_t gfp) | ||
| 942 | { | ||
| 943 | int err; | ||
| 944 | |||
| 945 | xa_lock_bh(xa); | ||
| 946 | err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | ||
| 947 | xa_unlock_bh(xa); | ||
| 948 | |||
| 949 | return err; | ||
| 950 | } | ||
| 951 | |||
| 952 | /** | ||
| 953 | * xa_alloc_cyclic_irq() - Find somewhere to store this entry in the XArray. | ||
| 954 | * @xa: XArray. | ||
| 955 | * @id: Pointer to ID. | ||
| 956 | * @entry: New entry. | ||
| 957 | * @limit: Range of allocated ID. | ||
| 958 | * @next: Pointer to next ID to allocate. | ||
| 959 | * @gfp: Memory allocation flags. | ||
| 960 | * | ||
| 961 | * Finds an empty entry in @xa between @limit.min and @limit.max, | ||
| 962 | * stores the index into the @id pointer, then stores the entry at | ||
| 963 | * that index. A concurrent lookup will not see an uninitialised @id. | ||
| 964 | * The search for an empty entry will start at @next and will wrap | ||
| 965 | * around if necessary. | ||
| 966 | * | ||
| 967 | * Context: Process context. Takes and releases the xa_lock while | ||
| 968 | * disabling interrupts. May sleep if the @gfp flags permit. | ||
| 969 | * Return: 0 if the allocation succeeded without wrapping. 1 if the | ||
| 970 | * allocation succeeded after wrapping, -ENOMEM if memory could not be | ||
| 971 | * allocated or -EBUSY if there are no free entries in @limit. | ||
| 972 | */ | ||
| 973 | static inline int xa_alloc_cyclic_irq(struct xarray *xa, u32 *id, void *entry, | ||
| 974 | struct xa_limit limit, u32 *next, gfp_t gfp) | ||
| 975 | { | ||
| 976 | int err; | ||
| 977 | |||
| 978 | xa_lock_irq(xa); | ||
| 979 | err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | ||
| 822 | xa_unlock_irq(xa); | 980 | xa_unlock_irq(xa); |
| 823 | 981 | ||
| 824 | return err; | 982 | return err; |
| @@ -842,16 +1000,10 @@ static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, · 17 unmodified lines | |||
| 842 | * May sleep if the @gfp flags permit. | 1000 | * May sleep if the @gfp flags permit. |
| 843 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | 1001 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 844 | */ | 1002 | */ |
| 845 | static inline | 1003 | static inline __must_check |
| 846 | int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) | 1004 | int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 847 | { | 1005 | { |
| 848 | int ret; | 1006 | return xa_err(xa_cmpxchg(xa, index, NULL, XA_ZERO_ENTRY, gfp)); |
| 849 | |||
| 850 | xa_lock(xa); | ||
| 851 | ret = __xa_reserve(xa, index, gfp); | ||
| 852 | xa_unlock(xa); | ||
| 853 | |||
| 854 | return ret; | ||
| 855 | } | 1007 | } |
| 856 | 1008 | ||
| 857 | /** | 1009 | /** |
| @@ -866,16 +1018,10 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines | |||
| 866 | * disabling softirqs. | 1018 | * disabling softirqs. |
| 867 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | 1019 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 868 | */ | 1020 | */ |
| 869 | static inline | 1021 | static inline __must_check |
| 870 | int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) | 1022 | int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 871 | { | 1023 | { |
| 872 | int ret; | 1024 | return xa_err(xa_cmpxchg_bh(xa, index, NULL, XA_ZERO_ENTRY, gfp)); |
| 873 | |||
| 874 | xa_lock_bh(xa); | ||
| 875 | ret = __xa_reserve(xa, index, gfp); | ||
| 876 | xa_unlock_bh(xa); | ||
| 877 | |||
| 878 | return ret; | ||
| 879 | } | 1025 | } |
| 880 | 1026 | ||
| 881 | /** | 1027 | /** |
| @@ -890,16 +1036,10 @@ int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines | |||
| 890 | * disabling interrupts. | 1036 | * disabling interrupts. |
| 891 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | 1037 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 892 | */ | 1038 | */ |
| 893 | static inline | 1039 | static inline __must_check |
| 894 | int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) | 1040 | int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 895 | { | 1041 | { |
| 896 | int ret; | 1042 | return xa_err(xa_cmpxchg_irq(xa, index, NULL, XA_ZERO_ENTRY, gfp)); |
| 897 | |||
| 898 | xa_lock_irq(xa); | ||
| 899 | ret = __xa_reserve(xa, index, gfp); | ||
| 900 | xa_unlock_irq(xa); | ||
| 901 | |||
| 902 | return ret; | ||
| 903 | } | 1043 | } |
| 904 | 1044 | ||
| 905 | /** | 1045 | /** |
| @@ -913,7 +1053,7 @@ int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) · 7 unmodified lines | |||
| 913 | */ | 1053 | */ |
| 914 | static inline void xa_release(struct xarray *xa, unsigned long index) | 1054 | static inline void xa_release(struct xarray *xa, unsigned long index) |
| 915 | { | 1055 | { |
| 916 | xa_cmpxchg(xa, index, NULL, NULL, 0); | 1056 | xa_cmpxchg(xa, index, XA_ZERO_ENTRY, NULL, 0); |
| 917 | } | 1057 | } |
| 918 | 1058 | ||
| 919 | /* Everything below here is the Advanced API. Proceed with caution. */ | 1059 | /* Everything below here is the Advanced API. Proceed with caution. */ |
| @@ -1073,18 +1213,6 @@ static inline bool xa_is_sibling(const void *entry) · 153 unmodified lines | |||
| 1073 | } | 1213 | } |
| 1074 | 1214 | ||
| 1075 | #define XA_RETRY_ENTRY xa_mk_internal(256) | 1215 | #define XA_RETRY_ENTRY xa_mk_internal(256) |
| 1076 | #define XA_ZERO_ENTRY xa_mk_internal(257) | ||
| 1077 | |||
| 1078 | /** | ||
| 1079 | * xa_is_zero() - Is the entry a zero entry? | ||
| 1080 | * @entry: Entry retrieved from the XArray | ||
| 1081 | * | ||
| 1082 | * Return: %true if the entry is a zero entry. | ||
| 1083 | */ | ||
| 1084 | static inline bool xa_is_zero(const void *entry) | ||
| 1085 | { | ||
| 1086 | return unlikely(entry == XA_ZERO_ENTRY); | ||
| 1087 | } | ||
| 1088 | 1216 | ||
| 1089 | /** | 1217 | /** |
| 1090 | * xa_is_retry() - Is the entry a retry entry? | 1218 | * xa_is_retry() - Is the entry a retry entry? |
| @@ -40,9 +40,9 @@ static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp) · 39 unmodified lines | |||
| 40 | 40 | ||
| 41 | static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp) | 41 | static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 42 | { | 42 | { |
| 43 | u32 id = 0; | 43 | u32 id; |
| 44 | 44 | ||
| 45 | XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, xa_mk_index(index), | 45 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(index), xa_limit_32b, |
| 46 | gfp) != 0); | 46 | gfp) != 0); |
| 47 | XA_BUG_ON(xa, id != index); | 47 | XA_BUG_ON(xa, id != index); |
| 48 | } | 48 | } |
| @@ -107,8 +107,11 @@ static noinline void check_xas_retry(struct xarray *xa) · 58 unmodified lines | |||
| 107 | XA_BUG_ON(xa, xas.xa_node != XAS_RESTART); | 107 | XA_BUG_ON(xa, xas.xa_node != XAS_RESTART); |
| 108 | XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0)); | 108 | XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0)); |
| 109 | XA_BUG_ON(xa, xas.xa_node != NULL); | 109 | XA_BUG_ON(xa, xas.xa_node != NULL); |
| 110 | rcu_read_unlock(); | ||
| 110 | 111 | ||
| 111 | XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL); | 112 | XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL); |
| 113 | |||
| 114 | rcu_read_lock(); | ||
| 112 | XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas))); | 115 | XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas))); |
| 113 | xas.xa_node = XAS_RESTART; | 116 | xas.xa_node = XAS_RESTART; |
| 114 | XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0)); | 117 | XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0)); |
| @@ -343,7 +346,7 @@ static noinline void check_cmpxchg(struct xarray *xa) · 228 unmodified lines | |||
| 343 | 346 | ||
| 344 | XA_BUG_ON(xa, !xa_empty(xa)); | 347 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 345 | XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL); | 348 | XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL); |
| 346 | XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST); | 349 | XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EBUSY); |
| 347 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS); | 350 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS); |
| 348 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS); | 351 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS); |
| 349 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE); | 352 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE); |
| @@ -358,46 +361,65 @@ static noinline void check_reserve(struct xarray *xa) · 8 unmodified lines | |||
| 358 | { | 361 | { |
| 359 | void *entry; | 362 | void *entry; |
| 360 | unsigned long index; | 363 | unsigned long index; |
| 364 | int count; | ||
| 361 | 365 | ||
| 362 | /* An array with a reserved entry is not empty */ | 366 | /* An array with a reserved entry is not empty */ |
| 363 | XA_BUG_ON(xa, !xa_empty(xa)); | 367 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 364 | xa_reserve(xa, 12345678, GFP_KERNEL); | 368 | XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); |
| 365 | XA_BUG_ON(xa, xa_empty(xa)); | 369 | XA_BUG_ON(xa, xa_empty(xa)); |
| 366 | XA_BUG_ON(xa, xa_load(xa, 12345678)); | 370 | XA_BUG_ON(xa, xa_load(xa, 12345678)); |
| 367 | xa_release(xa, 12345678); | 371 | xa_release(xa, 12345678); |
| 368 | XA_BUG_ON(xa, !xa_empty(xa)); | 372 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 369 | 373 | ||
| 370 | /* Releasing a used entry does nothing */ | 374 | /* Releasing a used entry does nothing */ |
| 371 | xa_reserve(xa, 12345678, GFP_KERNEL); | 375 | XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); |
| 372 | XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_NOWAIT) != NULL); | 376 | XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_NOWAIT) != NULL); |
| 373 | xa_release(xa, 12345678); | 377 | xa_release(xa, 12345678); |
| 374 | xa_erase_index(xa, 12345678); | 378 | xa_erase_index(xa, 12345678); |
| 375 | XA_BUG_ON(xa, !xa_empty(xa)); | 379 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 376 | 380 | ||
| 377 | /* cmpxchg sees a reserved entry as NULL */ | 381 | /* cmpxchg sees a reserved entry as ZERO */ |
| 378 | xa_reserve(xa, 12345678, GFP_KERNEL); | 382 | XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); |
| 379 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, NULL, xa_mk_value(12345678), | 383 | XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, XA_ZERO_ENTRY, |
| 380 | GFP_NOWAIT) != NULL); | 384 | xa_mk_value(12345678), GFP_NOWAIT) != NULL); |
| 381 | xa_release(xa, 12345678); | 385 | xa_release(xa, 12345678); |
| 382 | xa_erase_index(xa, 12345678); | 386 | xa_erase_index(xa, 12345678); |
| 383 | XA_BUG_ON(xa, !xa_empty(xa)); | 387 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 384 | 388 | ||
| 385 | /* But xa_insert does not */ | 389 | /* xa_insert treats it as busy */ |
| 386 | xa_reserve(xa, 12345678, GFP_KERNEL); | 390 | XA_BUG_ON(xa, xa_reserve(xa, 12345678, GFP_KERNEL) != 0); |
| 387 | XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) != | 391 | XA_BUG_ON(xa, xa_insert(xa, 12345678, xa_mk_value(12345678), 0) != |
| 388 | -EEXIST); | 392 | -EBUSY); |
| 389 | XA_BUG_ON(xa, xa_empty(xa)); | 393 | XA_BUG_ON(xa, xa_empty(xa)); |
| 390 | XA_BUG_ON(xa, xa_erase(xa, 12345678) != NULL); | 394 | XA_BUG_ON(xa, xa_erase(xa, 12345678) != NULL); |
| 391 | XA_BUG_ON(xa, !xa_empty(xa)); | 395 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 392 | 396 | ||
| 393 | /* Can iterate through a reserved entry */ | 397 | /* Can iterate through a reserved entry */ |
| 394 | xa_store_index(xa, 5, GFP_KERNEL); | 398 | xa_store_index(xa, 5, GFP_KERNEL); |
| 395 | xa_reserve(xa, 6, GFP_KERNEL); | 399 | XA_BUG_ON(xa, xa_reserve(xa, 6, GFP_KERNEL) != 0); |
| 396 | xa_store_index(xa, 7, GFP_KERNEL); | 400 | xa_store_index(xa, 7, GFP_KERNEL); |
| 397 | 401 | ||
| 402 | count = 0; | ||
| 398 | xa_for_each(xa, index, entry) { | 403 | xa_for_each(xa, index, entry) { |
| 399 | XA_BUG_ON(xa, index != 5 && index != 7); | 404 | XA_BUG_ON(xa, index != 5 && index != 7); |
| 405 | count++; | ||
| 406 | } | ||
| 407 | XA_BUG_ON(xa, count != 2); | ||
| 408 | |||
| 409 | /* If we free a reserved entry, we should be able to allocate it */ | ||
| 410 | if (xa->xa_flags & XA_FLAGS_ALLOC) { | ||
| 411 | u32 id; | ||
| 412 | |||
| 413 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(8), | ||
| 414 | XA_LIMIT(5, 10), GFP_KERNEL) != 0); | ||
| 415 | XA_BUG_ON(xa, id != 8); | ||
| 416 | |||
| 417 | xa_release(xa, 6); | ||
| 418 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_value(6), | ||
| 419 | XA_LIMIT(5, 10), GFP_KERNEL) != 0); | ||
| 420 | XA_BUG_ON(xa, id != 6); | ||
| 400 | } | 421 | } |
| 422 | |||
| 401 | xa_destroy(xa); | 423 | xa_destroy(xa); |
| 402 | } | 424 | } |
| 403 | 425 | ||
| @@ -586,64 +608,194 @@ static noinline void check_multi_store(struct xarray *xa) · 182 unmodified lines | |||
| 586 | #endif | 608 | #endif |
| 587 | } | 609 | } |
| 588 | 610 | ||
| 589 | static DEFINE_XARRAY_ALLOC(xa0); | 611 | static noinline void check_xa_alloc_1(struct xarray *xa, unsigned int base) |
| 590 | |||
| 591 | static noinline void check_xa_alloc(void) | ||
| 592 | { | 612 | { |
| 593 | int i; | 613 | int i; |
| 594 | u32 id; | 614 | u32 id; |
| 595 | 615 | ||
| 596 | /* An empty array should assign 0 to the first alloc */ | 616 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 597 | xa_alloc_index(&xa0, 0, GFP_KERNEL); | 617 | /* An empty array should assign %base to the first alloc */ |
| 618 | xa_alloc_index(xa, base, GFP_KERNEL); | ||
| 598 | 619 | ||
| 599 | /* Erasing it should make the array empty again */ | 620 | /* Erasing it should make the array empty again */ |
| 600 | xa_erase_index(&xa0, 0); | 621 | xa_erase_index(xa, base); |
| 601 | XA_BUG_ON(&xa0, !xa_empty(&xa0)); | 622 | XA_BUG_ON(xa, !xa_empty(xa)); |
| 623 | |||
| 624 | /* And it should assign %base again */ | ||
| 625 | xa_alloc_index(xa, base, GFP_KERNEL); | ||
| 626 | |||
| 627 | /* Allocating and then erasing a lot should not lose base */ | ||
| 628 | for (i = base + 1; i < 2 * XA_CHUNK_SIZE; i++) | ||
| 629 | xa_alloc_index(xa, i, GFP_KERNEL); | ||
| 630 | for (i = base; i < 2 * XA_CHUNK_SIZE; i++) | ||
| 631 | xa_erase_index(xa, i); | ||
| 632 | xa_alloc_index(xa, base, GFP_KERNEL); | ||
| 602 | 633 | ||
| 603 | /* And it should assign 0 again */ | 634 | /* Destroying the array should do the same as erasing */ |
| 604 | xa_alloc_index(&xa0, 0, GFP_KERNEL); | 635 | xa_destroy(xa); |
| 636 | |||
| 637 | /* And it should assign %base again */ | ||
| 638 | xa_alloc_index(xa, base, GFP_KERNEL); | ||
| 605 | 639 | ||
| 606 | /* The next assigned ID should be 1 */ | 640 | /* The next assigned ID should be base+1 */ |
| 607 | xa_alloc_index(&xa0, 1, GFP_KERNEL); | 641 | xa_alloc_index(xa, base + 1, GFP_KERNEL); |
| 608 | xa_erase_index(&xa0, 1); | 642 | xa_erase_index(xa, base + 1); |
| 609 | 643 | ||
| 610 | /* Storing a value should mark it used */ | 644 | /* Storing a value should mark it used */ |
| 611 | xa_store_index(&xa0, 1, GFP_KERNEL); | 645 | xa_store_index(xa, base + 1, GFP_KERNEL); |
| 612 | xa_alloc_index(&xa0, 2, GFP_KERNEL); | 646 | xa_alloc_index(xa, base + 2, GFP_KERNEL); |
| 613 | 647 | ||
| 614 | /* If we then erase 0, it should be free */ | 648 | /* If we then erase base, it should be free */ |
| 615 | xa_erase_index(&xa0, 0); | 649 | xa_erase_index(xa, base); |
| 616 | xa_alloc_index(&xa0, 0, GFP_KERNEL); | 650 | xa_alloc_index(xa, base, GFP_KERNEL); |
| 617 | 651 | ||
| 618 | xa_erase_index(&xa0, 1); | 652 | xa_erase_index(xa, base + 1); |
| 619 | xa_erase_index(&xa0, 2); | 653 | xa_erase_index(xa, base + 2); |
| 620 | 654 | ||
| 621 | for (i = 1; i < 5000; i++) { | 655 | for (i = 1; i < 5000; i++) { |
| 622 | xa_alloc_index(&xa0, i, GFP_KERNEL); | 656 | xa_alloc_index(xa, base + i, GFP_KERNEL); |
| 623 | } | 657 | } |
| 624 | 658 | ||
| 625 | xa_destroy(&xa0); | 659 | xa_destroy(xa); |
| 626 | 660 | ||
| 627 | id = 0xfffffffeU; | 661 | /* Check that we fail properly at the limit of allocation */ |
| 628 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id), | 662 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX - 1), |
| 663 | XA_LIMIT(UINT_MAX - 1, UINT_MAX), | ||
| 629 | GFP_KERNEL) != 0); | 664 | GFP_KERNEL) != 0); |
| 630 | XA_BUG_ON(&xa0, id != 0xfffffffeU); | 665 | XA_BUG_ON(xa, id != 0xfffffffeU); |
| 631 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id), | 666 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(UINT_MAX), |
| 667 | XA_LIMIT(UINT_MAX - 1, UINT_MAX), | ||
| 632 | GFP_KERNEL) != 0); | 668 | GFP_KERNEL) != 0); |
| 633 | XA_BUG_ON(&xa0, id != 0xffffffffU); | 669 | XA_BUG_ON(xa, id != 0xffffffffU); |
| 634 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_index(id), | 670 | id = 3; |
| 635 | GFP_KERNEL) != -ENOSPC); | 671 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(0), |
| 636 | XA_BUG_ON(&xa0, id != 0xffffffffU); | 672 | XA_LIMIT(UINT_MAX - 1, UINT_MAX), |
| 637 | xa_destroy(&xa0); | 673 | GFP_KERNEL) != -EBUSY); |
| 638 | 674 | XA_BUG_ON(xa, id != 3); | |
| 639 | id = 10; | 675 | xa_destroy(xa); |
| 640 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id), | 676 | |
| 641 | GFP_KERNEL) != -ENOSPC); | 677 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5), |
| 642 | XA_BUG_ON(&xa0, xa_store_index(&xa0, 3, GFP_KERNEL) != 0); | 678 | GFP_KERNEL) != -EBUSY); |
| 643 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, 5, xa_mk_index(id), | 679 | XA_BUG_ON(xa, xa_store_index(xa, 3, GFP_KERNEL) != 0); |
| 644 | GFP_KERNEL) != -ENOSPC); | 680 | XA_BUG_ON(xa, xa_alloc(xa, &id, xa_mk_index(10), XA_LIMIT(10, 5), |
| 645 | xa_erase_index(&xa0, 3); | 681 | GFP_KERNEL) != -EBUSY); |
| 646 | XA_BUG_ON(&xa0, !xa_empty(&xa0)); | 682 | xa_erase_index(xa, 3); |
| 683 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 684 | } | ||
| 685 | |||
| 686 | static noinline void check_xa_alloc_2(struct xarray *xa, unsigned int base) | ||
| 687 | { | ||
| 688 | unsigned int i, id; | ||
| 689 | unsigned long index; | ||
| 690 | void *entry; | ||
| 691 | |||
| 692 | /* Allocate and free a NULL and check xa_empty() behaves */ | ||
| 693 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 694 | XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0); | ||
| 695 | XA_BUG_ON(xa, id != base); | ||
| 696 | XA_BUG_ON(xa, xa_empty(xa)); | ||
| 697 | XA_BUG_ON(xa, xa_erase(xa, id) != NULL); | ||
| 698 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 699 | |||
| 700 | /* Ditto, but check destroy instead of erase */ | ||
| 701 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 702 | XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0); | ||
| 703 | XA_BUG_ON(xa, id != base); | ||
| 704 | XA_BUG_ON(xa, xa_empty(xa)); | ||
| 705 | xa_destroy(xa); | ||
| 706 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 707 | |||
| 708 | for (i = base; i < base + 10; i++) { | ||
| 709 | XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, | ||
| 710 | GFP_KERNEL) != 0); | ||
| 711 | XA_BUG_ON(xa, id != i); | ||
| 712 | } | ||
| 713 | |||
| 714 | XA_BUG_ON(xa, xa_store(xa, 3, xa_mk_index(3), GFP_KERNEL) != NULL); | ||
| 715 | XA_BUG_ON(xa, xa_store(xa, 4, xa_mk_index(4), GFP_KERNEL) != NULL); | ||
| 716 | XA_BUG_ON(xa, xa_store(xa, 4, NULL, GFP_KERNEL) != xa_mk_index(4)); | ||
| 717 | XA_BUG_ON(xa, xa_erase(xa, 5) != NULL); | ||
| 718 | XA_BUG_ON(xa, xa_alloc(xa, &id, NULL, xa_limit_32b, GFP_KERNEL) != 0); | ||
| 719 | XA_BUG_ON(xa, id != 5); | ||
| 720 | |||
| 721 | xa_for_each(xa, index, entry) { | ||
| 722 | xa_erase_index(xa, index); | ||
| 723 | } | ||
| 724 | |||
| 725 | for (i = base; i < base + 9; i++) { | ||
| 726 | XA_BUG_ON(xa, xa_erase(xa, i) != NULL); | ||
| 727 | XA_BUG_ON(xa, xa_empty(xa)); | ||
| 728 | } | ||
| 729 | XA_BUG_ON(xa, xa_erase(xa, 8) != NULL); | ||
| 730 | XA_BUG_ON(xa, xa_empty(xa)); | ||
| 731 | XA_BUG_ON(xa, xa_erase(xa, base + 9) != NULL); | ||
| 732 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 733 | |||
| 734 | xa_destroy(xa); | ||
| 735 | } | ||
| 736 | |||
| 737 | static noinline void check_xa_alloc_3(struct xarray *xa, unsigned int base) | ||
| 738 | { | ||
| 739 | struct xa_limit limit = XA_LIMIT(1, 0x3fff); | ||
| 740 | u32 next = 0; | ||
| 741 | unsigned int i, id; | ||
| 742 | unsigned long index; | ||
| 743 | void *entry; | ||
| 744 | |||
| 745 | XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(1), limit, | ||
| 746 | &next, GFP_KERNEL) != 0); | ||
| 747 | XA_BUG_ON(xa, id != 1); | ||
| 748 | |||
| 749 | next = 0x3ffd; | ||
| 750 | XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(0x3ffd), limit, | ||
| 751 | &next, GFP_KERNEL) != 0); | ||
| 752 | XA_BUG_ON(xa, id != 0x3ffd); | ||
| 753 | xa_erase_index(xa, 0x3ffd); | ||
| 754 | xa_erase_index(xa, 1); | ||
| 755 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 756 | |||
| 757 | for (i = 0x3ffe; i < 0x4003; i++) { | ||
| 758 | if (i < 0x4000) | ||
| 759 | entry = xa_mk_index(i); | ||
| 760 | else | ||
| 761 | entry = xa_mk_index(i - 0x3fff); | ||
| 762 | XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, entry, limit, | ||
| 763 | &next, GFP_KERNEL) != (id == 1)); | ||
| 764 | XA_BUG_ON(xa, xa_mk_index(id) != entry); | ||
| 765 | } | ||
| 766 | |||
| 767 | /* Check wrap-around is handled correctly */ | ||
| 768 | if (base != 0) | ||
| 769 | xa_erase_index(xa, base); | ||
| 770 | xa_erase_index(xa, base + 1); | ||
| 771 | next = UINT_MAX; | ||
| 772 | XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(UINT_MAX), | ||
| 773 | xa_limit_32b, &next, GFP_KERNEL) != 0); | ||
| 774 | XA_BUG_ON(xa, id != UINT_MAX); | ||
| 775 | XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base), | ||
| 776 | xa_limit_32b, &next, GFP_KERNEL) != 1); | ||
| 777 | XA_BUG_ON(xa, id != base); | ||
| 778 | XA_BUG_ON(xa, xa_alloc_cyclic(xa, &id, xa_mk_index(base + 1), | ||
| 779 | xa_limit_32b, &next, GFP_KERNEL) != 0); | ||
| 780 | XA_BUG_ON(xa, id != base + 1); | ||
| 781 | |||
| 782 | xa_for_each(xa, index, entry) | ||
| 783 | xa_erase_index(xa, index); | ||
| 784 | |||
| 785 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 786 | } | ||
| 787 | |||
| 788 | static DEFINE_XARRAY_ALLOC(xa0); | ||
| 789 | static DEFINE_XARRAY_ALLOC1(xa1); | ||
| 790 | |||
| 791 | static noinline void check_xa_alloc(void) | ||
| 792 | { | ||
| 793 | check_xa_alloc_1(&xa0, 0); | ||
| 794 | check_xa_alloc_1(&xa1, 1); | ||
| 795 | check_xa_alloc_2(&xa0, 0); | ||
| 796 | check_xa_alloc_2(&xa1, 1); | ||
| 797 | check_xa_alloc_3(&xa0, 0); | ||
| 798 | check_xa_alloc_3(&xa1, 1); | ||
| 647 | } | 799 | } |
| 648 | 800 | ||
| 649 | static noinline void __check_store_iter(struct xarray *xa, unsigned long start, | 801 | static noinline void __check_store_iter(struct xarray *xa, unsigned long start, |
| @@ -1194,9 +1346,8 @@ static void check_align_1(struct xarray *xa, char *name) · 544 unmodified lines | |||
| 1194 | void *entry; | 1346 | void *entry; |
| 1195 | 1347 | ||
| 1196 | for (i = 0; i < 8; i++) { | 1348 | for (i = 0; i < 8; i++) { |
| 1197 | id = 0; | 1349 | XA_BUG_ON(xa, xa_alloc(xa, &id, name + i, xa_limit_32b, |
| 1198 | XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, name + i, GFP_KERNEL) | 1350 | GFP_KERNEL) != 0); |
| 1199 | != 0); | ||
| 1200 | XA_BUG_ON(xa, id != i); | 1351 | XA_BUG_ON(xa, id != i); |
| 1201 | } | 1352 | } |
| 1202 | xa_for_each(xa, index, entry) | 1353 | xa_for_each(xa, index, entry) |
| @@ -1204,6 +1355,30 @@ static void check_align_1(struct xarray *xa, char *name) · 1 unmodified line | |||
| 1204 | xa_destroy(xa); | 1355 | xa_destroy(xa); |
| 1205 | } | 1356 | } |
| 1206 | 1357 | ||
| 1358 | /* | ||
| 1359 | * We should always be able to store without allocating memory after | ||
| 1360 | * reserving a slot. | ||
| 1361 | */ | ||
| 1362 | static void check_align_2(struct xarray *xa, char *name) | ||
| 1363 | { | ||
| 1364 | int i; | ||
| 1365 | |||
| 1366 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 1367 | |||
| 1368 | for (i = 0; i < 8; i++) { | ||
| 1369 | XA_BUG_ON(xa, xa_store(xa, 0, name + i, GFP_KERNEL) != NULL); | ||
| 1370 | xa_erase(xa, 0); | ||
| 1371 | } | ||
| 1372 | |||
| 1373 | for (i = 0; i < 8; i++) { | ||
| 1374 | XA_BUG_ON(xa, xa_reserve(xa, 0, GFP_KERNEL) != 0); | ||
| 1375 | XA_BUG_ON(xa, xa_store(xa, 0, name + i, 0) != NULL); | ||
| 1376 | xa_erase(xa, 0); | ||
| 1377 | } | ||
| 1378 | |||
| 1379 | XA_BUG_ON(xa, !xa_empty(xa)); | ||
| 1380 | } | ||
| 1381 | |||
| 1207 | static noinline void check_align(struct xarray *xa) | 1382 | static noinline void check_align(struct xarray *xa) |
| 1208 | { | 1383 | { |
| 1209 | char name[] = "Motorola 68000"; | 1384 | char name[] = "Motorola 68000"; |
| @@ -1212,7 +1387,7 @@ static noinline void check_align(struct xarray *xa) · 2 unmodified lines | |||
| 1212 | check_align_1(xa, name + 1); | 1387 | check_align_1(xa, name + 1); |
| 1213 | check_align_1(xa, name + 2); | 1388 | check_align_1(xa, name + 2); |
| 1214 | check_align_1(xa, name + 3); | 1389 | check_align_1(xa, name + 3); |
| 1215 | // check_align_2(xa, name); | 1390 | check_align_2(xa, name); |
| 1216 | } | 1391 | } |
| 1217 | 1392 | ||
| 1218 | static LIST_HEAD(shadow_nodes); | 1393 | static LIST_HEAD(shadow_nodes); |
| @@ -1354,6 +1529,7 @@ static int xarray_checks(void) · 135 unmodified lines | |||
| 1354 | check_xas_erase(&array); | 1529 | check_xas_erase(&array); |
| 1355 | check_cmpxchg(&array); | 1530 | check_cmpxchg(&array); |
| 1356 | check_reserve(&array); | 1531 | check_reserve(&array); |
| 1532 | check_reserve(&xa0); | ||
| 1357 | check_multi_store(&array); | 1533 | check_multi_store(&array); |
| 1358 | check_xa_alloc(); | 1534 | check_xa_alloc(); |
| 1359 | check_find(&array); | 1535 | check_find(&array); |
| @@ -57,6 +57,11 @@ static inline bool xa_track_free(const struct xarray *xa) · 56 unmodified lines | |||
| 57 | return xa->xa_flags & XA_FLAGS_TRACK_FREE; | 57 | return xa->xa_flags & XA_FLAGS_TRACK_FREE; |
| 58 | } | 58 | } |
| 59 | 59 | ||
| 60 | static inline bool xa_zero_busy(const struct xarray *xa) | ||
| 61 | { | ||
| 62 | return xa->xa_flags & XA_FLAGS_ZERO_BUSY; | ||
| 63 | } | ||
| 64 | |||
| 60 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) | 65 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) |
| 61 | { | 66 | { |
| 62 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) | 67 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) |
| @@ -432,6 +437,8 @@ static void xas_shrink(struct xa_state *xas) · 369 unmodified lines | |||
| 432 | break; | 437 | break; |
| 433 | if (!xa_is_node(entry) && node->shift) | 438 | if (!xa_is_node(entry) && node->shift) |
| 434 | break; | 439 | break; |
| 440 | if (xa_is_zero(entry) && xa_zero_busy(xa)) | ||
| 441 | entry = NULL; | ||
| 435 | xas->xa_node = XAS_BOUNDS; | 442 | xas->xa_node = XAS_BOUNDS; |
| 436 | 443 | ||
| 437 | RCU_INIT_POINTER(xa->xa_head, entry); | 444 | RCU_INIT_POINTER(xa->xa_head, entry); |
| @@ -628,6 +635,8 @@ static void *xas_create(struct xa_state *xas, bool allow_root) · 190 unmodified lines | |||
| 628 | if (xas_top(node)) { | 635 | if (xas_top(node)) { |
| 629 | entry = xa_head_locked(xa); | 636 | entry = xa_head_locked(xa); |
| 630 | xas->xa_node = NULL; | 637 | xas->xa_node = NULL; |
| 638 | if (!entry && xa_zero_busy(xa)) | ||
| 639 | entry = XA_ZERO_ENTRY; | ||
| 631 | shift = xas_expand(xas, entry); | 640 | shift = xas_expand(xas, entry); |
| 632 | if (shift < 0) | 641 | if (shift < 0) |
| 633 | return NULL; | 642 | return NULL; |
| @@ -758,10 +767,12 @@ void *xas_store(struct xa_state *xas, void *entry) · 124 unmodified lines | |||
| 758 | void *first, *next; | 767 | void *first, *next; |
| 759 | bool value = xa_is_value(entry); | 768 | bool value = xa_is_value(entry); |
| 760 | 769 | ||
| 761 | if (entry) | 770 | if (entry) { |
| 762 | first = xas_create(xas, !xa_is_node(entry)); | 771 | bool allow_root = !xa_is_node(entry) && !xa_is_zero(entry); |
| 763 | else | 772 | first = xas_create(xas, allow_root); |
| 773 | } else { | ||
| 764 | first = xas_load(xas); | 774 | first = xas_load(xas); |
| 775 | } | ||
| 765 | 776 | ||
| 766 | if (xas_invalid(xas)) | 777 | if (xas_invalid(xas)) |
| 767 | return first; | 778 | return first; |
| @@ -791,7 +802,7 @@ void *xas_store(struct xa_state *xas, void *entry) · 23 unmodified lines | |||
| 791 | * entry is set to NULL. | 802 | * entry is set to NULL. |
| 792 | */ | 803 | */ |
| 793 | rcu_assign_pointer(*slot, entry); | 804 | rcu_assign_pointer(*slot, entry); |
| 794 | if (xa_is_node(next)) | 805 | if (xa_is_node(next) && (!node || node->shift)) |
| 795 | xas_free_nodes(xas, xa_to_node(next)); | 806 | xas_free_nodes(xas, xa_to_node(next)); |
| 796 | if (!node) | 807 | if (!node) |
| 797 | break; | 808 | break; |
| @@ -1294,13 +1305,12 @@ static void *xas_result(struct xa_state *xas, void *curr) · 496 unmodified lines | |||
| 1294 | * @xa: XArray. | 1305 | * @xa: XArray. |
| 1295 | * @index: Index into array. | 1306 | * @index: Index into array. |
| 1296 | * | 1307 | * |
| 1297 | * If the entry at this index is a multi-index entry then all indices will | 1308 | * After this function returns, loading from @index will return %NULL. |
| 1298 | * be erased, and the entry will no longer be a multi-index entry. | 1309 | * If the index is part of a multi-index entry, all indices will be erased |
| 1299 | * This function expects the xa_lock to be held on entry. | 1310 | * and none of the entries will be part of a multi-index entry. |
| 1300 | * | 1311 | * |
| 1301 | * Context: Any context. Expects xa_lock to be held on entry. May | 1312 | * Context: Any context. Expects xa_lock to be held on entry. |
| 1302 | * release and reacquire xa_lock if @gfp flags permit. | 1313 | * Return: The entry which used to be at this index. |
| 1303 | * Return: The old entry at this index. | ||
| 1304 | */ | 1314 | */ |
| 1305 | void *__xa_erase(struct xarray *xa, unsigned long index) | 1315 | void *__xa_erase(struct xarray *xa, unsigned long index) |
| 1306 | { | 1316 | { |
| @@ -1314,9 +1324,9 @@ EXPORT_SYMBOL(__xa_erase); · 7 unmodified lines | |||
| 1314 | * @xa: XArray. | 1324 | * @xa: XArray. |
| 1315 | * @index: Index of entry. | 1325 | * @index: Index of entry. |
| 1316 | * | 1326 | * |
| 1317 | * This function is the equivalent of calling xa_store() with %NULL as | 1327 | * After this function returns, loading from @index will return %NULL. |
| 1318 | * the third argument. The XArray does not need to allocate memory, so | 1328 | * If the index is part of a multi-index entry, all indices will be erased |
| 1319 | * the user does not need to provide GFP flags. | 1329 | * and none of the entries will be part of a multi-index entry. |
| 1320 | * | 1330 | * |
| 1321 | * Context: Any context. Takes and releases the xa_lock. | 1331 | * Context: Any context. Takes and releases the xa_lock. |
| 1322 | * Return: The entry which used to be at this index. | 1332 | * Return: The entry which used to be at this index. |
| @@ -1421,16 +1431,12 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, · 98 unmodified lines | |||
| 1421 | 1431 | ||
| 1422 | if (WARN_ON_ONCE(xa_is_advanced(entry))) | 1432 | if (WARN_ON_ONCE(xa_is_advanced(entry))) |
| 1423 | return XA_ERROR(-EINVAL); | 1433 | return XA_ERROR(-EINVAL); |
| 1424 | if (xa_track_free(xa) && !entry) | ||
| 1425 | entry = XA_ZERO_ENTRY; | ||
| 1426 | 1434 | ||
| 1427 | do { | 1435 | do { |
| 1428 | curr = xas_load(&xas); | 1436 | curr = xas_load(&xas); |
| 1429 | if (curr == XA_ZERO_ENTRY) | ||
| 1430 | curr = NULL; | ||
| 1431 | if (curr == old) { | 1437 | if (curr == old) { |
| 1432 | xas_store(&xas, entry); | 1438 | xas_store(&xas, entry); |
| 1433 | if (xa_track_free(xa)) | 1439 | if (xa_track_free(xa) && entry && !curr) |
| 1434 | xas_clear_mark(&xas, XA_FREE_MARK); | 1440 | xas_clear_mark(&xas, XA_FREE_MARK); |
| 1435 | } | 1441 | } |
| 1436 | } while (__xas_nomem(&xas, gfp)); | 1442 | } while (__xas_nomem(&xas, gfp)); |
| @@ -1452,7 +1458,7 @@ EXPORT_SYMBOL(__xa_cmpxchg); · 15 unmodified lines | |||
| 1452 | * | 1458 | * |
| 1453 | * Context: Any context. Expects xa_lock to be held on entry. May | 1459 | * Context: Any context. Expects xa_lock to be held on entry. May |
| 1454 | * release and reacquire xa_lock if @gfp flags permit. | 1460 | * release and reacquire xa_lock if @gfp flags permit. |
| 1455 | * Return: 0 if the store succeeded. -EEXIST if another entry was present. | 1461 | * Return: 0 if the store succeeded. -EBUSY if another entry was present. |
| 1456 | * -ENOMEM if memory could not be allocated. | 1462 | * -ENOMEM if memory could not be allocated. |
| 1457 | */ | 1463 | */ |
| 1458 | int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) | 1464 | int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) |
| @@ -1472,7 +1478,7 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 13 unmodified lines | |||
| 1472 | if (xa_track_free(xa)) | 1478 | if (xa_track_free(xa)) |
| 1473 | xas_clear_mark(&xas, XA_FREE_MARK); | 1479 | xas_clear_mark(&xas, XA_FREE_MARK); |
| 1474 | } else { | 1480 | } else { |
| 1475 | xas_set_err(&xas, -EEXIST); | 1481 | xas_set_err(&xas, -EBUSY); |
| 1476 | } | 1482 | } |
| 1477 | } while (__xas_nomem(&xas, gfp)); | 1483 | } while (__xas_nomem(&xas, gfp)); |
| 1478 | 1484 | ||
| @@ -1480,42 +1486,6 @@ int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 1 unmodified line | |||
| 1480 | } | 1486 | } |
| 1481 | EXPORT_SYMBOL(__xa_insert); | 1487 | EXPORT_SYMBOL(__xa_insert); |
| 1482 | 1488 | ||
| 1483 | /** | ||
| 1484 | * __xa_reserve() - Reserve this index in the XArray. | ||
| 1485 | * @xa: XArray. | ||
| 1486 | * @index: Index into array. | ||
| 1487 | * @gfp: Memory allocation flags. | ||
| 1488 | * | ||
| 1489 | * Ensures there is somewhere to store an entry at @index in the array. | ||
| 1490 | * If there is already something stored at @index, this function does | ||
| 1491 | * nothing. If there was nothing there, the entry is marked as reserved. | ||
| 1492 | * Loading from a reserved entry returns a %NULL pointer. | ||
| 1493 | * | ||
| 1494 | * If you do not use the entry that you have reserved, call xa_release() | ||
| 1495 | * or xa_erase() to free any unnecessary memory. | ||
| 1496 | * | ||
| 1497 | * Context: Any context. Expects the xa_lock to be held on entry. May | ||
| 1498 | * release the lock, sleep and reacquire the lock if the @gfp flags permit. | ||
| 1499 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | ||
| 1500 | */ | ||
| 1501 | int __xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) | ||
| 1502 | { | ||
| 1503 | XA_STATE(xas, xa, index); | ||
| 1504 | void *curr; | ||
| 1505 | |||
| 1506 | do { | ||
| 1507 | curr = xas_load(&xas); | ||
| 1508 | if (!curr) { | ||
| 1509 | xas_store(&xas, XA_ZERO_ENTRY); | ||
| 1510 | if (xa_track_free(xa)) | ||
| 1511 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1512 | } | ||
| 1513 | } while (__xas_nomem(&xas, gfp)); | ||
| 1514 | |||
| 1515 | return xas_error(&xas); | ||
| 1516 | } | ||
| 1517 | EXPORT_SYMBOL(__xa_reserve); | ||
| 1518 | |||
| 1519 | #ifdef CONFIG_XARRAY_MULTI | 1489 | #ifdef CONFIG_XARRAY_MULTI |
| 1520 | static void xas_set_range(struct xa_state *xas, unsigned long first, | 1490 | static void xas_set_range(struct xa_state *xas, unsigned long first, |
| 1521 | unsigned long last) | 1491 | unsigned long last) |
| @@ -1607,23 +1577,23 @@ EXPORT_SYMBOL(xa_store_range); · 85 unmodified lines | |||
| 1607 | * __xa_alloc() - Find somewhere to store this entry in the XArray. | 1577 | * __xa_alloc() - Find somewhere to store this entry in the XArray. |
| 1608 | * @xa: XArray. | 1578 | * @xa: XArray. |
| 1609 | * @id: Pointer to ID. | 1579 | * @id: Pointer to ID. |
| 1610 | * @max: Maximum ID to allocate (inclusive). | 1580 | * @limit: Range for allocated ID. |
| 1611 | * @entry: New entry. | 1581 | * @entry: New entry. |
| 1612 | * @gfp: Memory allocation flags. | 1582 | * @gfp: Memory allocation flags. |
| 1613 | * | 1583 | * |
| 1614 | * Allocates an unused ID in the range specified by @id and @max. | 1584 | * Finds an empty entry in @xa between @limit.min and @limit.max, |
| 1615 | * Updates the @id pointer with the index, then stores the entry at that | 1585 | * stores the index into the @id pointer, then stores the entry at |
| 1616 | * index. A concurrent lookup will not see an uninitialised @id. | 1586 | * that index. A concurrent lookup will not see an uninitialised @id. |
| 1617 | * | 1587 | * |
| 1618 | * Context: Any context. Expects xa_lock to be held on entry. May | 1588 | * Context: Any context. Expects xa_lock to be held on entry. May |
| 1619 | * release and reacquire xa_lock if @gfp flags permit. | 1589 | * release and reacquire xa_lock if @gfp flags permit. |
| 1620 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | 1590 | * Return: 0 on success, -ENOMEM if memory could not be allocated or |
| 1621 | * there is no more space in the XArray. | 1591 | * -EBUSY if there are no free entries in @limit. |
| 1622 | */ | 1592 | */ |
| 1623 | int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) | 1593 | int __xa_alloc(struct xarray *xa, u32 *id, void *entry, |
| 1594 | struct xa_limit limit, gfp_t gfp) | ||
| 1624 | { | 1595 | { |
| 1625 | XA_STATE(xas, xa, 0); | 1596 | XA_STATE(xas, xa, 0); |
| 1626 | int err; | ||
| 1627 | 1597 | ||
| 1628 | if (WARN_ON_ONCE(xa_is_advanced(entry))) | 1598 | if (WARN_ON_ONCE(xa_is_advanced(entry))) |
| 1629 | return -EINVAL; | 1599 | return -EINVAL; |
| @@ -1634,21 +1604,70 @@ int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) · 4 unmodified lines | |||
| 1634 | entry = XA_ZERO_ENTRY; | 1604 | entry = XA_ZERO_ENTRY; |
| 1635 | 1605 | ||
| 1636 | do { | 1606 | do { |
| 1637 | xas.xa_index = *id; | 1607 | xas.xa_index = limit.min; |
| 1638 | xas_find_marked(&xas, max, XA_FREE_MARK); | 1608 | xas_find_marked(&xas, limit.max, XA_FREE_MARK); |
| 1639 | if (xas.xa_node == XAS_RESTART) | 1609 | if (xas.xa_node == XAS_RESTART) |
| 1640 | xas_set_err(&xas, -ENOSPC); | 1610 | xas_set_err(&xas, -EBUSY); |
| 1611 | else | ||
| 1612 | *id = xas.xa_index; | ||
| 1641 | xas_store(&xas, entry); | 1613 | xas_store(&xas, entry); |
| 1642 | xas_clear_mark(&xas, XA_FREE_MARK); | 1614 | xas_clear_mark(&xas, XA_FREE_MARK); |
| 1643 | } while (__xas_nomem(&xas, gfp)); | 1615 | } while (__xas_nomem(&xas, gfp)); |
| 1644 | 1616 | ||
| 1645 | err = xas_error(&xas); | 1617 | return xas_error(&xas); |
| 1646 | if (!err) | ||
| 1647 | *id = xas.xa_index; | ||
| 1648 | return err; | ||
| 1649 | } | 1618 | } |
| 1650 | EXPORT_SYMBOL(__xa_alloc); | 1619 | EXPORT_SYMBOL(__xa_alloc); |
| 1651 | 1620 | ||
| 1621 | /** | ||
| 1622 | * __xa_alloc_cyclic() - Find somewhere to store this entry in the XArray. | ||
| 1623 | * @xa: XArray. | ||
| 1624 | * @id: Pointer to ID. | ||
| 1625 | * @entry: New entry. | ||
| 1626 | * @limit: Range of allocated ID. | ||
| 1627 | * @next: Pointer to next ID to allocate. | ||
| 1628 | * @gfp: Memory allocation flags. | ||
| 1629 | * | ||
| 1630 | * Finds an empty entry in @xa between @limit.min and @limit.max, | ||
| 1631 | * stores the index into the @id pointer, then stores the entry at | ||
| 1632 | * that index. A concurrent lookup will not see an uninitialised @id. | ||
| 1633 | * The search for an empty entry will start at @next and will wrap | ||
| 1634 | * around if necessary. | ||
| 1635 | * | ||
| 1636 | * Context: Any context. Expects xa_lock to be held on entry. May | ||
| 1637 | * release and reacquire xa_lock if @gfp flags permit. | ||
| 1638 | * Return: 0 if the allocation succeeded without wrapping. 1 if the | ||
| 1639 | * allocation succeeded after wrapping, -ENOMEM if memory could not be | ||
| 1640 | * allocated or -EBUSY if there are no free entries in @limit. | ||
| 1641 | */ | ||
| 1642 | int __xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, | ||
| 1643 | struct xa_limit limit, u32 *next, gfp_t gfp) | ||
| 1644 | { | ||
| 1645 | u32 min = limit.min; | ||
| 1646 | int ret; | ||
| 1647 | |||
| 1648 | limit.min = max(min, *next); | ||
| 1649 | ret = __xa_alloc(xa, id, entry, limit, gfp); | ||
| 1650 | if ((xa->xa_flags & XA_FLAGS_ALLOC_WRAPPED) && ret == 0) { | ||
| 1651 | xa->xa_flags &= ~XA_FLAGS_ALLOC_WRAPPED; | ||
| 1652 | ret = 1; | ||
| 1653 | } | ||
| 1654 | |||
| 1655 | if (ret < 0 && limit.min > min) { | ||
| 1656 | limit.min = min; | ||
| 1657 | ret = __xa_alloc(xa, id, entry, limit, gfp); | ||
| 1658 | if (ret == 0) | ||
| 1659 | ret = 1; | ||
| 1660 | } | ||
| 1661 | |||
| 1662 | if (ret >= 0) { | ||
| 1663 | *next = *id + 1; | ||
| 1664 | if (*next == 0) | ||
| 1665 | xa->xa_flags |= XA_FLAGS_ALLOC_WRAPPED; | ||
| 1666 | } | ||
| 1667 | return ret; | ||
| 1668 | } | ||
| 1669 | EXPORT_SYMBOL(__xa_alloc_cyclic); | ||
| 1670 | |||
| 1652 | /** | 1671 | /** |
| 1653 | * __xa_set_mark() - Set this mark on this entry while locked. | 1672 | * __xa_set_mark() - Set this mark on this entry while locked. |
| 1654 | * @xa: XArray. | 1673 | * @xa: XArray. |
| @@ -1943,6 +1962,8 @@ void xa_destroy(struct xarray *xa) · 288 unmodified lines | |||
| 1943 | entry = xa_head_locked(xa); | 1962 | entry = xa_head_locked(xa); |
| 1944 | RCU_INIT_POINTER(xa->xa_head, NULL); | 1963 | RCU_INIT_POINTER(xa->xa_head, NULL); |
| 1945 | xas_init_marks(&xas); | 1964 | xas_init_marks(&xas); |
| 1965 | if (xa_zero_busy(xa)) | ||
| 1966 | xa_mark_clear(xa, XA_FREE_MARK); | ||
| 1946 | /* lockdep checks we're still holding the lock in xas_free_nodes() */ | 1967 | /* lockdep checks we're still holding the lock in xas_free_nodes() */ |
| 1947 | if (xa_is_node(entry)) | 1968 | if (xa_is_node(entry)) |
| 1948 | xas_free_nodes(&xas, xa_to_node(entry)); | 1969 | xas_free_nodes(&xas, xa_to_node(entry)); |