Add the optional ability to track which entries in an XArray are free and provide xa_alloc() to replace most of the functionality of the IDR.
Unified · Split
| @@ -103,12 +103,25 @@ Finally, you can remove all entries from an XArray by calling · 102 unmodified lines | |||
| 103 | to free the entries first. You can do this by iterating over all present | 103 | to free the entries first. You can do this by iterating over all present |
| 104 | entries in the XArray using the :c:func:`xa_for_each` iterator. | 104 | entries in the XArray using the :c:func:`xa_for_each` iterator. |
| 105 | 105 | ||
| 106 | ID assignment | ||
| 107 | ------------- | ||
| 108 | |||
| 109 | You can call :c:func:`xa_alloc` to store the entry at any unused index | ||
| 110 | in the XArray. If you need to modify the array from interrupt context, | ||
| 111 | you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable | ||
| 112 | interrupts while allocating the ID. Unlike :c:func:`xa_store`, allocating | ||
| 113 | a ``NULL`` pointer does not delete an entry. Instead it reserves an | ||
| 114 | entry like :c:func:`xa_reserve` and you can release it using either | ||
| 115 | :c:func:`xa_erase` or :c:func:`xa_release`. To use ID assignment, the | ||
| 116 | XArray must be defined with :c:func:`DEFINE_XARRAY_ALLOC`, or initialised | ||
| 117 | by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`, | ||
| 118 | |||
| 106 | Memory allocation | 119 | Memory allocation |
| 107 | ----------------- | 120 | ----------------- |
| 108 | 121 | ||
| 109 | The :c:func:`xa_store`, :c:func:`xa_cmpxchg`, :c:func:`xa_reserve` | 122 | The :c:func:`xa_store`, :c:func:`xa_cmpxchg`, :c:func:`xa_alloc`, |
| 110 | and :c:func:`xa_insert` functions take a gfp_t parameter in case | 123 | :c:func:`xa_reserve` and :c:func:`xa_insert` functions take a gfp_t |
| 111 | the XArray needs to allocate memory to store this entry. | 124 | parameter in case the XArray needs to allocate memory to store this entry. |
| 112 | If the entry is being deleted, no memory allocation needs to be performed, | 125 | If the entry is being deleted, no memory allocation needs to be performed, |
| 113 | and the GFP flags specified will be ignored. | 126 | and the GFP flags specified will be ignored. |
| 114 | 127 | ||
| @@ -143,6 +156,9 @@ Takes xa_lock internally: · 28 unmodified lines | |||
| 143 | * :c:func:`xa_erase_bh` | 156 | * :c:func:`xa_erase_bh` |
| 144 | * :c:func:`xa_erase_irq` | 157 | * :c:func:`xa_erase_irq` |
| 145 | * :c:func:`xa_cmpxchg` | 158 | * :c:func:`xa_cmpxchg` |
| 159 | * :c:func:`xa_alloc` | ||
| 160 | * :c:func:`xa_alloc_bh` | ||
| 161 | * :c:func:`xa_alloc_irq` | ||
| 146 | * :c:func:`xa_destroy` | 162 | * :c:func:`xa_destroy` |
| 147 | * :c:func:`xa_set_mark` | 163 | * :c:func:`xa_set_mark` |
| 148 | * :c:func:`xa_clear_mark` | 164 | * :c:func:`xa_clear_mark` |
| @@ -152,6 +168,7 @@ Assumes xa_lock held on entry: · 3 unmodified lines | |||
| 152 | * :c:func:`__xa_insert` | 168 | * :c:func:`__xa_insert` |
| 153 | * :c:func:`__xa_erase` | 169 | * :c:func:`__xa_erase` |
| 154 | * :c:func:`__xa_cmpxchg` | 170 | * :c:func:`__xa_cmpxchg` |
| 171 | * :c:func:`__xa_alloc` | ||
| 155 | * :c:func:`__xa_set_mark` | 172 | * :c:func:`__xa_set_mark` |
| 156 | * :c:func:`__xa_clear_mark` | 173 | * :c:func:`__xa_clear_mark` |
| 157 | 174 | ||
| @@ -205,6 +205,7 @@ typedef unsigned __bitwise xa_mark_t; · 204 unmodified lines | |||
| 205 | #define XA_MARK_2 ((__force xa_mark_t)2U) | 205 | #define XA_MARK_2 ((__force xa_mark_t)2U) |
| 206 | #define XA_PRESENT ((__force xa_mark_t)8U) | 206 | #define XA_PRESENT ((__force xa_mark_t)8U) |
| 207 | #define XA_MARK_MAX XA_MARK_2 | 207 | #define XA_MARK_MAX XA_MARK_2 |
| 208 | #define XA_FREE_MARK XA_MARK_0 | ||
| 208 | 209 | ||
| 209 | enum xa_lock_type { | 210 | enum xa_lock_type { |
| 210 | XA_LOCK_IRQ = 1, | 211 | XA_LOCK_IRQ = 1, |
| @@ -217,9 +218,12 @@ enum xa_lock_type { · 6 unmodified lines | |||
| 217 | */ | 218 | */ |
| 218 | #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) | 219 | #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) |
| 219 | #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) | 220 | #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) |
| 221 | #define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U) | ||
| 220 | #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ | 222 | #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ |
| 221 | (__force unsigned)(mark))) | 223 | (__force unsigned)(mark))) |
| 222 | 224 | ||
| 225 | #define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK)) | ||
| 226 | |||
| 223 | /** | 227 | /** |
| 224 | * struct xarray - The anchor of the XArray. | 228 | * struct xarray - The anchor of the XArray. |
| 225 | * @xa_lock: Lock that protects the contents of the XArray. | 229 | * @xa_lock: Lock that protects the contents of the XArray. |
| @@ -273,6 +277,15 @@ struct xarray { · 47 unmodified lines | |||
| 273 | */ | 277 | */ |
| 274 | #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) | 278 | #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) |
| 275 | 279 | ||
| 280 | /** | ||
| 281 | * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs. | ||
| 282 | * @name: A string that names your XArray. | ||
| 283 | * | ||
| 284 | * This is intended for file scope definitions of allocating XArrays. | ||
| 285 | * See also DEFINE_XARRAY(). | ||
| 286 | */ | ||
| 287 | #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC) | ||
| 288 | |||
| 276 | void xa_init_flags(struct xarray *, gfp_t flags); | 289 | void xa_init_flags(struct xarray *, gfp_t flags); |
| 277 | void *xa_load(struct xarray *, unsigned long index); | 290 | void *xa_load(struct xarray *, unsigned long index); |
| 278 | void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); | 291 | void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| @@ -439,6 +452,7 @@ void *__xa_erase(struct xarray *, unsigned long index); · 160 unmodified lines | |||
| 439 | void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); | 452 | void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| 440 | void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, | 453 | void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, |
| 441 | void *entry, gfp_t); | 454 | void *entry, gfp_t); |
| 455 | int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t); | ||
| 442 | void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); | 456 | void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 443 | void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); | 457 | void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 444 | 458 | ||
| @@ -518,6 +532,93 @@ static inline void *xa_erase_irq(struct xarray *xa, unsigned long index) · 73 unmodified lines | |||
| 518 | return entry; | 532 | return entry; |
| 519 | } | 533 | } |
| 520 | 534 | ||
| 535 | /** | ||
| 536 | * xa_alloc() - Find somewhere to store this entry in the XArray. | ||
| 537 | * @xa: XArray. | ||
| 538 | * @id: Pointer to ID. | ||
| 539 | * @max: Maximum ID to allocate (inclusive). | ||
| 540 | * @entry: New entry. | ||
| 541 | * @gfp: Memory allocation flags. | ||
| 542 | * | ||
| 543 | * Allocates an unused ID in the range specified by @id and @max. | ||
| 544 | * Updates the @id pointer with the index, then stores the entry at that | ||
| 545 | * index. A concurrent lookup will not see an uninitialised @id. | ||
| 546 | * | ||
| 547 | * Context: Process context. Takes and releases the xa_lock. May sleep if | ||
| 548 | * the @gfp flags permit. | ||
| 549 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | ||
| 550 | * there is no more space in the XArray. | ||
| 551 | */ | ||
| 552 | static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, | ||
| 553 | gfp_t gfp) | ||
| 554 | { | ||
| 555 | int err; | ||
| 556 | |||
| 557 | xa_lock(xa); | ||
| 558 | err = __xa_alloc(xa, id, max, entry, gfp); | ||
| 559 | xa_unlock(xa); | ||
| 560 | |||
| 561 | return err; | ||
| 562 | } | ||
| 563 | |||
| 564 | /** | ||
| 565 | * xa_alloc_bh() - Find somewhere to store this entry in the XArray. | ||
| 566 | * @xa: XArray. | ||
| 567 | * @id: Pointer to ID. | ||
| 568 | * @max: Maximum ID to allocate (inclusive). | ||
| 569 | * @entry: New entry. | ||
| 570 | * @gfp: Memory allocation flags. | ||
| 571 | * | ||
| 572 | * Allocates an unused ID in the range specified by @id and @max. | ||
| 573 | * Updates the @id pointer with the index, then stores the entry at that | ||
| 574 | * index. A concurrent lookup will not see an uninitialised @id. | ||
| 575 | * | ||
| 576 | * Context: Process context. Takes and releases the xa_lock while | ||
| 577 | * disabling softirqs. May sleep if the @gfp flags permit. | ||
| 578 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | ||
| 579 | * there is no more space in the XArray. | ||
| 580 | */ | ||
| 581 | static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, | ||
| 582 | gfp_t gfp) | ||
| 583 | { | ||
| 584 | int err; | ||
| 585 | |||
| 586 | xa_lock_bh(xa); | ||
| 587 | err = __xa_alloc(xa, id, max, entry, gfp); | ||
| 588 | xa_unlock_bh(xa); | ||
| 589 | |||
| 590 | return err; | ||
| 591 | } | ||
| 592 | |||
| 593 | /** | ||
| 594 | * xa_alloc_irq() - Find somewhere to store this entry in the XArray. | ||
| 595 | * @xa: XArray. | ||
| 596 | * @id: Pointer to ID. | ||
| 597 | * @max: Maximum ID to allocate (inclusive). | ||
| 598 | * @entry: New entry. | ||
| 599 | * @gfp: Memory allocation flags. | ||
| 600 | * | ||
| 601 | * Allocates an unused ID in the range specified by @id and @max. | ||
| 602 | * Updates the @id pointer with the index, then stores the entry at that | ||
| 603 | * index. A concurrent lookup will not see an uninitialised @id. | ||
| 604 | * | ||
| 605 | * Context: Process context. Takes and releases the xa_lock while | ||
| 606 | * disabling interrupts. May sleep if the @gfp flags permit. | ||
| 607 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | ||
| 608 | * there is no more space in the XArray. | ||
| 609 | */ | ||
| 610 | static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, | ||
| 611 | gfp_t gfp) | ||
| 612 | { | ||
| 613 | int err; | ||
| 614 | |||
| 615 | xa_lock_irq(xa); | ||
| 616 | err = __xa_alloc(xa, id, max, entry, gfp); | ||
| 617 | xa_unlock_irq(xa); | ||
| 618 | |||
| 619 | return err; | ||
| 620 | } | ||
| 621 | |||
| 521 | /* Everything below here is the Advanced API. Proceed with caution. */ | 622 | /* Everything below here is the Advanced API. Proceed with caution. */ |
| 522 | 623 | ||
| 523 | /* | 624 | /* |
| @@ -33,6 +33,15 @@ static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp) · 32 unmodified lines | |||
| 33 | return xa_store(xa, index, xa_mk_value(index & LONG_MAX), gfp); | 33 | return xa_store(xa, index, xa_mk_value(index & LONG_MAX), gfp); |
| 34 | } | 34 | } |
| 35 | 35 | ||
| 36 | static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp) | ||
| 37 | { | ||
| 38 | u32 id = 0; | ||
| 39 | |||
| 40 | XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, xa_mk_value(index & LONG_MAX), | ||
| 41 | gfp) != 0); | ||
| 42 | XA_BUG_ON(xa, id != index); | ||
| 43 | } | ||
| 44 | |||
| 36 | static void xa_erase_index(struct xarray *xa, unsigned long index) | 45 | static void xa_erase_index(struct xarray *xa, unsigned long index) |
| 37 | { | 46 | { |
| 38 | XA_BUG_ON(xa, xa_erase(xa, index) != xa_mk_value(index & LONG_MAX)); | 47 | XA_BUG_ON(xa, xa_erase(xa, index) != xa_mk_value(index & LONG_MAX)); |
| @@ -404,6 +413,57 @@ static noinline void check_multi_store(struct xarray *xa) · 365 unmodified lines | |||
| 404 | #endif | 413 | #endif |
| 405 | } | 414 | } |
| 406 | 415 | ||
| 416 | static DEFINE_XARRAY_ALLOC(xa0); | ||
| 417 | |||
| 418 | static noinline void check_xa_alloc(void) | ||
| 419 | { | ||
| 420 | int i; | ||
| 421 | u32 id; | ||
| 422 | |||
| 423 | /* An empty array should assign 0 to the first alloc */ | ||
| 424 | xa_alloc_index(&xa0, 0, GFP_KERNEL); | ||
| 425 | |||
| 426 | /* Erasing it should make the array empty again */ | ||
| 427 | xa_erase_index(&xa0, 0); | ||
| 428 | XA_BUG_ON(&xa0, !xa_empty(&xa0)); | ||
| 429 | |||
| 430 | /* And it should assign 0 again */ | ||
| 431 | xa_alloc_index(&xa0, 0, GFP_KERNEL); | ||
| 432 | |||
| 433 | /* The next assigned ID should be 1 */ | ||
| 434 | xa_alloc_index(&xa0, 1, GFP_KERNEL); | ||
| 435 | xa_erase_index(&xa0, 1); | ||
| 436 | |||
| 437 | /* Storing a value should mark it used */ | ||
| 438 | xa_store_index(&xa0, 1, GFP_KERNEL); | ||
| 439 | xa_alloc_index(&xa0, 2, GFP_KERNEL); | ||
| 440 | |||
| 441 | /* If we then erase 0, it should be free */ | ||
| 442 | xa_erase_index(&xa0, 0); | ||
| 443 | xa_alloc_index(&xa0, 0, GFP_KERNEL); | ||
| 444 | |||
| 445 | xa_erase_index(&xa0, 1); | ||
| 446 | xa_erase_index(&xa0, 2); | ||
| 447 | |||
| 448 | for (i = 1; i < 5000; i++) { | ||
| 449 | xa_alloc_index(&xa0, i, GFP_KERNEL); | ||
| 450 | } | ||
| 451 | |||
| 452 | xa_destroy(&xa0); | ||
| 453 | |||
| 454 | id = 0xfffffffeU; | ||
| 455 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0), | ||
| 456 | GFP_KERNEL) != 0); | ||
| 457 | XA_BUG_ON(&xa0, id != 0xfffffffeU); | ||
| 458 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0), | ||
| 459 | GFP_KERNEL) != 0); | ||
| 460 | XA_BUG_ON(&xa0, id != 0xffffffffU); | ||
| 461 | XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0), | ||
| 462 | GFP_KERNEL) != -ENOSPC); | ||
| 463 | XA_BUG_ON(&xa0, id != 0xffffffffU); | ||
| 464 | xa_destroy(&xa0); | ||
| 465 | } | ||
| 466 | |||
| 407 | static noinline void __check_store_iter(struct xarray *xa, unsigned long start, | 467 | static noinline void __check_store_iter(struct xarray *xa, unsigned long start, |
| 408 | unsigned int order, unsigned int present) | 468 | unsigned int order, unsigned int present) |
| 409 | { | 469 | { |
| @@ -849,6 +909,7 @@ static int xarray_checks(void) · 439 unmodified lines | |||
| 849 | check_cmpxchg(&array); | 909 | check_cmpxchg(&array); |
| 850 | check_reserve(&array); | 910 | check_reserve(&array); |
| 851 | check_multi_store(&array); | 911 | check_multi_store(&array); |
| 912 | check_xa_alloc(); | ||
| 852 | check_find(&array); | 913 | check_find(&array); |
| 853 | check_destroy(&array); | 914 | check_destroy(&array); |
| 854 | check_move(&array); | 915 | check_move(&array); |
| @@ -52,6 +52,11 @@ static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type) · 51 unmodified lines | |||
| 52 | xas_unlock(xas); | 52 | xas_unlock(xas); |
| 53 | } | 53 | } |
| 54 | 54 | ||
| 55 | static inline bool xa_track_free(const struct xarray *xa) | ||
| 56 | { | ||
| 57 | return xa->xa_flags & XA_FLAGS_TRACK_FREE; | ||
| 58 | } | ||
| 59 | |||
| 55 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) | 60 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) |
| 56 | { | 61 | { |
| 57 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) | 62 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) |
| @@ -94,6 +99,11 @@ static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark) · 36 unmodified lines | |||
| 94 | return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE); | 99 | return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE); |
| 95 | } | 100 | } |
| 96 | 101 | ||
| 102 | static inline void node_mark_all(struct xa_node *node, xa_mark_t mark) | ||
| 103 | { | ||
| 104 | bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE); | ||
| 105 | } | ||
| 106 | |||
| 97 | #define mark_inc(mark) do { \ | 107 | #define mark_inc(mark) do { \ |
| 98 | mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \ | 108 | mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \ |
| 99 | } while (0) | 109 | } while (0) |
| @@ -416,6 +426,8 @@ static void xas_shrink(struct xa_state *xas) · 316 unmodified lines | |||
| 416 | xas->xa_node = XAS_BOUNDS; | 426 | xas->xa_node = XAS_BOUNDS; |
| 417 | 427 | ||
| 418 | RCU_INIT_POINTER(xa->xa_head, entry); | 428 | RCU_INIT_POINTER(xa->xa_head, entry); |
| 429 | if (xa_track_free(xa) && !node_get_mark(node, 0, XA_FREE_MARK)) | ||
| 430 | xa_mark_clear(xa, XA_FREE_MARK); | ||
| 419 | 431 | ||
| 420 | node->count = 0; | 432 | node->count = 0; |
| 421 | node->nr_values = 0; | 433 | node->nr_values = 0; |
| @@ -549,8 +561,15 @@ static int xas_expand(struct xa_state *xas, void *head) · 127 unmodified lines | |||
| 549 | 561 | ||
| 550 | /* Propagate the aggregated mark info to the new child */ | 562 | /* Propagate the aggregated mark info to the new child */ |
| 551 | for (;;) { | 563 | for (;;) { |
| 552 | if (xa_marked(xa, mark)) | 564 | if (xa_track_free(xa) && mark == XA_FREE_MARK) { |
| 565 | node_mark_all(node, XA_FREE_MARK); | ||
| 566 | if (!xa_marked(xa, XA_FREE_MARK)) { | ||
| 567 | node_clear_mark(node, 0, XA_FREE_MARK); | ||
| 568 | xa_mark_set(xa, XA_FREE_MARK); | ||
| 569 | } | ||
| 570 | } else if (xa_marked(xa, mark)) { | ||
| 553 | node_set_mark(node, 0, mark); | 571 | node_set_mark(node, 0, mark); |
| 572 | } | ||
| 554 | if (mark == XA_MARK_MAX) | 573 | if (mark == XA_MARK_MAX) |
| 555 | break; | 574 | break; |
| 556 | mark_inc(mark); | 575 | mark_inc(mark); |
| @@ -624,6 +643,8 @@ static void *xas_create(struct xa_state *xas) · 67 unmodified lines | |||
| 624 | node = xas_alloc(xas, shift); | 643 | node = xas_alloc(xas, shift); |
| 625 | if (!node) | 644 | if (!node) |
| 626 | break; | 645 | break; |
| 646 | if (xa_track_free(xa)) | ||
| 647 | node_mark_all(node, XA_FREE_MARK); | ||
| 627 | rcu_assign_pointer(*slot, xa_mk_node(node)); | 648 | rcu_assign_pointer(*slot, xa_mk_node(node)); |
| 628 | } else if (xa_is_node(entry)) { | 649 | } else if (xa_is_node(entry)) { |
| 629 | node = xa_to_node(entry); | 650 | node = xa_to_node(entry); |
| @@ -882,7 +903,10 @@ void xas_init_marks(const struct xa_state *xas) · 252 unmodified lines | |||
| 882 | xa_mark_t mark = 0; | 903 | xa_mark_t mark = 0; |
| 883 | 904 | ||
| 884 | for (;;) { | 905 | for (;;) { |
| 885 | xas_clear_mark(xas, mark); | 906 | if (xa_track_free(xas->xa) && mark == XA_FREE_MARK) |
| 907 | xas_set_mark(xas, mark); | ||
| 908 | else | ||
| 909 | xas_clear_mark(xas, mark); | ||
| 886 | if (mark == XA_MARK_MAX) | 910 | if (mark == XA_MARK_MAX) |
| 887 | break; | 911 | break; |
| 888 | mark_inc(mark); | 912 | mark_inc(mark); |
| @@ -1333,6 +1357,8 @@ void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 444 unmodified lines | |||
| 1333 | do { | 1357 | do { |
| 1334 | xas_lock(&xas); | 1358 | xas_lock(&xas); |
| 1335 | curr = xas_store(&xas, entry); | 1359 | curr = xas_store(&xas, entry); |
| 1360 | if (xa_track_free(xa) && entry) | ||
| 1361 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1336 | xas_unlock(&xas); | 1362 | xas_unlock(&xas); |
| 1337 | } while (xas_nomem(&xas, gfp)); | 1363 | } while (xas_nomem(&xas, gfp)); |
| 1338 | 1364 | ||
| @@ -1365,6 +1391,8 @@ void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 26 unmodified lines | |||
| 1365 | 1391 | ||
| 1366 | do { | 1392 | do { |
| 1367 | curr = xas_store(&xas, entry); | 1393 | curr = xas_store(&xas, entry); |
| 1394 | if (xa_track_free(xa) && entry) | ||
| 1395 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1368 | } while (__xas_nomem(&xas, gfp)); | 1396 | } while (__xas_nomem(&xas, gfp)); |
| 1369 | 1397 | ||
| 1370 | return xas_result(&xas, curr); | 1398 | return xas_result(&xas, curr); |
| @@ -1400,8 +1428,11 @@ void *xa_cmpxchg(struct xarray *xa, unsigned long index, · 29 unmodified lines | |||
| 1400 | curr = xas_load(&xas); | 1428 | curr = xas_load(&xas); |
| 1401 | if (curr == XA_ZERO_ENTRY) | 1429 | if (curr == XA_ZERO_ENTRY) |
| 1402 | curr = NULL; | 1430 | curr = NULL; |
| 1403 | if (curr == old) | 1431 | if (curr == old) { |
| 1404 | xas_store(&xas, entry); | 1432 | xas_store(&xas, entry); |
| 1433 | if (xa_track_free(xa) && entry) | ||
| 1434 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1435 | } | ||
| 1405 | xas_unlock(&xas); | 1436 | xas_unlock(&xas); |
| 1406 | } while (xas_nomem(&xas, gfp)); | 1437 | } while (xas_nomem(&xas, gfp)); |
| 1407 | 1438 | ||
| @@ -1438,8 +1469,11 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, · 30 unmodified lines | |||
| 1438 | curr = xas_load(&xas); | 1469 | curr = xas_load(&xas); |
| 1439 | if (curr == XA_ZERO_ENTRY) | 1470 | if (curr == XA_ZERO_ENTRY) |
| 1440 | curr = NULL; | 1471 | curr = NULL; |
| 1441 | if (curr == old) | 1472 | if (curr == old) { |
| 1442 | xas_store(&xas, entry); | 1473 | xas_store(&xas, entry); |
| 1474 | if (xa_track_free(xa) && entry) | ||
| 1475 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1476 | } | ||
| 1443 | } while (__xas_nomem(&xas, gfp)); | 1477 | } while (__xas_nomem(&xas, gfp)); |
| 1444 | 1478 | ||
| 1445 | return xas_result(&xas, curr); | 1479 | return xas_result(&xas, curr); |
| @@ -1483,6 +1517,52 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 37 unmodified lines | |||
| 1483 | } | 1517 | } |
| 1484 | EXPORT_SYMBOL(xa_reserve); | 1518 | EXPORT_SYMBOL(xa_reserve); |
| 1485 | 1519 | ||
| 1520 | /** | ||
| 1521 | * __xa_alloc() - Find somewhere to store this entry in the XArray. | ||
| 1522 | * @xa: XArray. | ||
| 1523 | * @id: Pointer to ID. | ||
| 1524 | * @max: Maximum ID to allocate (inclusive). | ||
| 1525 | * @entry: New entry. | ||
| 1526 | * @gfp: Memory allocation flags. | ||
| 1527 | * | ||
| 1528 | * Allocates an unused ID in the range specified by @id and @max. | ||
| 1529 | * Updates the @id pointer with the index, then stores the entry at that | ||
| 1530 | * index. A concurrent lookup will not see an uninitialised @id. | ||
| 1531 | * | ||
| 1532 | * Context: Any context. Expects xa_lock to be held on entry. May | ||
| 1533 | * release and reacquire xa_lock if @gfp flags permit. | ||
| 1534 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | ||
| 1535 | * there is no more space in the XArray. | ||
| 1536 | */ | ||
| 1537 | int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) | ||
| 1538 | { | ||
| 1539 | XA_STATE(xas, xa, 0); | ||
| 1540 | int err; | ||
| 1541 | |||
| 1542 | if (WARN_ON_ONCE(xa_is_internal(entry))) | ||
| 1543 | return -EINVAL; | ||
| 1544 | if (WARN_ON_ONCE(!xa_track_free(xa))) | ||
| 1545 | return -EINVAL; | ||
| 1546 | |||
| 1547 | if (!entry) | ||
| 1548 | entry = XA_ZERO_ENTRY; | ||
| 1549 | |||
| 1550 | do { | ||
| 1551 | xas.xa_index = *id; | ||
| 1552 | xas_find_marked(&xas, max, XA_FREE_MARK); | ||
| 1553 | if (xas.xa_node == XAS_RESTART) | ||
| 1554 | xas_set_err(&xas, -ENOSPC); | ||
| 1555 | xas_store(&xas, entry); | ||
| 1556 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1557 | } while (__xas_nomem(&xas, gfp)); | ||
| 1558 | |||
| 1559 | err = xas_error(&xas); | ||
| 1560 | if (!err) | ||
| 1561 | *id = xas.xa_index; | ||
| 1562 | return err; | ||
| 1563 | } | ||
| 1564 | EXPORT_SYMBOL(__xa_alloc); | ||
| 1565 | |||
| 1486 | /** | 1566 | /** |
| 1487 | * __xa_set_mark() - Set this mark on this entry while locked. | 1567 | * __xa_set_mark() - Set this mark on this entry while locked. |
| 1488 | * @xa: XArray. | 1568 | * @xa: XArray. |