Pull XArray updates from Matthew Wilcox: "This pull request changes the xa_alloc() API. I'm only aware of one subsystem that has started trying to use it, and we agree on the fixup as part of the merge. The xa_insert() error code also changed to match xa_alloc() (EEXIST to EBUSY), and I added xa_alloc_cyclic(). Beyond that, the usual bugfixes, optimisations and tweaking. I now have a git tree with all users of the radix tree and IDR converted over to the XArray that I'll be feeding to maintainers over the next few weeks" * tag 'xarray-5.1-rc1' of git://git.infradead.org/users/willy/linux-dax: XArray: Fix xa_reserve for 2-byte aligned entries XArray: Fix xa_erase of 2-byte aligned entries XArray: Use xa_cmpxchg to implement xa_reserve XArray: Fix xa_release in allocating arrays XArray: Mark xa_insert and xa_reserve as must_check XArray: Add cyclic allocation XArray: Redesign xa_alloc API XArray: Add support for 1s-based allocation XArray: Change xa_insert to return -EBUSY XArray: Update xa_erase family descriptions XArray tests: RCU lock prohibits GFP_KERNEL
Showing the changes to include/linux (1 of 7 files). Show all
Unified · Split
| @@ -131,6 +131,12 @@ static inline unsigned int xa_pointer_tag(void *entry) · 130 unmodified lines | |||
| 131 | * xa_mk_internal() - Create an internal entry. | 131 | * xa_mk_internal() - Create an internal entry. |
| 132 | * @v: Value to turn into an internal entry. | 132 | * @v: Value to turn into an internal entry. |
| 133 | * | 133 | * |
| 134 | * Internal entries are used for a number of purposes. Entries 0-255 are | ||
| 135 | * used for sibling entries (only 0-62 are used by the current code). 256 | ||
| 136 | * is used for the retry entry. 257 is used for the reserved / zero entry. | ||
| 137 | * Negative internal entries are used to represent errnos. Node pointers | ||
| 138 | * are also tagged as internal entries in some situations. | ||
| 139 | * | ||
| 134 | * Context: Any context. | 140 | * Context: Any context. |
| 135 | * Return: An XArray internal entry corresponding to this value. | 141 | * Return: An XArray internal entry corresponding to this value. |
| 136 | */ | 142 | */ |
| @@ -163,6 +169,22 @@ static inline bool xa_is_internal(const void *entry) · 26 unmodified lines | |||
| 163 | return ((unsigned long)entry & 3) == 2; | 169 | return ((unsigned long)entry & 3) == 2; |
| 164 | } | 170 | } |
| 165 | 171 | ||
| 172 | #define XA_ZERO_ENTRY xa_mk_internal(257) | ||
| 173 | |||
| 174 | /** | ||
| 175 | * xa_is_zero() - Is the entry a zero entry? | ||
| 176 | * @entry: Entry retrieved from the XArray | ||
| 177 | * | ||
| 178 | * The normal API will return NULL as the contents of a slot containing | ||
| 179 | * a zero entry. You can only see zero entries by using the advanced API. | ||
| 180 | * | ||
| 181 | * Return: %true if the entry is a zero entry. | ||
| 182 | */ | ||
| 183 | static inline bool xa_is_zero(const void *entry) | ||
| 184 | { | ||
| 185 | return unlikely(entry == XA_ZERO_ENTRY); | ||
| 186 | } | ||
| 187 | |||
| 166 | /** | 188 | /** |
| 167 | * xa_is_err() - Report whether an XArray operation returned an error | 189 | * xa_is_err() - Report whether an XArray operation returned an error |
| 168 | * @entry: Result from calling an XArray function | 190 | * @entry: Result from calling an XArray function |
| @@ -200,6 +222,27 @@ static inline int xa_err(void *entry) · 31 unmodified lines | |||
| 200 | return 0; | 222 | return 0; |
| 201 | } | 223 | } |
| 202 | 224 | ||
| 225 | /** | ||
| 226 | * struct xa_limit - Represents a range of IDs. | ||
| 227 | * @min: The lowest ID to allocate (inclusive). | ||
| 228 | * @max: The maximum ID to allocate (inclusive). | ||
| 229 | * | ||
| 230 | * This structure is used either directly or via the XA_LIMIT() macro | ||
| 231 | * to communicate the range of IDs that are valid for allocation. | ||
| 232 | * Two common ranges are predefined for you: | ||
| 233 | * * xa_limit_32b - [0 - UINT_MAX] | ||
| 234 | * * xa_limit_31b - [0 - INT_MAX] | ||
| 235 | */ | ||
| 236 | struct xa_limit { | ||
| 237 | u32 max; | ||
| 238 | u32 min; | ||
| 239 | }; | ||
| 240 | |||
| 241 | #define XA_LIMIT(_min, _max) (struct xa_limit) { .min = _min, .max = _max } | ||
| 242 | |||
| 243 | #define xa_limit_32b XA_LIMIT(0, UINT_MAX) | ||
| 244 | #define xa_limit_31b XA_LIMIT(0, INT_MAX) | ||
| 245 | |||
| 203 | typedef unsigned __bitwise xa_mark_t; | 246 | typedef unsigned __bitwise xa_mark_t; |
| 204 | #define XA_MARK_0 ((__force xa_mark_t)0U) | 247 | #define XA_MARK_0 ((__force xa_mark_t)0U) |
| 205 | #define XA_MARK_1 ((__force xa_mark_t)1U) | 248 | #define XA_MARK_1 ((__force xa_mark_t)1U) |
| @@ -220,10 +263,14 @@ enum xa_lock_type { · 14 unmodified lines | |||
| 220 | #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) | 263 | #define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) |
| 221 | #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) | 264 | #define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) |
| 222 | #define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U) | 265 | #define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U) |
| 266 | #define XA_FLAGS_ZERO_BUSY ((__force gfp_t)8U) | ||
| 267 | #define XA_FLAGS_ALLOC_WRAPPED ((__force gfp_t)16U) | ||
| 223 | #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ | 268 | #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ |
| 224 | (__force unsigned)(mark))) | 269 | (__force unsigned)(mark))) |
| 225 | 270 | ||
| 271 | /* ALLOC is for a normal 0-based alloc. ALLOC1 is for an 1-based alloc */ | ||
| 226 | #define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK)) | 272 | #define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE | XA_FLAGS_MARK(XA_FREE_MARK)) |
| 273 | #define XA_FLAGS_ALLOC1 (XA_FLAGS_TRACK_FREE | XA_FLAGS_ZERO_BUSY) | ||
| 227 | 274 | ||
| 228 | /** | 275 | /** |
| 229 | * struct xarray - The anchor of the XArray. | 276 | * struct xarray - The anchor of the XArray. |
| @@ -279,7 +326,7 @@ struct xarray { · 49 unmodified lines | |||
| 279 | #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) | 326 | #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) |
| 280 | 327 | ||
| 281 | /** | 328 | /** |
| 282 | * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs. | 329 | * DEFINE_XARRAY_ALLOC() - Define an XArray which allocates IDs starting at 0. |
| 283 | * @name: A string that names your XArray. | 330 | * @name: A string that names your XArray. |
| 284 | * | 331 | * |
| 285 | * This is intended for file scope definitions of allocating XArrays. | 332 | * This is intended for file scope definitions of allocating XArrays. |
| @@ -287,6 +334,15 @@ struct xarray { · 1 unmodified line | |||
| 287 | */ | 334 | */ |
| 288 | #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC) | 335 | #define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC) |
| 289 | 336 | ||
| 337 | /** | ||
| 338 | * DEFINE_XARRAY_ALLOC1() - Define an XArray which allocates IDs starting at 1. | ||
| 339 | * @name: A string that names your XArray. | ||
| 340 | * | ||
| 341 | * This is intended for file scope definitions of allocating XArrays. | ||
| 342 | * See also DEFINE_XARRAY(). | ||
| 343 | */ | ||
| 344 | #define DEFINE_XARRAY_ALLOC1(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC1) | ||
| 345 | |||
| 290 | void *xa_load(struct xarray *, unsigned long index); | 346 | void *xa_load(struct xarray *, unsigned long index); |
| 291 | void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); | 347 | void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| 292 | void *xa_erase(struct xarray *, unsigned long index); | 348 | void *xa_erase(struct xarray *, unsigned long index); |
| @@ -463,9 +519,12 @@ void *__xa_erase(struct xarray *, unsigned long index); · 170 unmodified lines | |||
| 463 | void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); | 519 | void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); |
| 464 | void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, | 520 | void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, |
| 465 | void *entry, gfp_t); | 521 | void *entry, gfp_t); |
| 466 | int __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t); | 522 | int __must_check __xa_insert(struct xarray *, unsigned long index, |
| 467 | int __xa_alloc(struct xarray *, u32 *id, u32 max, void *entry, gfp_t); | 523 | void *entry, gfp_t); |
| 468 | int __xa_reserve(struct xarray *, unsigned long index, gfp_t); | 524 | int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry, |
| 525 | struct xa_limit, gfp_t); | ||
| 526 | int __must_check __xa_alloc_cyclic(struct xarray *, u32 *id, void *entry, | ||
| 527 | struct xa_limit, u32 *next, gfp_t); | ||
| 469 | void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); | 528 | void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 470 | void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); | 529 | void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); |
| 471 | 530 | ||
| @@ -526,9 +585,9 @@ static inline void *xa_store_irq(struct xarray *xa, unsigned long index, · 54 unmodified lines | |||
| 526 | * @xa: XArray. | 585 | * @xa: XArray. |
| 527 | * @index: Index of entry. | 586 | * @index: Index of entry. |
| 528 | * | 587 | * |
| 529 | * This function is the equivalent of calling xa_store() with %NULL as | 588 | * After this function returns, loading from @index will return %NULL. |
| 530 | * the third argument. The XArray does not need to allocate memory, so | 589 | * If the index is part of a multi-index entry, all indices will be erased |
| 531 | * the user does not need to provide GFP flags. | 590 | * and none of the entries will be part of a multi-index entry. |
| 532 | * | 591 | * |
| 533 | * Context: Any context. Takes and releases the xa_lock while | 592 | * Context: Any context. Takes and releases the xa_lock while |
| 534 | * disabling softirqs. | 593 | * disabling softirqs. |
| @@ -550,9 +609,9 @@ static inline void *xa_erase_bh(struct xarray *xa, unsigned long index) · 15 unmodified lines | |||
| 550 | * @xa: XArray. | 609 | * @xa: XArray. |
| 551 | * @index: Index of entry. | 610 | * @index: Index of entry. |
| 552 | * | 611 | * |
| 553 | * This function is the equivalent of calling xa_store() with %NULL as | 612 | * After this function returns, loading from @index will return %NULL. |
| 554 | * the third argument. The XArray does not need to allocate memory, so | 613 | * If the index is part of a multi-index entry, all indices will be erased |
| 555 | * the user does not need to provide GFP flags. | 614 | * and none of the entries will be part of a multi-index entry. |
| 556 | * | 615 | * |
| 557 | * Context: Process context. Takes and releases the xa_lock while | 616 | * Context: Process context. Takes and releases the xa_lock while |
| 558 | * disabling interrupts. | 617 | * disabling interrupts. |
| @@ -664,11 +723,11 @@ static inline void *xa_cmpxchg_irq(struct xarray *xa, unsigned long index, · 105 unmodified lines | |||
| 664 | * | 723 | * |
| 665 | * Context: Any context. Takes and releases the xa_lock. May sleep if | 724 | * Context: Any context. Takes and releases the xa_lock. May sleep if |
| 666 | * the @gfp flags permit. | 725 | * the @gfp flags permit. |
| 667 | * Return: 0 if the store succeeded. -EEXIST if another entry was present. | 726 | * Return: 0 if the store succeeded. -EBUSY if another entry was present. |
| 668 | * -ENOMEM if memory could not be allocated. | 727 | * -ENOMEM if memory could not be allocated. |
| 669 | */ | 728 | */ |
| 670 | static inline int xa_insert(struct xarray *xa, unsigned long index, | 729 | static inline int __must_check xa_insert(struct xarray *xa, |
| 671 | void *entry, gfp_t gfp) | 730 | unsigned long index, void *entry, gfp_t gfp) |
| 672 | { | 731 | { |
| 673 | int err; | 732 | int err; |
| 674 | 733 | ||
| @@ -693,11 +752,11 @@ static inline int xa_insert(struct xarray *xa, unsigned long index, · 18 unmodified lines | |||
| 693 | * | 752 | * |
| 694 | * Context: Any context. Takes and releases the xa_lock while | 753 | * Context: Any context. Takes and releases the xa_lock while |
| 695 | * disabling softirqs. May sleep if the @gfp flags permit. | 754 | * disabling softirqs. May sleep if the @gfp flags permit. |
| 696 | * Return: 0 if the store succeeded. -EEXIST if another entry was present. | 755 | * Return: 0 if the store succeeded. -EBUSY if another entry was present. |
| 697 | * -ENOMEM if memory could not be allocated. | 756 | * -ENOMEM if memory could not be allocated. |
| 698 | */ | 757 | */ |
| 699 | static inline int xa_insert_bh(struct xarray *xa, unsigned long index, | 758 | static inline int __must_check xa_insert_bh(struct xarray *xa, |
| 700 | void *entry, gfp_t gfp) | 759 | unsigned long index, void *entry, gfp_t gfp) |
| 701 | { | 760 | { |
| 702 | int err; | 761 | int err; |
| 703 | 762 | ||
| @@ -722,11 +781,11 @@ static inline int xa_insert_bh(struct xarray *xa, unsigned long index, · 18 unmodified lines | |||
| 722 | * | 781 | * |
| 723 | * Context: Process context. Takes and releases the xa_lock while | 782 | * Context: Process context. Takes and releases the xa_lock while |
| 724 | * disabling interrupts. May sleep if the @gfp flags permit. | 783 | * disabling interrupts. May sleep if the @gfp flags permit. |
| 725 | * Return: 0 if the store succeeded. -EEXIST if another entry was present. | 784 | * Return: 0 if the store succeeded. -EBUSY if another entry was present. |
| 726 | * -ENOMEM if memory could not be allocated. | 785 | * -ENOMEM if memory could not be allocated. |
| 727 | */ | 786 | */ |
| 728 | static inline int xa_insert_irq(struct xarray *xa, unsigned long index, | 787 | static inline int __must_check xa_insert_irq(struct xarray *xa, |
| 729 | void *entry, gfp_t gfp) | 788 | unsigned long index, void *entry, gfp_t gfp) |
| 730 | { | 789 | { |
| 731 | int err; | 790 | int err; |
| 732 | 791 | ||
| @@ -741,26 +800,26 @@ static inline int xa_insert_irq(struct xarray *xa, unsigned long index, · 8 unmodified lines | |||
| 741 | * xa_alloc() - Find somewhere to store this entry in the XArray. | 800 | * xa_alloc() - Find somewhere to store this entry in the XArray. |
| 742 | * @xa: XArray. | 801 | * @xa: XArray. |
| 743 | * @id: Pointer to ID. | 802 | * @id: Pointer to ID. |
| 744 | * @max: Maximum ID to allocate (inclusive). | ||
| 745 | * @entry: New entry. | 803 | * @entry: New entry. |
| 804 | * @limit: Range of ID to allocate. | ||
| 746 | * @gfp: Memory allocation flags. | 805 | * @gfp: Memory allocation flags. |
| 747 | * | 806 | * |
| 748 | * Allocates an unused ID in the range specified by @id and @max. | 807 | * Finds an empty entry in @xa between @limit.min and @limit.max, |
| 749 | * Updates the @id pointer with the index, then stores the entry at that | 808 | * stores the index into the @id pointer, then stores the entry at |
| 750 | * index. A concurrent lookup will not see an uninitialised @id. | 809 | * that index. A concurrent lookup will not see an uninitialised @id. |
| 751 | * | 810 | * |
| 752 | * Context: Process context. Takes and releases the xa_lock. May sleep if | 811 | * Context: Any context. Takes and releases the xa_lock. May sleep if |
| 753 | * the @gfp flags permit. | 812 | * the @gfp flags permit. |
| 754 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | 813 | * Return: 0 on success, -ENOMEM if memory could not be allocated or |
| 755 | * there is no more space in the XArray. | 814 | * -EBUSY if there are no free entries in @limit. |
| 756 | */ | 815 | */ |
| 757 | static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, | 816 | static inline __must_check int xa_alloc(struct xarray *xa, u32 *id, |
| 758 | gfp_t gfp) | 817 | void *entry, struct xa_limit limit, gfp_t gfp) |
| 759 | { | 818 | { |
| 760 | int err; | 819 | int err; |
| 761 | 820 | ||
| 762 | xa_lock(xa); | 821 | xa_lock(xa); |
| 763 | err = __xa_alloc(xa, id, max, entry, gfp); | 822 | err = __xa_alloc(xa, id, entry, limit, gfp); |
| 764 | xa_unlock(xa); | 823 | xa_unlock(xa); |
| 765 | 824 | ||
| 766 | return err; | 825 | return err; |
| @@ -770,26 +829,26 @@ static inline int xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines | |||
| 770 | * xa_alloc_bh() - Find somewhere to store this entry in the XArray. | 829 | * xa_alloc_bh() - Find somewhere to store this entry in the XArray. |
| 771 | * @xa: XArray. | 830 | * @xa: XArray. |
| 772 | * @id: Pointer to ID. | 831 | * @id: Pointer to ID. |
| 773 | * @max: Maximum ID to allocate (inclusive). | ||
| 774 | * @entry: New entry. | 832 | * @entry: New entry. |
| 833 | * @limit: Range of ID to allocate. | ||
| 775 | * @gfp: Memory allocation flags. | 834 | * @gfp: Memory allocation flags. |
| 776 | * | 835 | * |
| 777 | * Allocates an unused ID in the range specified by @id and @max. | 836 | * Finds an empty entry in @xa between @limit.min and @limit.max, |
| 778 | * Updates the @id pointer with the index, then stores the entry at that | 837 | * stores the index into the @id pointer, then stores the entry at |
| 779 | * index. A concurrent lookup will not see an uninitialised @id. | 838 | * that index. A concurrent lookup will not see an uninitialised @id. |
| 780 | * | 839 | * |
| 781 | * Context: Any context. Takes and releases the xa_lock while | 840 | * Context: Any context. Takes and releases the xa_lock while |
| 782 | * disabling softirqs. May sleep if the @gfp flags permit. | 841 | * disabling softirqs. May sleep if the @gfp flags permit. |
| 783 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | 842 | * Return: 0 on success, -ENOMEM if memory could not be allocated or |
| 784 | * there is no more space in the XArray. | 843 | * -EBUSY if there are no free entries in @limit. |
| 785 | */ | 844 | */ |
| 786 | static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, | 845 | static inline int __must_check xa_alloc_bh(struct xarray *xa, u32 *id, |
| 787 | gfp_t gfp) | 846 | void *entry, struct xa_limit limit, gfp_t gfp) |
| 788 | { | 847 | { |
| 789 | int err; | 848 | int err; |
| 790 | 849 | ||
| 791 | xa_lock_bh(xa); | 850 | xa_lock_bh(xa); |
| 792 | err = __xa_alloc(xa, id, max, entry, gfp); | 851 | err = __xa_alloc(xa, id, entry, limit, gfp); |
| 793 | xa_unlock_bh(xa); | 852 | xa_unlock_bh(xa); |
| 794 | 853 | ||
| 795 | return err; | 854 | return err; |
| @@ -799,26 +858,125 @@ static inline int xa_alloc_bh(struct xarray *xa, u32 *id, u32 max, void *entry, · 3 unmodified lines | |||
| 799 | * xa_alloc_irq() - Find somewhere to store this entry in the XArray. | 858 | * xa_alloc_irq() - Find somewhere to store this entry in the XArray. |
| 800 | * @xa: XArray. | 859 | * @xa: XArray. |
| 801 | * @id: Pointer to ID. | 860 | * @id: Pointer to ID. |
| 802 | * @max: Maximum ID to allocate (inclusive). | ||
| 803 | * @entry: New entry. | 861 | * @entry: New entry. |
| 862 | * @limit: Range of ID to allocate. | ||
| 804 | * @gfp: Memory allocation flags. | 863 | * @gfp: Memory allocation flags. |
| 805 | * | 864 | * |
| 806 | * Allocates an unused ID in the range specified by @id and @max. | 865 | * Finds an empty entry in @xa between @limit.min and @limit.max, |
| 807 | * Updates the @id pointer with the index, then stores the entry at that | 866 | * stores the index into the @id pointer, then stores the entry at |
| 808 | * index. A concurrent lookup will not see an uninitialised @id. | 867 | * that index. A concurrent lookup will not see an uninitialised @id. |
| 809 | * | 868 | * |
| 810 | * Context: Process context. Takes and releases the xa_lock while | 869 | * Context: Process context. Takes and releases the xa_lock while |
| 811 | * disabling interrupts. May sleep if the @gfp flags permit. | 870 | * disabling interrupts. May sleep if the @gfp flags permit. |
| 812 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | 871 | * Return: 0 on success, -ENOMEM if memory could not be allocated or |
| 813 | * there is no more space in the XArray. | 872 | * -EBUSY if there are no free entries in @limit. |
| 814 | */ | 873 | */ |
| 815 | static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, | 874 | static inline int __must_check xa_alloc_irq(struct xarray *xa, u32 *id, |
| 816 | gfp_t gfp) | 875 | void *entry, struct xa_limit limit, gfp_t gfp) |
| 817 | { | 876 | { |
| 818 | int err; | 877 | int err; |
| 819 | 878 | ||
| 820 | xa_lock_irq(xa); | 879 | xa_lock_irq(xa); |
| 821 | err = __xa_alloc(xa, id, max, entry, gfp); | 880 | err = __xa_alloc(xa, id, entry, limit, gfp); |
| 881 | xa_unlock_irq(xa); | ||
| 882 | |||
| 883 | return err; | ||
| 884 | } | ||
| 885 | |||
| 886 | /** | ||
| 887 | * xa_alloc_cyclic() - Find somewhere to store this entry in the XArray. | ||
| 888 | * @xa: XArray. | ||
| 889 | * @id: Pointer to ID. | ||
| 890 | * @entry: New entry. | ||
| 891 | * @limit: Range of allocated ID. | ||
| 892 | * @next: Pointer to next ID to allocate. | ||
| 893 | * @gfp: Memory allocation flags. | ||
| 894 | * | ||
| 895 | * Finds an empty entry in @xa between @limit.min and @limit.max, | ||
| 896 | * stores the index into the @id pointer, then stores the entry at | ||
| 897 | * that index. A concurrent lookup will not see an uninitialised @id. | ||
| 898 | * The search for an empty entry will start at @next and will wrap | ||
| 899 | * around if necessary. | ||
| 900 | * | ||
| 901 | * Context: Any context. Takes and releases the xa_lock. May sleep if | ||
| 902 | * the @gfp flags permit. | ||
| 903 | * Return: 0 if the allocation succeeded without wrapping. 1 if the | ||
| 904 | * allocation succeeded after wrapping, -ENOMEM if memory could not be | ||
| 905 | * allocated or -EBUSY if there are no free entries in @limit. | ||
| 906 | */ | ||
| 907 | static inline int xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, | ||
| 908 | struct xa_limit limit, u32 *next, gfp_t gfp) | ||
| 909 | { | ||
| 910 | int err; | ||
| 911 | |||
| 912 | xa_lock(xa); | ||
| 913 | err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | ||
| 914 | xa_unlock(xa); | ||
| 915 | |||
| 916 | return err; | ||
| 917 | } | ||
| 918 | |||
| 919 | /** | ||
| 920 | * xa_alloc_cyclic_bh() - Find somewhere to store this entry in the XArray. | ||
| 921 | * @xa: XArray. | ||
| 922 | * @id: Pointer to ID. | ||
| 923 | * @entry: New entry. | ||
| 924 | * @limit: Range of allocated ID. | ||
| 925 | * @next: Pointer to next ID to allocate. | ||
| 926 | * @gfp: Memory allocation flags. | ||
| 927 | * | ||
| 928 | * Finds an empty entry in @xa between @limit.min and @limit.max, | ||
| 929 | * stores the index into the @id pointer, then stores the entry at | ||
| 930 | * that index. A concurrent lookup will not see an uninitialised @id. | ||
| 931 | * The search for an empty entry will start at @next and will wrap | ||
| 932 | * around if necessary. | ||
| 933 | * | ||
| 934 | * Context: Any context. Takes and releases the xa_lock while | ||
| 935 | * disabling softirqs. May sleep if the @gfp flags permit. | ||
| 936 | * Return: 0 if the allocation succeeded without wrapping. 1 if the | ||
| 937 | * allocation succeeded after wrapping, -ENOMEM if memory could not be | ||
| 938 | * allocated or -EBUSY if there are no free entries in @limit. | ||
| 939 | */ | ||
| 940 | static inline int xa_alloc_cyclic_bh(struct xarray *xa, u32 *id, void *entry, | ||
| 941 | struct xa_limit limit, u32 *next, gfp_t gfp) | ||
| 942 | { | ||
| 943 | int err; | ||
| 944 | |||
| 945 | xa_lock_bh(xa); | ||
| 946 | err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | ||
| 947 | xa_unlock_bh(xa); | ||
| 948 | |||
| 949 | return err; | ||
| 950 | } | ||
| 951 | |||
| 952 | /** | ||
| 953 | * xa_alloc_cyclic_irq() - Find somewhere to store this entry in the XArray. | ||
| 954 | * @xa: XArray. | ||
| 955 | * @id: Pointer to ID. | ||
| 956 | * @entry: New entry. | ||
| 957 | * @limit: Range of allocated ID. | ||
| 958 | * @next: Pointer to next ID to allocate. | ||
| 959 | * @gfp: Memory allocation flags. | ||
| 960 | * | ||
| 961 | * Finds an empty entry in @xa between @limit.min and @limit.max, | ||
| 962 | * stores the index into the @id pointer, then stores the entry at | ||
| 963 | * that index. A concurrent lookup will not see an uninitialised @id. | ||
| 964 | * The search for an empty entry will start at @next and will wrap | ||
| 965 | * around if necessary. | ||
| 966 | * | ||
| 967 | * Context: Process context. Takes and releases the xa_lock while | ||
| 968 | * disabling interrupts. May sleep if the @gfp flags permit. | ||
| 969 | * Return: 0 if the allocation succeeded without wrapping. 1 if the | ||
| 970 | * allocation succeeded after wrapping, -ENOMEM if memory could not be | ||
| 971 | * allocated or -EBUSY if there are no free entries in @limit. | ||
| 972 | */ | ||
| 973 | static inline int xa_alloc_cyclic_irq(struct xarray *xa, u32 *id, void *entry, | ||
| 974 | struct xa_limit limit, u32 *next, gfp_t gfp) | ||
| 975 | { | ||
| 976 | int err; | ||
| 977 | |||
| 978 | xa_lock_irq(xa); | ||
| 979 | err = __xa_alloc_cyclic(xa, id, entry, limit, next, gfp); | ||
| 822 | xa_unlock_irq(xa); | 980 | xa_unlock_irq(xa); |
| 823 | 981 | ||
| 824 | return err; | 982 | return err; |
| @@ -842,16 +1000,10 @@ static inline int xa_alloc_irq(struct xarray *xa, u32 *id, u32 max, void *entry, · 17 unmodified lines | |||
| 842 | * May sleep if the @gfp flags permit. | 1000 | * May sleep if the @gfp flags permit. |
| 843 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | 1001 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 844 | */ | 1002 | */ |
| 845 | static inline | 1003 | static inline __must_check |
| 846 | int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) | 1004 | int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 847 | { | 1005 | { |
| 848 | int ret; | 1006 | return xa_err(xa_cmpxchg(xa, index, NULL, XA_ZERO_ENTRY, gfp)); |
| 849 | |||
| 850 | xa_lock(xa); | ||
| 851 | ret = __xa_reserve(xa, index, gfp); | ||
| 852 | xa_unlock(xa); | ||
| 853 | |||
| 854 | return ret; | ||
| 855 | } | 1007 | } |
| 856 | 1008 | ||
| 857 | /** | 1009 | /** |
| @@ -866,16 +1018,10 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines | |||
| 866 | * disabling softirqs. | 1018 | * disabling softirqs. |
| 867 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | 1019 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 868 | */ | 1020 | */ |
| 869 | static inline | 1021 | static inline __must_check |
| 870 | int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) | 1022 | int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 871 | { | 1023 | { |
| 872 | int ret; | 1024 | return xa_err(xa_cmpxchg_bh(xa, index, NULL, XA_ZERO_ENTRY, gfp)); |
| 873 | |||
| 874 | xa_lock_bh(xa); | ||
| 875 | ret = __xa_reserve(xa, index, gfp); | ||
| 876 | xa_unlock_bh(xa); | ||
| 877 | |||
| 878 | return ret; | ||
| 879 | } | 1025 | } |
| 880 | 1026 | ||
| 881 | /** | 1027 | /** |
| @@ -890,16 +1036,10 @@ int xa_reserve_bh(struct xarray *xa, unsigned long index, gfp_t gfp) · 8 unmodified lines | |||
| 890 | * disabling interrupts. | 1036 | * disabling interrupts. |
| 891 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. | 1037 | * Return: 0 if the reservation succeeded or -ENOMEM if it failed. |
| 892 | */ | 1038 | */ |
| 893 | static inline | 1039 | static inline __must_check |
| 894 | int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) | 1040 | int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) |
| 895 | { | 1041 | { |
| 896 | int ret; | 1042 | return xa_err(xa_cmpxchg_irq(xa, index, NULL, XA_ZERO_ENTRY, gfp)); |
| 897 | |||
| 898 | xa_lock_irq(xa); | ||
| 899 | ret = __xa_reserve(xa, index, gfp); | ||
| 900 | xa_unlock_irq(xa); | ||
| 901 | |||
| 902 | return ret; | ||
| 903 | } | 1043 | } |
| 904 | 1044 | ||
| 905 | /** | 1045 | /** |
| @@ -913,7 +1053,7 @@ int xa_reserve_irq(struct xarray *xa, unsigned long index, gfp_t gfp) · 7 unmodified lines | |||
| 913 | */ | 1053 | */ |
| 914 | static inline void xa_release(struct xarray *xa, unsigned long index) | 1054 | static inline void xa_release(struct xarray *xa, unsigned long index) |
| 915 | { | 1055 | { |
| 916 | xa_cmpxchg(xa, index, NULL, NULL, 0); | 1056 | xa_cmpxchg(xa, index, XA_ZERO_ENTRY, NULL, 0); |
| 917 | } | 1057 | } |
| 918 | 1058 | ||
| 919 | /* Everything below here is the Advanced API. Proceed with caution. */ | 1059 | /* Everything below here is the Advanced API. Proceed with caution. */ |
| @@ -1073,18 +1213,6 @@ static inline bool xa_is_sibling(const void *entry) · 153 unmodified lines | |||
| 1073 | } | 1213 | } |
| 1074 | 1214 | ||
| 1075 | #define XA_RETRY_ENTRY xa_mk_internal(256) | 1215 | #define XA_RETRY_ENTRY xa_mk_internal(256) |
| 1076 | #define XA_ZERO_ENTRY xa_mk_internal(257) | ||
| 1077 | |||
| 1078 | /** | ||
| 1079 | * xa_is_zero() - Is the entry a zero entry? | ||
| 1080 | * @entry: Entry retrieved from the XArray | ||
| 1081 | * | ||
| 1082 | * Return: %true if the entry is a zero entry. | ||
| 1083 | */ | ||
| 1084 | static inline bool xa_is_zero(const void *entry) | ||
| 1085 | { | ||
| 1086 | return unlikely(entry == XA_ZERO_ENTRY); | ||
| 1087 | } | ||
| 1088 | 1216 | ||
| 1089 | /** | 1217 | /** |
| 1090 | * xa_is_retry() - Is the entry a retry entry? | 1218 | * xa_is_retry() - Is the entry a retry entry? |