Add the optional ability to track which entries in an XArray are free and provide xa_alloc() to replace most of the functionality of the IDR.
Showing the changes to lib/xarray.c (1 of 4 files). Show all
Unified · Split
| @@ -52,6 +52,11 @@ static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type) · 51 unmodified lines | |||
| 52 | xas_unlock(xas); | 52 | xas_unlock(xas); |
| 53 | } | 53 | } |
| 54 | 54 | ||
| 55 | static inline bool xa_track_free(const struct xarray *xa) | ||
| 56 | { | ||
| 57 | return xa->xa_flags & XA_FLAGS_TRACK_FREE; | ||
| 58 | } | ||
| 59 | |||
| 55 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) | 60 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) |
| 56 | { | 61 | { |
| 57 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) | 62 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) |
| @@ -94,6 +99,11 @@ static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark) · 36 unmodified lines | |||
| 94 | return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE); | 99 | return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE); |
| 95 | } | 100 | } |
| 96 | 101 | ||
| 102 | static inline void node_mark_all(struct xa_node *node, xa_mark_t mark) | ||
| 103 | { | ||
| 104 | bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE); | ||
| 105 | } | ||
| 106 | |||
| 97 | #define mark_inc(mark) do { \ | 107 | #define mark_inc(mark) do { \ |
| 98 | mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \ | 108 | mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \ |
| 99 | } while (0) | 109 | } while (0) |
| @@ -416,6 +426,8 @@ static void xas_shrink(struct xa_state *xas) · 316 unmodified lines | |||
| 416 | xas->xa_node = XAS_BOUNDS; | 426 | xas->xa_node = XAS_BOUNDS; |
| 417 | 427 | ||
| 418 | RCU_INIT_POINTER(xa->xa_head, entry); | 428 | RCU_INIT_POINTER(xa->xa_head, entry); |
| 429 | if (xa_track_free(xa) && !node_get_mark(node, 0, XA_FREE_MARK)) | ||
| 430 | xa_mark_clear(xa, XA_FREE_MARK); | ||
| 419 | 431 | ||
| 420 | node->count = 0; | 432 | node->count = 0; |
| 421 | node->nr_values = 0; | 433 | node->nr_values = 0; |
| @@ -549,8 +561,15 @@ static int xas_expand(struct xa_state *xas, void *head) · 127 unmodified lines | |||
| 549 | 561 | ||
| 550 | /* Propagate the aggregated mark info to the new child */ | 562 | /* Propagate the aggregated mark info to the new child */ |
| 551 | for (;;) { | 563 | for (;;) { |
| 552 | if (xa_marked(xa, mark)) | 564 | if (xa_track_free(xa) && mark == XA_FREE_MARK) { |
| 565 | node_mark_all(node, XA_FREE_MARK); | ||
| 566 | if (!xa_marked(xa, XA_FREE_MARK)) { | ||
| 567 | node_clear_mark(node, 0, XA_FREE_MARK); | ||
| 568 | xa_mark_set(xa, XA_FREE_MARK); | ||
| 569 | } | ||
| 570 | } else if (xa_marked(xa, mark)) { | ||
| 553 | node_set_mark(node, 0, mark); | 571 | node_set_mark(node, 0, mark); |
| 572 | } | ||
| 554 | if (mark == XA_MARK_MAX) | 573 | if (mark == XA_MARK_MAX) |
| 555 | break; | 574 | break; |
| 556 | mark_inc(mark); | 575 | mark_inc(mark); |
| @@ -624,6 +643,8 @@ static void *xas_create(struct xa_state *xas) · 67 unmodified lines | |||
| 624 | node = xas_alloc(xas, shift); | 643 | node = xas_alloc(xas, shift); |
| 625 | if (!node) | 644 | if (!node) |
| 626 | break; | 645 | break; |
| 646 | if (xa_track_free(xa)) | ||
| 647 | node_mark_all(node, XA_FREE_MARK); | ||
| 627 | rcu_assign_pointer(*slot, xa_mk_node(node)); | 648 | rcu_assign_pointer(*slot, xa_mk_node(node)); |
| 628 | } else if (xa_is_node(entry)) { | 649 | } else if (xa_is_node(entry)) { |
| 629 | node = xa_to_node(entry); | 650 | node = xa_to_node(entry); |
| @@ -882,7 +903,10 @@ void xas_init_marks(const struct xa_state *xas) · 252 unmodified lines | |||
| 882 | xa_mark_t mark = 0; | 903 | xa_mark_t mark = 0; |
| 883 | 904 | ||
| 884 | for (;;) { | 905 | for (;;) { |
| 885 | xas_clear_mark(xas, mark); | 906 | if (xa_track_free(xas->xa) && mark == XA_FREE_MARK) |
| 907 | xas_set_mark(xas, mark); | ||
| 908 | else | ||
| 909 | xas_clear_mark(xas, mark); | ||
| 886 | if (mark == XA_MARK_MAX) | 910 | if (mark == XA_MARK_MAX) |
| 887 | break; | 911 | break; |
| 888 | mark_inc(mark); | 912 | mark_inc(mark); |
| @@ -1333,6 +1357,8 @@ void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 444 unmodified lines | |||
| 1333 | do { | 1357 | do { |
| 1334 | xas_lock(&xas); | 1358 | xas_lock(&xas); |
| 1335 | curr = xas_store(&xas, entry); | 1359 | curr = xas_store(&xas, entry); |
| 1360 | if (xa_track_free(xa) && entry) | ||
| 1361 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1336 | xas_unlock(&xas); | 1362 | xas_unlock(&xas); |
| 1337 | } while (xas_nomem(&xas, gfp)); | 1363 | } while (xas_nomem(&xas, gfp)); |
| 1338 | 1364 | ||
| @@ -1365,6 +1391,8 @@ void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 26 unmodified lines | |||
| 1365 | 1391 | ||
| 1366 | do { | 1392 | do { |
| 1367 | curr = xas_store(&xas, entry); | 1393 | curr = xas_store(&xas, entry); |
| 1394 | if (xa_track_free(xa) && entry) | ||
| 1395 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1368 | } while (__xas_nomem(&xas, gfp)); | 1396 | } while (__xas_nomem(&xas, gfp)); |
| 1369 | 1397 | ||
| 1370 | return xas_result(&xas, curr); | 1398 | return xas_result(&xas, curr); |
| @@ -1400,8 +1428,11 @@ void *xa_cmpxchg(struct xarray *xa, unsigned long index, · 29 unmodified lines | |||
| 1400 | curr = xas_load(&xas); | 1428 | curr = xas_load(&xas); |
| 1401 | if (curr == XA_ZERO_ENTRY) | 1429 | if (curr == XA_ZERO_ENTRY) |
| 1402 | curr = NULL; | 1430 | curr = NULL; |
| 1403 | if (curr == old) | 1431 | if (curr == old) { |
| 1404 | xas_store(&xas, entry); | 1432 | xas_store(&xas, entry); |
| 1433 | if (xa_track_free(xa) && entry) | ||
| 1434 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1435 | } | ||
| 1405 | xas_unlock(&xas); | 1436 | xas_unlock(&xas); |
| 1406 | } while (xas_nomem(&xas, gfp)); | 1437 | } while (xas_nomem(&xas, gfp)); |
| 1407 | 1438 | ||
| @@ -1438,8 +1469,11 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, · 30 unmodified lines | |||
| 1438 | curr = xas_load(&xas); | 1469 | curr = xas_load(&xas); |
| 1439 | if (curr == XA_ZERO_ENTRY) | 1470 | if (curr == XA_ZERO_ENTRY) |
| 1440 | curr = NULL; | 1471 | curr = NULL; |
| 1441 | if (curr == old) | 1472 | if (curr == old) { |
| 1442 | xas_store(&xas, entry); | 1473 | xas_store(&xas, entry); |
| 1474 | if (xa_track_free(xa) && entry) | ||
| 1475 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1476 | } | ||
| 1443 | } while (__xas_nomem(&xas, gfp)); | 1477 | } while (__xas_nomem(&xas, gfp)); |
| 1444 | 1478 | ||
| 1445 | return xas_result(&xas, curr); | 1479 | return xas_result(&xas, curr); |
| @@ -1483,6 +1517,52 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 37 unmodified lines | |||
| 1483 | } | 1517 | } |
| 1484 | EXPORT_SYMBOL(xa_reserve); | 1518 | EXPORT_SYMBOL(xa_reserve); |
| 1485 | 1519 | ||
| 1520 | /** | ||
| 1521 | * __xa_alloc() - Find somewhere to store this entry in the XArray. | ||
| 1522 | * @xa: XArray. | ||
| 1523 | * @id: Pointer to ID. | ||
| 1524 | * @max: Maximum ID to allocate (inclusive). | ||
| 1525 | * @entry: New entry. | ||
| 1526 | * @gfp: Memory allocation flags. | ||
| 1527 | * | ||
| 1528 | * Allocates an unused ID in the range specified by @id and @max. | ||
| 1529 | * Updates the @id pointer with the index, then stores the entry at that | ||
| 1530 | * index. A concurrent lookup will not see an uninitialised @id. | ||
| 1531 | * | ||
| 1532 | * Context: Any context. Expects xa_lock to be held on entry. May | ||
| 1533 | * release and reacquire xa_lock if @gfp flags permit. | ||
| 1534 | * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | ||
| 1535 | * there is no more space in the XArray. | ||
| 1536 | */ | ||
| 1537 | int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) | ||
| 1538 | { | ||
| 1539 | XA_STATE(xas, xa, 0); | ||
| 1540 | int err; | ||
| 1541 | |||
| 1542 | if (WARN_ON_ONCE(xa_is_internal(entry))) | ||
| 1543 | return -EINVAL; | ||
| 1544 | if (WARN_ON_ONCE(!xa_track_free(xa))) | ||
| 1545 | return -EINVAL; | ||
| 1546 | |||
| 1547 | if (!entry) | ||
| 1548 | entry = XA_ZERO_ENTRY; | ||
| 1549 | |||
| 1550 | do { | ||
| 1551 | xas.xa_index = *id; | ||
| 1552 | xas_find_marked(&xas, max, XA_FREE_MARK); | ||
| 1553 | if (xas.xa_node == XAS_RESTART) | ||
| 1554 | xas_set_err(&xas, -ENOSPC); | ||
| 1555 | xas_store(&xas, entry); | ||
| 1556 | xas_clear_mark(&xas, XA_FREE_MARK); | ||
| 1557 | } while (__xas_nomem(&xas, gfp)); | ||
| 1558 | |||
| 1559 | err = xas_error(&xas); | ||
| 1560 | if (!err) | ||
| 1561 | *id = xas.xa_index; | ||
| 1562 | return err; | ||
| 1563 | } | ||
| 1564 | EXPORT_SYMBOL(__xa_alloc); | ||
| 1565 | |||
| 1486 | /** | 1566 | /** |
| 1487 | * __xa_set_mark() - Set this mark on this entry while locked. | 1567 | * __xa_set_mark() - Set this mark on this entry while locked. |
| 1488 | * @xa: XArray. | 1568 | * @xa: XArray. |