Add the optional ability to track which entries in an XArray are free and provide xa_alloc() to replace most of the functionality of the IDR.
Showing the changes to lib/xarray.c (1 of 4 files). Show all
Unified · Split
| @@ -52,6 +52,11 @@ static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type) · 51 unmodified lines | ||
| 52 | 52 | xas_unlock(xas); |
| 53 | 53 | } |
| 54 | 54 | |
| 55 | +static inline bool xa_track_free(const struct xarray *xa) | |
| 56 | +{ | |
| 57 | + return xa->xa_flags & XA_FLAGS_TRACK_FREE; | |
| 58 | +} | |
| 59 | + | |
| 55 | 60 | static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) |
| 56 | 61 | { |
| 57 | 62 | if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) |
| @@ -94,6 +99,11 @@ static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark) · 36 unmodified lines | ||
| 94 | 99 | return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE); |
| 95 | 100 | } |
| 96 | 101 | |
| 102 | +static inline void node_mark_all(struct xa_node *node, xa_mark_t mark) | |
| 103 | +{ | |
| 104 | + bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE); | |
| 105 | +} | |
| 106 | + | |
| 97 | 107 | #define mark_inc(mark) do { \ |
| 98 | 108 | mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \ |
| 99 | 109 | } while (0) |
| @@ -416,6 +426,8 @@ static void xas_shrink(struct xa_state *xas) · 316 unmodified lines | ||
| 416 | 426 | xas->xa_node = XAS_BOUNDS; |
| 417 | 427 | |
| 418 | 428 | RCU_INIT_POINTER(xa->xa_head, entry); |
| 429 | + if (xa_track_free(xa) && !node_get_mark(node, 0, XA_FREE_MARK)) | |
| 430 | + xa_mark_clear(xa, XA_FREE_MARK); | |
| 419 | 431 | |
| 420 | 432 | node->count = 0; |
| 421 | 433 | node->nr_values = 0; |
| @@ -549,8 +561,15 @@ static int xas_expand(struct xa_state *xas, void *head) · 127 unmodified lines | ||
| 549 | 561 | |
| 550 | 562 | /* Propagate the aggregated mark info to the new child */ |
| 551 | 563 | for (;;) { |
| 552 | − if (xa_marked(xa, mark)) | |
| 564 | + if (xa_track_free(xa) && mark == XA_FREE_MARK) { | |
| 565 | + node_mark_all(node, XA_FREE_MARK); | |
| 566 | + if (!xa_marked(xa, XA_FREE_MARK)) { | |
| 567 | + node_clear_mark(node, 0, XA_FREE_MARK); | |
| 568 | + xa_mark_set(xa, XA_FREE_MARK); | |
| 569 | + } | |
| 570 | + } else if (xa_marked(xa, mark)) { | |
| 553 | 571 | node_set_mark(node, 0, mark); |
| 572 | + } | |
| 554 | 573 | if (mark == XA_MARK_MAX) |
| 555 | 574 | break; |
| 556 | 575 | mark_inc(mark); |
| @@ -624,6 +643,8 @@ static void *xas_create(struct xa_state *xas) · 67 unmodified lines | ||
| 624 | 643 | node = xas_alloc(xas, shift); |
| 625 | 644 | if (!node) |
| 626 | 645 | break; |
| 646 | + if (xa_track_free(xa)) | |
| 647 | + node_mark_all(node, XA_FREE_MARK); | |
| 627 | 648 | rcu_assign_pointer(*slot, xa_mk_node(node)); |
| 628 | 649 | } else if (xa_is_node(entry)) { |
| 629 | 650 | node = xa_to_node(entry); |
| @@ -882,7 +903,10 @@ void xas_init_marks(const struct xa_state *xas) · 252 unmodified lines | ||
| 882 | 903 | xa_mark_t mark = 0; |
| 883 | 904 | |
| 884 | 905 | for (;;) { |
| 885 | − xas_clear_mark(xas, mark); | |
| 906 | + if (xa_track_free(xas->xa) && mark == XA_FREE_MARK) | |
| 907 | + xas_set_mark(xas, mark); | |
| 908 | + else | |
| 909 | + xas_clear_mark(xas, mark); | |
| 886 | 910 | if (mark == XA_MARK_MAX) |
| 887 | 911 | break; |
| 888 | 912 | mark_inc(mark); |
| @@ -1333,6 +1357,8 @@ void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 444 unmodified lines | ||
| 1333 | 1357 | do { |
| 1334 | 1358 | xas_lock(&xas); |
| 1335 | 1359 | curr = xas_store(&xas, entry); |
| 1360 | + if (xa_track_free(xa) && entry) | |
| 1361 | + xas_clear_mark(&xas, XA_FREE_MARK); | |
| 1336 | 1362 | xas_unlock(&xas); |
| 1337 | 1363 | } while (xas_nomem(&xas, gfp)); |
| 1338 | 1364 | |
| @@ -1365,6 +1391,8 @@ void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) · 26 unmodified lines | ||
| 1365 | 1391 | |
| 1366 | 1392 | do { |
| 1367 | 1393 | curr = xas_store(&xas, entry); |
| 1394 | + if (xa_track_free(xa) && entry) | |
| 1395 | + xas_clear_mark(&xas, XA_FREE_MARK); | |
| 1368 | 1396 | } while (__xas_nomem(&xas, gfp)); |
| 1369 | 1397 | |
| 1370 | 1398 | return xas_result(&xas, curr); |
| @@ -1400,8 +1428,11 @@ void *xa_cmpxchg(struct xarray *xa, unsigned long index, · 29 unmodified lines | ||
| 1400 | 1428 | curr = xas_load(&xas); |
| 1401 | 1429 | if (curr == XA_ZERO_ENTRY) |
| 1402 | 1430 | curr = NULL; |
| 1403 | − if (curr == old) | |
| 1431 | + if (curr == old) { | |
| 1404 | 1432 | xas_store(&xas, entry); |
| 1433 | + if (xa_track_free(xa) && entry) | |
| 1434 | + xas_clear_mark(&xas, XA_FREE_MARK); | |
| 1435 | + } | |
| 1405 | 1436 | xas_unlock(&xas); |
| 1406 | 1437 | } while (xas_nomem(&xas, gfp)); |
| 1407 | 1438 | |
| @@ -1438,8 +1469,11 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, · 30 unmodified lines | ||
| 1438 | 1469 | curr = xas_load(&xas); |
| 1439 | 1470 | if (curr == XA_ZERO_ENTRY) |
| 1440 | 1471 | curr = NULL; |
| 1441 | − if (curr == old) | |
| 1472 | + if (curr == old) { | |
| 1442 | 1473 | xas_store(&xas, entry); |
| 1474 | + if (xa_track_free(xa) && entry) | |
| 1475 | + xas_clear_mark(&xas, XA_FREE_MARK); | |
| 1476 | + } | |
| 1443 | 1477 | } while (__xas_nomem(&xas, gfp)); |
| 1444 | 1478 | |
| 1445 | 1479 | return xas_result(&xas, curr); |
| @@ -1483,6 +1517,52 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 37 unmodified lines | ||
| 1483 | 1517 | } |
| 1484 | 1518 | EXPORT_SYMBOL(xa_reserve); |
| 1485 | 1519 | |
| 1520 | +/** | |
| 1521 | + * __xa_alloc() - Find somewhere to store this entry in the XArray. | |
| 1522 | + * @xa: XArray. | |
| 1523 | + * @id: Pointer to ID. | |
| 1524 | + * @max: Maximum ID to allocate (inclusive). | |
| 1525 | + * @entry: New entry. | |
| 1526 | + * @gfp: Memory allocation flags. | |
| 1527 | + * | |
| 1528 | + * Allocates an unused ID in the range specified by @id and @max. | |
| 1529 | + * Updates the @id pointer with the index, then stores the entry at that | |
| 1530 | + * index. A concurrent lookup will not see an uninitialised @id. | |
| 1531 | + * | |
| 1532 | + * Context: Any context. Expects xa_lock to be held on entry. May | |
| 1533 | + * release and reacquire xa_lock if @gfp flags permit. | |
| 1534 | + * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if | |
| 1535 | + * there is no more space in the XArray. | |
| 1536 | + */ | |
| 1537 | +int __xa_alloc(struct xarray *xa, u32 *id, u32 max, void *entry, gfp_t gfp) | |
| 1538 | +{ | |
| 1539 | + XA_STATE(xas, xa, 0); | |
| 1540 | + int err; | |
| 1541 | + | |
| 1542 | + if (WARN_ON_ONCE(xa_is_internal(entry))) | |
| 1543 | + return -EINVAL; | |
| 1544 | + if (WARN_ON_ONCE(!xa_track_free(xa))) | |
| 1545 | + return -EINVAL; | |
| 1546 | + | |
| 1547 | + if (!entry) | |
| 1548 | + entry = XA_ZERO_ENTRY; | |
| 1549 | + | |
| 1550 | + do { | |
| 1551 | + xas.xa_index = *id; | |
| 1552 | + xas_find_marked(&xas, max, XA_FREE_MARK); | |
| 1553 | + if (xas.xa_node == XAS_RESTART) | |
| 1554 | + xas_set_err(&xas, -ENOSPC); | |
| 1555 | + xas_store(&xas, entry); | |
| 1556 | + xas_clear_mark(&xas, XA_FREE_MARK); | |
| 1557 | + } while (__xas_nomem(&xas, gfp)); | |
| 1558 | + | |
| 1559 | + err = xas_error(&xas); | |
| 1560 | + if (!err) | |
| 1561 | + *id = xas.xa_index; | |
| 1562 | + return err; | |
| 1563 | +} | |
| 1564 | +EXPORT_SYMBOL(__xa_alloc); | |
| 1565 | + | |
| 1486 | 1566 | /** |
| 1487 | 1567 | * __xa_set_mark() - Set this mark on this entry while locked. |
| 1488 | 1568 | * @xa: XArray. |