xarray: Track free entries in an XArray

Add the optional ability to track which entries in an XArray are free and provide xa_alloc() to replace most of the functionality of the IDR.

Matthew Wilcox committed 8 years ago

Commit 371c7521 parent 3d5bd6e4 files · +266 −7Browse files

Unified · Split

4 files

Documentation/core-api/xarray.rst +20 −3
include/linux/xarray.h +101 −0
lib/test_xarray.c +61 −0
lib/xarray.c +84 −4

Documentation/core-api/xarray.rst+20−3

@@ -103,12 +103,25 @@ Finally, you can remove all entries from an XArray by calling · 102 unmodified lines
103	103	to free the entries first. You can do this by iterating over all present
104	104	entries in the XArray using the :c:func:`xa_for_each` iterator.
105	105
	106	+ID assignment
	107	+-------------
	108	+
	109	+You can call :c:func:`xa_alloc` to store the entry at any unused index
	110	+in the XArray. If you need to modify the array from interrupt context,
	111	+you can use :c:func:`xa_alloc_bh` or :c:func:`xa_alloc_irq` to disable
	112	+interrupts while allocating the ID. Unlike :c:func:`xa_store`, allocating
	113	+a ``NULL`` pointer does not delete an entry. Instead it reserves an
	114	+entry like :c:func:`xa_reserve` and you can release it using either
	115	+:c:func:`xa_erase` or :c:func:`xa_release`. To use ID assignment, the
	116	+XArray must be defined with :c:func:`DEFINE_XARRAY_ALLOC`, or initialised
	117	+by passing ``XA_FLAGS_ALLOC`` to :c:func:`xa_init_flags`,
	118	+
106	119	Memory allocation
107	120	-----------------
108	121
109		−The :c:func:`xa_store`, :c:func:`xa_cmpxchg`, :c:func:`xa_reserve`
110		−and :c:func:`xa_insert` functions take a gfp_t parameter in case
111		−the XArray needs to allocate memory to store this entry.
	122	+The :c:func:`xa_store`, :c:func:`xa_cmpxchg`, :c:func:`xa_alloc`,
	123	+:c:func:`xa_reserve` and :c:func:`xa_insert` functions take a gfp_t
	124	+parameter in case the XArray needs to allocate memory to store this entry.
112	125	If the entry is being deleted, no memory allocation needs to be performed,
113	126	and the GFP flags specified will be ignored.
114	127
@@ -143,6 +156,9 @@ Takes xa_lock internally: · 28 unmodified lines
143	156	* :c:func:`xa_erase_bh`
144	157	* :c:func:`xa_erase_irq`
145	158	* :c:func:`xa_cmpxchg`
	159	+ * :c:func:`xa_alloc`
	160	+ * :c:func:`xa_alloc_bh`
	161	+ * :c:func:`xa_alloc_irq`
146	162	* :c:func:`xa_destroy`
147	163	* :c:func:`xa_set_mark`
148	164	* :c:func:`xa_clear_mark`
@@ -152,6 +168,7 @@ Assumes xa_lock held on entry: · 3 unmodified lines
152	168	* :c:func:`__xa_insert`
153	169	* :c:func:`__xa_erase`
154	170	* :c:func:`__xa_cmpxchg`
	171	+ * :c:func:`__xa_alloc`
155	172	* :c:func:`__xa_set_mark`
156	173	* :c:func:`__xa_clear_mark`
157	174

include/linux/xarray.h+101−0

@@ -205,6 +205,7 @@ typedef unsigned __bitwise xa_mark_t; · 204 unmodified lines
205	205	#define XA_MARK_2 ((__force xa_mark_t)2U)
206	206	#define XA_PRESENT ((__force xa_mark_t)8U)
207	207	#define XA_MARK_MAX XA_MARK_2
	208	+#define XA_FREE_MARK XA_MARK_0
208	209
209	210	enum xa_lock_type {
210	211	XA_LOCK_IRQ = 1,
@@ -217,9 +218,12 @@ enum xa_lock_type { · 6 unmodified lines
217	218	*/
218	219	#define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ)
219	220	#define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH)
	221	+#define XA_FLAGS_TRACK_FREE ((__force gfp_t)4U)
220	222	#define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \
221	223	(__force unsigned)(mark)))
222	224
	225	+#define XA_FLAGS_ALLOC (XA_FLAGS_TRACK_FREE \| XA_FLAGS_MARK(XA_FREE_MARK))
	226	+
223	227	/**
224	228	* struct xarray - The anchor of the XArray.
225	229	* @xa_lock: Lock that protects the contents of the XArray.
@@ -273,6 +277,15 @@ struct xarray { · 47 unmodified lines
273	277	*/
274	278	#define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0)
275	279
	280	+/**
	281	+ * DEFINE_XARRAY_ALLOC() - Define an XArray which can allocate IDs.
	282	+ * @name: A string that names your XArray.
	283	+ *
	284	+ * This is intended for file scope definitions of allocating XArrays.
	285	+ * See also DEFINE_XARRAY().
	286	+ */
	287	+#define DEFINE_XARRAY_ALLOC(name) DEFINE_XARRAY_FLAGS(name, XA_FLAGS_ALLOC)
	288	+
276	289	void xa_init_flags(struct xarray *, gfp_t flags);
277	290	void xa_load(struct xarray , unsigned long index);
278	291	void xa_store(struct xarray , unsigned long index, void *entry, gfp_t);
@@ -439,6 +452,7 @@ void __xa_erase(struct xarray , unsigned long index); · 160 unmodified lines
439	452	void __xa_store(struct xarray , unsigned long index, void *entry, gfp_t);
440	453	void __xa_cmpxchg(struct xarray , unsigned long index, void *old,
441	454	void *entry, gfp_t);
	455	+int __xa_alloc(struct xarray , u32 id, u32 max, void *entry, gfp_t);
442	456	void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t);
443	457	void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t);
444	458
@@ -518,6 +532,93 @@ static inline void xa_erase_irq(struct xarray xa, unsigned long index) · 73 unmodified lines
518	532	return entry;
519	533	}
520	534
	535	+/**
	536	+ * xa_alloc() - Find somewhere to store this entry in the XArray.
	537	+ * @xa: XArray.
	538	+ * @id: Pointer to ID.
	539	+ * @max: Maximum ID to allocate (inclusive).
	540	+ * @entry: New entry.
	541	+ * @gfp: Memory allocation flags.
	542	+ *
	543	+ * Allocates an unused ID in the range specified by @id and @max.
	544	+ * Updates the @id pointer with the index, then stores the entry at that
	545	+ * index. A concurrent lookup will not see an uninitialised @id.
	546	+ *
	547	+ * Context: Process context. Takes and releases the xa_lock. May sleep if
	548	+ * the @gfp flags permit.
	549	+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
	550	+ * there is no more space in the XArray.
	551	+ */
	552	+static inline int xa_alloc(struct xarray xa, u32 id, u32 max, void *entry,
	553	+ gfp_t gfp)
	554	+{
	555	+ int err;
	556	+
	557	+ xa_lock(xa);
	558	+ err = __xa_alloc(xa, id, max, entry, gfp);
	559	+ xa_unlock(xa);
	560	+
	561	+ return err;
	562	+}
	563	+
	564	+/**
	565	+ * xa_alloc_bh() - Find somewhere to store this entry in the XArray.
	566	+ * @xa: XArray.
	567	+ * @id: Pointer to ID.
	568	+ * @max: Maximum ID to allocate (inclusive).
	569	+ * @entry: New entry.
	570	+ * @gfp: Memory allocation flags.
	571	+ *
	572	+ * Allocates an unused ID in the range specified by @id and @max.
	573	+ * Updates the @id pointer with the index, then stores the entry at that
	574	+ * index. A concurrent lookup will not see an uninitialised @id.
	575	+ *
	576	+ * Context: Process context. Takes and releases the xa_lock while
	577	+ * disabling softirqs. May sleep if the @gfp flags permit.
	578	+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
	579	+ * there is no more space in the XArray.
	580	+ */
	581	+static inline int xa_alloc_bh(struct xarray xa, u32 id, u32 max, void *entry,
	582	+ gfp_t gfp)
	583	+{
	584	+ int err;
	585	+
	586	+ xa_lock_bh(xa);
	587	+ err = __xa_alloc(xa, id, max, entry, gfp);
	588	+ xa_unlock_bh(xa);
	589	+
	590	+ return err;
	591	+}
	592	+
	593	+/**
	594	+ * xa_alloc_irq() - Find somewhere to store this entry in the XArray.
	595	+ * @xa: XArray.
	596	+ * @id: Pointer to ID.
	597	+ * @max: Maximum ID to allocate (inclusive).
	598	+ * @entry: New entry.
	599	+ * @gfp: Memory allocation flags.
	600	+ *
	601	+ * Allocates an unused ID in the range specified by @id and @max.
	602	+ * Updates the @id pointer with the index, then stores the entry at that
	603	+ * index. A concurrent lookup will not see an uninitialised @id.
	604	+ *
	605	+ * Context: Process context. Takes and releases the xa_lock while
	606	+ * disabling interrupts. May sleep if the @gfp flags permit.
	607	+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
	608	+ * there is no more space in the XArray.
	609	+ */
	610	+static inline int xa_alloc_irq(struct xarray xa, u32 id, u32 max, void *entry,
	611	+ gfp_t gfp)
	612	+{
	613	+ int err;
	614	+
	615	+ xa_lock_irq(xa);
	616	+ err = __xa_alloc(xa, id, max, entry, gfp);
	617	+ xa_unlock_irq(xa);
	618	+
	619	+ return err;
	620	+}
	621	+
521	622	/* Everything below here is the Advanced API. Proceed with caution. */
522	623
523	624	/*

lib/test_xarray.c+61−0

@@ -33,6 +33,15 @@ static void xa_store_index(struct xarray xa, unsigned long index, gfp_t gfp) · 32 unmodified lines
33	33	return xa_store(xa, index, xa_mk_value(index & LONG_MAX), gfp);
34	34	}
35	35
	36	+static void xa_alloc_index(struct xarray *xa, unsigned long index, gfp_t gfp)
	37	+{
	38	+ u32 id = 0;
	39	+
	40	+ XA_BUG_ON(xa, xa_alloc(xa, &id, UINT_MAX, xa_mk_value(index & LONG_MAX),
	41	+ gfp) != 0);
	42	+ XA_BUG_ON(xa, id != index);
	43	+}
	44	+
36	45	static void xa_erase_index(struct xarray *xa, unsigned long index)
37	46	{
38	47	XA_BUG_ON(xa, xa_erase(xa, index) != xa_mk_value(index & LONG_MAX));
@@ -404,6 +413,57 @@ static noinline void check_multi_store(struct xarray *xa) · 365 unmodified lines
404	413	#endif
405	414	}
406	415
	416	+static DEFINE_XARRAY_ALLOC(xa0);
	417	+
	418	+static noinline void check_xa_alloc(void)
	419	+{
	420	+ int i;
	421	+ u32 id;
	422	+
	423	+ /* An empty array should assign 0 to the first alloc */
	424	+ xa_alloc_index(&xa0, 0, GFP_KERNEL);
	425	+
	426	+ /* Erasing it should make the array empty again */
	427	+ xa_erase_index(&xa0, 0);
	428	+ XA_BUG_ON(&xa0, !xa_empty(&xa0));
	429	+
	430	+ /* And it should assign 0 again */
	431	+ xa_alloc_index(&xa0, 0, GFP_KERNEL);
	432	+
	433	+ /* The next assigned ID should be 1 */
	434	+ xa_alloc_index(&xa0, 1, GFP_KERNEL);
	435	+ xa_erase_index(&xa0, 1);
	436	+
	437	+ /* Storing a value should mark it used */
	438	+ xa_store_index(&xa0, 1, GFP_KERNEL);
	439	+ xa_alloc_index(&xa0, 2, GFP_KERNEL);
	440	+
	441	+ /* If we then erase 0, it should be free */
	442	+ xa_erase_index(&xa0, 0);
	443	+ xa_alloc_index(&xa0, 0, GFP_KERNEL);
	444	+
	445	+ xa_erase_index(&xa0, 1);
	446	+ xa_erase_index(&xa0, 2);
	447	+
	448	+ for (i = 1; i < 5000; i++) {
	449	+ xa_alloc_index(&xa0, i, GFP_KERNEL);
	450	+ }
	451	+
	452	+ xa_destroy(&xa0);
	453	+
	454	+ id = 0xfffffffeU;
	455	+ XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0),
	456	+ GFP_KERNEL) != 0);
	457	+ XA_BUG_ON(&xa0, id != 0xfffffffeU);
	458	+ XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0),
	459	+ GFP_KERNEL) != 0);
	460	+ XA_BUG_ON(&xa0, id != 0xffffffffU);
	461	+ XA_BUG_ON(&xa0, xa_alloc(&xa0, &id, UINT_MAX, xa_mk_value(0),
	462	+ GFP_KERNEL) != -ENOSPC);
	463	+ XA_BUG_ON(&xa0, id != 0xffffffffU);
	464	+ xa_destroy(&xa0);
	465	+}
	466	+
407	467	static noinline void __check_store_iter(struct xarray *xa, unsigned long start,
408	468	unsigned int order, unsigned int present)
409	469	{
@@ -849,6 +909,7 @@ static int xarray_checks(void) · 439 unmodified lines
849	909	check_cmpxchg(&array);
850	910	check_reserve(&array);
851	911	check_multi_store(&array);
	912	+ check_xa_alloc();
852	913	check_find(&array);
853	914	check_destroy(&array);
854	915	check_move(&array);

lib/xarray.c+84−4

@@ -52,6 +52,11 @@ static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type) · 51 unmodified lines
52	52	xas_unlock(xas);
53	53	}
54	54
	55	+static inline bool xa_track_free(const struct xarray *xa)
	56	+{
	57	+ return xa->xa_flags & XA_FLAGS_TRACK_FREE;
	58	+}
	59	+
55	60	static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)
56	61	{
57	62	if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))
@@ -94,6 +99,11 @@ static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark) · 36 unmodified lines
94	99	return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE);
95	100	}
96	101
	102	+static inline void node_mark_all(struct xa_node *node, xa_mark_t mark)
	103	+{
	104	+ bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE);
	105	+}
	106	+
97	107	#define mark_inc(mark) do { \
98	108	mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \
99	109	} while (0)
@@ -416,6 +426,8 @@ static void xas_shrink(struct xa_state *xas) · 316 unmodified lines
416	426	xas->xa_node = XAS_BOUNDS;
417	427
418	428	RCU_INIT_POINTER(xa->xa_head, entry);
	429	+ if (xa_track_free(xa) && !node_get_mark(node, 0, XA_FREE_MARK))
	430	+ xa_mark_clear(xa, XA_FREE_MARK);
419	431
420	432	node->count = 0;
421	433	node->nr_values = 0;
@@ -549,8 +561,15 @@ static int xas_expand(struct xa_state xas, void head) · 127 unmodified lines
549	561
550	562	/* Propagate the aggregated mark info to the new child */
551	563	for (;;) {
552		− if (xa_marked(xa, mark))
	564	+ if (xa_track_free(xa) && mark == XA_FREE_MARK) {
	565	+ node_mark_all(node, XA_FREE_MARK);
	566	+ if (!xa_marked(xa, XA_FREE_MARK)) {
	567	+ node_clear_mark(node, 0, XA_FREE_MARK);
	568	+ xa_mark_set(xa, XA_FREE_MARK);
	569	+ }
	570	+ } else if (xa_marked(xa, mark)) {
553	571	node_set_mark(node, 0, mark);
	572	+ }
554	573	if (mark == XA_MARK_MAX)
555	574	break;
556	575	mark_inc(mark);
@@ -624,6 +643,8 @@ static void xas_create(struct xa_state xas) · 67 unmodified lines
624	643	node = xas_alloc(xas, shift);
625	644	if (!node)
626	645	break;
	646	+ if (xa_track_free(xa))
	647	+ node_mark_all(node, XA_FREE_MARK);
627	648	rcu_assign_pointer(*slot, xa_mk_node(node));
628	649	} else if (xa_is_node(entry)) {
629	650	node = xa_to_node(entry);
@@ -882,7 +903,10 @@ void xas_init_marks(const struct xa_state *xas) · 252 unmodified lines
882	903	xa_mark_t mark = 0;
883	904
884	905	for (;;) {
885		− xas_clear_mark(xas, mark);
	906	+ if (xa_track_free(xas->xa) && mark == XA_FREE_MARK)
	907	+ xas_set_mark(xas, mark);
	908	+ else
	909	+ xas_clear_mark(xas, mark);
886	910	if (mark == XA_MARK_MAX)
887	911	break;
888	912	mark_inc(mark);
@@ -1333,6 +1357,8 @@ void xa_store(struct xarray xa, unsigned long index, void *entry, gfp_t gfp) · 444 unmodified lines
1333	1357	do {
1334	1358	xas_lock(&xas);
1335	1359	curr = xas_store(&xas, entry);
	1360	+ if (xa_track_free(xa) && entry)
	1361	+ xas_clear_mark(&xas, XA_FREE_MARK);
1336	1362	xas_unlock(&xas);
1337	1363	} while (xas_nomem(&xas, gfp));
1338	1364
@@ -1365,6 +1391,8 @@ void __xa_store(struct xarray xa, unsigned long index, void *entry, gfp_t gfp) · 26 unmodified lines
1365	1391
1366	1392	do {
1367	1393	curr = xas_store(&xas, entry);
	1394	+ if (xa_track_free(xa) && entry)
	1395	+ xas_clear_mark(&xas, XA_FREE_MARK);
1368	1396	} while (__xas_nomem(&xas, gfp));
1369	1397
1370	1398	return xas_result(&xas, curr);
@@ -1400,8 +1428,11 @@ void xa_cmpxchg(struct xarray xa, unsigned long index, · 29 unmodified lines
1400	1428	curr = xas_load(&xas);
1401	1429	if (curr == XA_ZERO_ENTRY)
1402	1430	curr = NULL;
1403		− if (curr == old)
	1431	+ if (curr == old) {
1404	1432	xas_store(&xas, entry);
	1433	+ if (xa_track_free(xa) && entry)
	1434	+ xas_clear_mark(&xas, XA_FREE_MARK);
	1435	+ }
1405	1436	xas_unlock(&xas);
1406	1437	} while (xas_nomem(&xas, gfp));
1407	1438
@@ -1438,8 +1469,11 @@ void __xa_cmpxchg(struct xarray xa, unsigned long index, · 30 unmodified lines
1438	1469	curr = xas_load(&xas);
1439	1470	if (curr == XA_ZERO_ENTRY)
1440	1471	curr = NULL;
1441		− if (curr == old)
	1472	+ if (curr == old) {
1442	1473	xas_store(&xas, entry);
	1474	+ if (xa_track_free(xa) && entry)
	1475	+ xas_clear_mark(&xas, XA_FREE_MARK);
	1476	+ }
1443	1477	} while (__xas_nomem(&xas, gfp));
1444	1478
1445	1479	return xas_result(&xas, curr);
@@ -1483,6 +1517,52 @@ int xa_reserve(struct xarray *xa, unsigned long index, gfp_t gfp) · 37 unmodified lines
1483	1517	}
1484	1518	EXPORT_SYMBOL(xa_reserve);
1485	1519
	1520	+/**
	1521	+ * __xa_alloc() - Find somewhere to store this entry in the XArray.
	1522	+ * @xa: XArray.
	1523	+ * @id: Pointer to ID.
	1524	+ * @max: Maximum ID to allocate (inclusive).
	1525	+ * @entry: New entry.
	1526	+ * @gfp: Memory allocation flags.
	1527	+ *
	1528	+ * Allocates an unused ID in the range specified by @id and @max.
	1529	+ * Updates the @id pointer with the index, then stores the entry at that
	1530	+ * index. A concurrent lookup will not see an uninitialised @id.
	1531	+ *
	1532	+ * Context: Any context. Expects xa_lock to be held on entry. May
	1533	+ * release and reacquire xa_lock if @gfp flags permit.
	1534	+ * Return: 0 on success, -ENOMEM if memory allocation fails or -ENOSPC if
	1535	+ * there is no more space in the XArray.
	1536	+ */
	1537	+int __xa_alloc(struct xarray xa, u32 id, u32 max, void *entry, gfp_t gfp)
	1538	+{
	1539	+ XA_STATE(xas, xa, 0);
	1540	+ int err;
	1541	+
	1542	+ if (WARN_ON_ONCE(xa_is_internal(entry)))
	1543	+ return -EINVAL;
	1544	+ if (WARN_ON_ONCE(!xa_track_free(xa)))
	1545	+ return -EINVAL;
	1546	+
	1547	+ if (!entry)
	1548	+ entry = XA_ZERO_ENTRY;
	1549	+
	1550	+ do {
	1551	+ xas.xa_index = *id;
	1552	+ xas_find_marked(&xas, max, XA_FREE_MARK);
	1553	+ if (xas.xa_node == XAS_RESTART)
	1554	+ xas_set_err(&xas, -ENOSPC);
	1555	+ xas_store(&xas, entry);
	1556	+ xas_clear_mark(&xas, XA_FREE_MARK);
	1557	+ } while (__xas_nomem(&xas, gfp));
	1558	+
	1559	+ err = xas_error(&xas);
	1560	+ if (!err)
	1561	+ *id = xas.xa_index;
	1562	+ return err;
	1563	+}
	1564	+EXPORT_SYMBOL(__xa_alloc);
	1565	+
1486	1566	/**
1487	1567	* __xa_set_mark() - Set this mark on this entry while locked.
1488	1568	* @xa: XArray.

xarray: Track free entries in an XArray

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code