Merge branch 'locking/rcuref' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pulling rcurefs from Peter for tglx's work.

Jakub Kicinski committed 3 years ago

Commit b133fff2 parents 163c2c7 ee1ee6d9 files · +752 −19Browse files

Unified · Split

9 files

include/linux/atomic/atomic-arch-fallback.h +199 −9
include/linux/atomic/atomic-instrumented.h +67 −1
include/linux/atomic/atomic-long.h +37 −1
include/linux/rcuref.h +155 −0
include/linux/types.h +6 −0
lib/Makefile +1 −1
lib/rcuref.c +281 −0
scripts/atomic/atomics.tbl +1 −1
scripts/atomic/fallbacks/add_negative +5 −6

include/linux/atomic/atomic-arch-fallback.h+199−9

@@ -1208,15 +1208,21 @@ arch_atomic_inc_and_test(atomic_t *v) · 1207 unmodified lines
1208	#define arch_atomic_inc_and_test arch_atomic_inc_and_test	1208	#define arch_atomic_inc_and_test arch_atomic_inc_and_test
1209	#endif	1209	#endif
1210		1210
		1211	#ifndef arch_atomic_add_negative_relaxed
		1212	#ifdef arch_atomic_add_negative
		1213	#define arch_atomic_add_negative_acquire arch_atomic_add_negative
		1214	#define arch_atomic_add_negative_release arch_atomic_add_negative
		1215	#define arch_atomic_add_negative_relaxed arch_atomic_add_negative
		1216	#endif /* arch_atomic_add_negative */
		1217
1211	#ifndef arch_atomic_add_negative	1218	#ifndef arch_atomic_add_negative
1212	/**	1219	/**
1213	* arch_atomic_add_negative - add and test if negative	1220	* arch_atomic_add_negative - Add and test if negative
1214	* @i: integer value to add	1221	* @i: integer value to add
1215	* @v: pointer of type atomic_t	1222	* @v: pointer of type atomic_t
1216	*	1223	*
1217	* Atomically adds @i to @v and returns true	1224	* Atomically adds @i to @v and returns true if the result is negative,
1218	* if the result is negative, or false when	1225	* or false when the result is greater than or equal to zero.
1219	* result is greater than or equal to zero.
1220	*/	1226	*/
1221	static __always_inline bool	1227	static __always_inline bool
1222	arch_atomic_add_negative(int i, atomic_t *v)	1228	arch_atomic_add_negative(int i, atomic_t *v)
@@ -1226,6 +1232,95 @@ arch_atomic_add_negative(int i, atomic_t *v) · 3 unmodified lines
1226	#define arch_atomic_add_negative arch_atomic_add_negative	1232	#define arch_atomic_add_negative arch_atomic_add_negative
1227	#endif	1233	#endif
1228		1234
		1235	#ifndef arch_atomic_add_negative_acquire
		1236	/**
		1237	* arch_atomic_add_negative_acquire - Add and test if negative
		1238	* @i: integer value to add
		1239	* @v: pointer of type atomic_t
		1240	*
		1241	* Atomically adds @i to @v and returns true if the result is negative,
		1242	* or false when the result is greater than or equal to zero.
		1243	*/
		1244	static __always_inline bool
		1245	arch_atomic_add_negative_acquire(int i, atomic_t *v)
		1246	{
		1247	return arch_atomic_add_return_acquire(i, v) < 0;
		1248	}
		1249	#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
		1250	#endif
		1251
		1252	#ifndef arch_atomic_add_negative_release
		1253	/**
		1254	* arch_atomic_add_negative_release - Add and test if negative
		1255	* @i: integer value to add
		1256	* @v: pointer of type atomic_t
		1257	*
		1258	* Atomically adds @i to @v and returns true if the result is negative,
		1259	* or false when the result is greater than or equal to zero.
		1260	*/
		1261	static __always_inline bool
		1262	arch_atomic_add_negative_release(int i, atomic_t *v)
		1263	{
		1264	return arch_atomic_add_return_release(i, v) < 0;
		1265	}
		1266	#define arch_atomic_add_negative_release arch_atomic_add_negative_release
		1267	#endif
		1268
		1269	#ifndef arch_atomic_add_negative_relaxed
		1270	/**
		1271	* arch_atomic_add_negative_relaxed - Add and test if negative
		1272	* @i: integer value to add
		1273	* @v: pointer of type atomic_t
		1274	*
		1275	* Atomically adds @i to @v and returns true if the result is negative,
		1276	* or false when the result is greater than or equal to zero.
		1277	*/
		1278	static __always_inline bool
		1279	arch_atomic_add_negative_relaxed(int i, atomic_t *v)
		1280	{
		1281	return arch_atomic_add_return_relaxed(i, v) < 0;
		1282	}
		1283	#define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed
		1284	#endif
		1285
		1286	#else /* arch_atomic_add_negative_relaxed */
		1287
		1288	#ifndef arch_atomic_add_negative_acquire
		1289	static __always_inline bool
		1290	arch_atomic_add_negative_acquire(int i, atomic_t *v)
		1291	{
		1292	bool ret = arch_atomic_add_negative_relaxed(i, v);
		1293	__atomic_acquire_fence();
		1294	return ret;
		1295	}
		1296	#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
		1297	#endif
		1298
		1299	#ifndef arch_atomic_add_negative_release
		1300	static __always_inline bool
		1301	arch_atomic_add_negative_release(int i, atomic_t *v)
		1302	{
		1303	__atomic_release_fence();
		1304	return arch_atomic_add_negative_relaxed(i, v);
		1305	}
		1306	#define arch_atomic_add_negative_release arch_atomic_add_negative_release
		1307	#endif
		1308
		1309	#ifndef arch_atomic_add_negative
		1310	static __always_inline bool
		1311	arch_atomic_add_negative(int i, atomic_t *v)
		1312	{
		1313	bool ret;
		1314	__atomic_pre_full_fence();
		1315	ret = arch_atomic_add_negative_relaxed(i, v);
		1316	__atomic_post_full_fence();
		1317	return ret;
		1318	}
		1319	#define arch_atomic_add_negative arch_atomic_add_negative
		1320	#endif
		1321
		1322	#endif /* arch_atomic_add_negative_relaxed */
		1323
1229	#ifndef arch_atomic_fetch_add_unless	1324	#ifndef arch_atomic_fetch_add_unless
1230	/**	1325	/**
1231	* arch_atomic_fetch_add_unless - add unless the number is already a given value	1326	* arch_atomic_fetch_add_unless - add unless the number is already a given value
@@ -2329,15 +2424,21 @@ arch_atomic64_inc_and_test(atomic64_t *v) · 1097 unmodified lines
2329	#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test	2424	#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2330	#endif	2425	#endif
2331		2426
		2427	#ifndef arch_atomic64_add_negative_relaxed
		2428	#ifdef arch_atomic64_add_negative
		2429	#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative
		2430	#define arch_atomic64_add_negative_release arch_atomic64_add_negative
		2431	#define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative
		2432	#endif /* arch_atomic64_add_negative */
		2433
2332	#ifndef arch_atomic64_add_negative	2434	#ifndef arch_atomic64_add_negative
2333	/**	2435	/**
2334	* arch_atomic64_add_negative - add and test if negative	2436	* arch_atomic64_add_negative - Add and test if negative
2335	* @i: integer value to add	2437	* @i: integer value to add
2336	* @v: pointer of type atomic64_t	2438	* @v: pointer of type atomic64_t
2337	*	2439	*
2338	* Atomically adds @i to @v and returns true	2440	* Atomically adds @i to @v and returns true if the result is negative,
2339	* if the result is negative, or false when	2441	* or false when the result is greater than or equal to zero.
2340	* result is greater than or equal to zero.
2341	*/	2442	*/
2342	static __always_inline bool	2443	static __always_inline bool
2343	arch_atomic64_add_negative(s64 i, atomic64_t *v)	2444	arch_atomic64_add_negative(s64 i, atomic64_t *v)
@@ -2347,6 +2448,95 @@ arch_atomic64_add_negative(s64 i, atomic64_t *v) · 3 unmodified lines
2347	#define arch_atomic64_add_negative arch_atomic64_add_negative	2448	#define arch_atomic64_add_negative arch_atomic64_add_negative
2348	#endif	2449	#endif
2349		2450
		2451	#ifndef arch_atomic64_add_negative_acquire
		2452	/**
		2453	* arch_atomic64_add_negative_acquire - Add and test if negative
		2454	* @i: integer value to add
		2455	* @v: pointer of type atomic64_t
		2456	*
		2457	* Atomically adds @i to @v and returns true if the result is negative,
		2458	* or false when the result is greater than or equal to zero.
		2459	*/
		2460	static __always_inline bool
		2461	arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
		2462	{
		2463	return arch_atomic64_add_return_acquire(i, v) < 0;
		2464	}
		2465	#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire
		2466	#endif
		2467
		2468	#ifndef arch_atomic64_add_negative_release
		2469	/**
		2470	* arch_atomic64_add_negative_release - Add and test if negative
		2471	* @i: integer value to add
		2472	* @v: pointer of type atomic64_t
		2473	*
		2474	* Atomically adds @i to @v and returns true if the result is negative,
		2475	* or false when the result is greater than or equal to zero.
		2476	*/
		2477	static __always_inline bool
		2478	arch_atomic64_add_negative_release(s64 i, atomic64_t *v)
		2479	{
		2480	return arch_atomic64_add_return_release(i, v) < 0;
		2481	}
		2482	#define arch_atomic64_add_negative_release arch_atomic64_add_negative_release
		2483	#endif
		2484
		2485	#ifndef arch_atomic64_add_negative_relaxed
		2486	/**
		2487	* arch_atomic64_add_negative_relaxed - Add and test if negative
		2488	* @i: integer value to add
		2489	* @v: pointer of type atomic64_t
		2490	*
		2491	* Atomically adds @i to @v and returns true if the result is negative,
		2492	* or false when the result is greater than or equal to zero.
		2493	*/
		2494	static __always_inline bool
		2495	arch_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
		2496	{
		2497	return arch_atomic64_add_return_relaxed(i, v) < 0;
		2498	}
		2499	#define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed
		2500	#endif
		2501
		2502	#else /* arch_atomic64_add_negative_relaxed */
		2503
		2504	#ifndef arch_atomic64_add_negative_acquire
		2505	static __always_inline bool
		2506	arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
		2507	{
		2508	bool ret = arch_atomic64_add_negative_relaxed(i, v);
		2509	__atomic_acquire_fence();
		2510	return ret;
		2511	}
		2512	#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire
		2513	#endif
		2514
		2515	#ifndef arch_atomic64_add_negative_release
		2516	static __always_inline bool
		2517	arch_atomic64_add_negative_release(s64 i, atomic64_t *v)
		2518	{
		2519	__atomic_release_fence();
		2520	return arch_atomic64_add_negative_relaxed(i, v);
		2521	}
		2522	#define arch_atomic64_add_negative_release arch_atomic64_add_negative_release
		2523	#endif
		2524
		2525	#ifndef arch_atomic64_add_negative
		2526	static __always_inline bool
		2527	arch_atomic64_add_negative(s64 i, atomic64_t *v)
		2528	{
		2529	bool ret;
		2530	__atomic_pre_full_fence();
		2531	ret = arch_atomic64_add_negative_relaxed(i, v);
		2532	__atomic_post_full_fence();
		2533	return ret;
		2534	}
		2535	#define arch_atomic64_add_negative arch_atomic64_add_negative
		2536	#endif
		2537
		2538	#endif /* arch_atomic64_add_negative_relaxed */
		2539
2350	#ifndef arch_atomic64_fetch_add_unless	2540	#ifndef arch_atomic64_fetch_add_unless
2351	/**	2541	/**
2352	* arch_atomic64_fetch_add_unless - add unless the number is already a given value	2542	* arch_atomic64_fetch_add_unless - add unless the number is already a given value
@@ -2456,4 +2646,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) · 103 unmodified lines
2456	#endif	2646	#endif
2457		2647
2458	#endif /* _LINUX_ATOMIC_FALLBACK_H */	2648	#endif /* _LINUX_ATOMIC_FALLBACK_H */
2459	// b5e87bdd5ede61470c29f7a7e4de781af3770f09	2649	// 00071fffa021cec66f6290d706d69c91df87bade

include/linux/atomic/atomic-instrumented.h+67−1

@@ -592,6 +592,28 @@ atomic_add_negative(int i, atomic_t *v) · 591 unmodified lines
592	return arch_atomic_add_negative(i, v);	592	return arch_atomic_add_negative(i, v);
593	}	593	}
594		594
		595	static __always_inline bool
		596	atomic_add_negative_acquire(int i, atomic_t *v)
		597	{
		598	instrument_atomic_read_write(v, sizeof(*v));
		599	return arch_atomic_add_negative_acquire(i, v);
		600	}
		601
		602	static __always_inline bool
		603	atomic_add_negative_release(int i, atomic_t *v)
		604	{
		605	kcsan_release();
		606	instrument_atomic_read_write(v, sizeof(*v));
		607	return arch_atomic_add_negative_release(i, v);
		608	}
		609
		610	static __always_inline bool
		611	atomic_add_negative_relaxed(int i, atomic_t *v)
		612	{
		613	instrument_atomic_read_write(v, sizeof(*v));
		614	return arch_atomic_add_negative_relaxed(i, v);
		615	}
		616
595	static __always_inline int	617	static __always_inline int
596	atomic_fetch_add_unless(atomic_t *v, int a, int u)	618	atomic_fetch_add_unless(atomic_t *v, int a, int u)
597	{	619	{
@@ -1211,6 +1233,28 @@ atomic64_add_negative(s64 i, atomic64_t *v) · 613 unmodified lines
1211	return arch_atomic64_add_negative(i, v);	1233	return arch_atomic64_add_negative(i, v);
1212	}	1234	}
1213		1235
		1236	static __always_inline bool
		1237	atomic64_add_negative_acquire(s64 i, atomic64_t *v)
		1238	{
		1239	instrument_atomic_read_write(v, sizeof(*v));
		1240	return arch_atomic64_add_negative_acquire(i, v);
		1241	}
		1242
		1243	static __always_inline bool
		1244	atomic64_add_negative_release(s64 i, atomic64_t *v)
		1245	{
		1246	kcsan_release();
		1247	instrument_atomic_read_write(v, sizeof(*v));
		1248	return arch_atomic64_add_negative_release(i, v);
		1249	}
		1250
		1251	static __always_inline bool
		1252	atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
		1253	{
		1254	instrument_atomic_read_write(v, sizeof(*v));
		1255	return arch_atomic64_add_negative_relaxed(i, v);
		1256	}
		1257
1214	static __always_inline s64	1258	static __always_inline s64
1215	atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)	1259	atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216	{	1260	{
@@ -1830,6 +1874,28 @@ atomic_long_add_negative(long i, atomic_long_t *v) · 613 unmodified lines
1830	return arch_atomic_long_add_negative(i, v);	1874	return arch_atomic_long_add_negative(i, v);
1831	}	1875	}
1832		1876
		1877	static __always_inline bool
		1878	atomic_long_add_negative_acquire(long i, atomic_long_t *v)
		1879	{
		1880	instrument_atomic_read_write(v, sizeof(*v));
		1881	return arch_atomic_long_add_negative_acquire(i, v);
		1882	}
		1883
		1884	static __always_inline bool
		1885	atomic_long_add_negative_release(long i, atomic_long_t *v)
		1886	{
		1887	kcsan_release();
		1888	instrument_atomic_read_write(v, sizeof(*v));
		1889	return arch_atomic_long_add_negative_release(i, v);
		1890	}
		1891
		1892	static __always_inline bool
		1893	atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
		1894	{
		1895	instrument_atomic_read_write(v, sizeof(*v));
		1896	return arch_atomic_long_add_negative_relaxed(i, v);
		1897	}
		1898
1833	static __always_inline long	1899	static __always_inline long
1834	atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)	1900	atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835	{	1901	{
@@ -2083,4 +2149,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) · 247 unmodified lines
2083	})	2149	})
2084		2150
2085	#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */	2151	#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086	// 764f741eb77a7ad565dc8d99ce2837d5542e8aee	2152	// 1b485de9cbaa4900de59e14ee2084357eaeb1c3a

include/linux/atomic/atomic-long.h+37−1

@@ -479,6 +479,24 @@ arch_atomic_long_add_negative(long i, atomic_long_t *v) · 478 unmodified lines
479	return arch_atomic64_add_negative(i, v);	479	return arch_atomic64_add_negative(i, v);
480	}	480	}
481		481
		482	static __always_inline bool
		483	arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
		484	{
		485	return arch_atomic64_add_negative_acquire(i, v);
		486	}
		487
		488	static __always_inline bool
		489	arch_atomic_long_add_negative_release(long i, atomic_long_t *v)
		490	{
		491	return arch_atomic64_add_negative_release(i, v);
		492	}
		493
		494	static __always_inline bool
		495	arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
		496	{
		497	return arch_atomic64_add_negative_relaxed(i, v);
		498	}
		499
482	static __always_inline long	500	static __always_inline long
483	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)	501	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
484	{	502	{
@@ -973,6 +991,24 @@ arch_atomic_long_add_negative(long i, atomic_long_t *v) · 488 unmodified lines
973	return arch_atomic_add_negative(i, v);	991	return arch_atomic_add_negative(i, v);
974	}	992	}
975		993
		994	static __always_inline bool
		995	arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
		996	{
		997	return arch_atomic_add_negative_acquire(i, v);
		998	}
		999
		1000	static __always_inline bool
		1001	arch_atomic_long_add_negative_release(long i, atomic_long_t *v)
		1002	{
		1003	return arch_atomic_add_negative_release(i, v);
		1004	}
		1005
		1006	static __always_inline bool
		1007	arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
		1008	{
		1009	return arch_atomic_add_negative_relaxed(i, v);
		1010	}
		1011
976	static __always_inline long	1012	static __always_inline long
977	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)	1013	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
978	{	1014	{
@@ -1011,4 +1047,4 @@ arch_atomic_long_dec_if_positive(atomic_long_t *v) · 32 unmodified lines
1011		1047
1012	#endif /* CONFIG_64BIT */	1048	#endif /* CONFIG_64BIT */
1013	#endif /* _LINUX_ATOMIC_LONG_H */	1049	#endif /* _LINUX_ATOMIC_LONG_H */
1014	// e8f0e08ff072b74d180eabe2ad001282b38c2c88	1050	// a194c07d7d2f4b0e178d3c118c919775d5d65f50

include/linux/rcuref.h+155−0

@@ -0,0 +1,155 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	#ifndef _LINUX_RCUREF_H
	3	#define _LINUX_RCUREF_H
	4
	5	#include <linux/atomic.h>
	6	#include <linux/bug.h>
	7	#include <linux/limits.h>
	8	#include <linux/lockdep.h>
	9	#include <linux/preempt.h>
	10	#include <linux/rcupdate.h>
	11
	12	#define RCUREF_ONEREF 0x00000000U
	13	#define RCUREF_MAXREF 0x7FFFFFFFU
	14	#define RCUREF_SATURATED 0xA0000000U
	15	#define RCUREF_RELEASED 0xC0000000U
	16	#define RCUREF_DEAD 0xE0000000U
	17	#define RCUREF_NOREF 0xFFFFFFFFU
	18
	19	/**
	20	* rcuref_init - Initialize a rcuref reference count with the given reference count
	21	* @ref: Pointer to the reference count
	22	* @cnt: The initial reference count typically '1'
	23	*/
	24	static inline void rcuref_init(rcuref_t *ref, unsigned int cnt)
	25	{
	26	atomic_set(&ref->refcnt, cnt - 1);
	27	}
	28
	29	/**
	30	* rcuref_read - Read the number of held reference counts of a rcuref
	31	* @ref: Pointer to the reference count
	32	*
	33	* Return: The number of held references (0 ... N)
	34	*/
	35	static inline unsigned int rcuref_read(rcuref_t *ref)
	36	{
	37	unsigned int c = atomic_read(&ref->refcnt);
	38
	39	/* Return 0 if within the DEAD zone. */
	40	return c >= RCUREF_RELEASED ? 0 : c + 1;
	41	}
	42
	43	extern __must_check bool rcuref_get_slowpath(rcuref_t *ref);
	44
	45	/**
	46	* rcuref_get - Acquire one reference on a rcuref reference count
	47	* @ref: Pointer to the reference count
	48	*
	49	* Similar to atomic_inc_not_zero() but saturates at RCUREF_MAXREF.
	50	*
	51	* Provides no memory ordering, it is assumed the caller has guaranteed the
	52	* object memory to be stable (RCU, etc.). It does provide a control dependency
	53	* and thereby orders future stores. See documentation in lib/rcuref.c
	54	*
	55	* Return:
	56	* False if the attempt to acquire a reference failed. This happens
	57	* when the last reference has been put already
	58	*
	59	* True if a reference was successfully acquired
	60	*/
	61	static inline __must_check bool rcuref_get(rcuref_t *ref)
	62	{
	63	/*
	64	* Unconditionally increase the reference count. The saturation and
	65	* dead zones provide enough tolerance for this.
	66	*/
	67	if (likely(!atomic_add_negative_relaxed(1, &ref->refcnt)))
	68	return true;
	69
	70	/* Handle the cases inside the saturation and dead zones */
	71	return rcuref_get_slowpath(ref);
	72	}
	73
	74	extern __must_check bool rcuref_put_slowpath(rcuref_t *ref);
	75
	76	/*
	77	* Internal helper. Do not invoke directly.
	78	*/
	79	static __always_inline __must_check bool __rcuref_put(rcuref_t *ref)
	80	{
	81	RCU_LOCKDEP_WARN(!rcu_read_lock_held() && preemptible(),
	82	"suspicious rcuref_put_rcusafe() usage");
	83	/*
	84	* Unconditionally decrease the reference count. The saturation and
	85	* dead zones provide enough tolerance for this.
	86	*/
	87	if (likely(!atomic_add_negative_release(-1, &ref->refcnt)))
	88	return false;
	89
	90	/*
	91	* Handle the last reference drop and cases inside the saturation
	92	* and dead zones.
	93	*/
	94	return rcuref_put_slowpath(ref);
	95	}
	96
	97	/**
	98	* rcuref_put_rcusafe -- Release one reference for a rcuref reference count RCU safe
	99	* @ref: Pointer to the reference count
	100	*
	101	* Provides release memory ordering, such that prior loads and stores are done
	102	* before, and provides an acquire ordering on success such that free()
	103	* must come after.
	104	*
	105	* Can be invoked from contexts, which guarantee that no grace period can
	106	* happen which would free the object concurrently if the decrement drops
	107	* the last reference and the slowpath races against a concurrent get() and
	108	* put() pair. rcu_read_lock()'ed and atomic contexts qualify.
	109	*
	110	* Return:
	111	* True if this was the last reference with no future references
	112	* possible. This signals the caller that it can safely release the
	113	* object which is protected by the reference counter.
	114	*
	115	* False if there are still active references or the put() raced
	116	* with a concurrent get()/put() pair. Caller is not allowed to
	117	* release the protected object.
	118	*/
	119	static inline __must_check bool rcuref_put_rcusafe(rcuref_t *ref)
	120	{
	121	return __rcuref_put(ref);
	122	}
	123
	124	/**
	125	* rcuref_put -- Release one reference for a rcuref reference count
	126	* @ref: Pointer to the reference count
	127	*
	128	* Can be invoked from any context.
	129	*
	130	* Provides release memory ordering, such that prior loads and stores are done
	131	* before, and provides an acquire ordering on success such that free()
	132	* must come after.
	133	*
	134	* Return:
	135	*
	136	* True if this was the last reference with no future references
	137	* possible. This signals the caller that it can safely schedule the
	138	* object, which is protected by the reference counter, for
	139	* deconstruction.
	140	*
	141	* False if there are still active references or the put() raced
	142	* with a concurrent get()/put() pair. Caller is not allowed to
	143	* deconstruct the protected object.
	144	*/
	145	static inline __must_check bool rcuref_put(rcuref_t *ref)
	146	{
	147	bool released;
	148
	149	preempt_disable();
	150	released = __rcuref_put(ref);
	151	preempt_enable();
	152	return released;
	153	}
	154
	155	#endif

include/linux/types.h+6−0

 } atomic64_t;
 #endif
+typedef struct {
+	atomic_t refcnt;
+} rcuref_t;
+#define RCUREF_INIT(i)	{ .refcnt = ATOMIC_INIT(i - 1) }
 struct list_head {
 	struct list_head *next, *prev;
 };

lib/Makefile+1−1

@@ -47,7 +47,7 @@ obj-y += bcd.o sort.o parser.o debug_locks.o random32.o \ · 46 unmodified lines
47	list_sort.o uuid.o iov_iter.o clz_ctz.o \	47	list_sort.o uuid.o iov_iter.o clz_ctz.o \
48	bsearch.o find_bit.o llist.o memweight.o kfifo.o \	48	bsearch.o find_bit.o llist.o memweight.o kfifo.o \
49	percpu-refcount.o rhashtable.o base64.o \	49	percpu-refcount.o rhashtable.o base64.o \
50	once.o refcount.o usercopy.o errseq.o bucket_locks.o \	50	once.o refcount.o rcuref.o usercopy.o errseq.o bucket_locks.o \
51	generic-radix-tree.o	51	generic-radix-tree.o
52	obj-$(CONFIG_STRING_SELFTEST) += test_string.o	52	obj-$(CONFIG_STRING_SELFTEST) += test_string.o
53	obj-y += string_helpers.o	53	obj-y += string_helpers.o

lib/rcuref.c+281−0

@@ -0,0 +1,281 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2
	3	/*
	4	* rcuref - A scalable reference count implementation for RCU managed objects
	5	*
	6	* rcuref is provided to replace open coded reference count implementations
	7	* based on atomic_t. It protects explicitely RCU managed objects which can
	8	* be visible even after the last reference has been dropped and the object
	9	* is heading towards destruction.
	10	*
	11	* A common usage pattern is:
	12	*
	13	* get()
	14	* rcu_read_lock();
	15	* p = get_ptr();
	16	* if (p && !atomic_inc_not_zero(&p->refcnt))
	17	* p = NULL;
	18	* rcu_read_unlock();
	19	* return p;
	20	*
	21	* put()
	22	* if (!atomic_dec_return(&->refcnt)) {
	23	* remove_ptr(p);
	24	* kfree_rcu((p, rcu);
	25	* }
	26	*
	27	* atomic_inc_not_zero() is implemented with a try_cmpxchg() loop which has
	28	* O(N^2) behaviour under contention with N concurrent operations.
	29	*
	30	* rcuref uses atomic_add_negative_relaxed() for the fast path, which scales
	31	* better under contention.
	32	*
	33	* Why not refcount?
	34	* =================
	35	*
	36	* In principle it should be possible to make refcount use the rcuref
	37	* scheme, but the destruction race described below cannot be prevented
	38	* unless the protected object is RCU managed.
	39	*
	40	* Theory of operation
	41	* ===================
	42	*
	43	* rcuref uses an unsigned integer reference counter. As long as the
	44	* counter value is greater than or equal to RCUREF_ONEREF and not larger
	45	* than RCUREF_MAXREF the reference is alive:
	46	*
	47	* ONEREF MAXREF SATURATED RELEASED DEAD NOREF
	48	* 0 0x7FFFFFFF 0x8000000 0xA0000000 0xBFFFFFFF 0xC0000000 0xE0000000 0xFFFFFFFF
	49	* <---valid --------> <-------saturation zone-------> <-----dead zone----->
	50	*
	51	* The get() and put() operations do unconditional increments and
	52	* decrements. The result is checked after the operation. This optimizes
	53	* for the fast path.
	54	*
	55	* If the reference count is saturated or dead, then the increments and
	56	* decrements are not harmful as the reference count still stays in the
	57	* respective zones and is always set back to STATURATED resp. DEAD. The
	58	* zones have room for 2^28 racing operations in each direction, which
	59	* makes it practically impossible to escape the zones.
	60	*
	61	* Once the last reference is dropped the reference count becomes
	62	* RCUREF_NOREF which forces rcuref_put() into the slowpath operation. The
	63	* slowpath then tries to set the reference count from RCUREF_NOREF to
	64	* RCUREF_DEAD via a cmpxchg(). This opens a small window where a
	65	* concurrent rcuref_get() can acquire the reference count and bring it
	66	* back to RCUREF_ONEREF or even drop the reference again and mark it DEAD.
	67	*
	68	* If the cmpxchg() succeeds then a concurrent rcuref_get() will result in
	69	* DEAD + 1, which is inside the dead zone. If that happens the reference
	70	* count is put back to DEAD.
	71	*
	72	* The actual race is possible due to the unconditional increment and
	73	* decrements in rcuref_get() and rcuref_put():
	74	*
	75	* T1 T2
	76	* get() put()
	77	* if (atomic_add_negative(-1, &ref->refcnt))
	78	* succeeds-> atomic_cmpxchg(&ref->refcnt, NOREF, DEAD);
	79	*
	80	* atomic_add_negative(1, &ref->refcnt); <- Elevates refcount to DEAD + 1
	81	*
	82	* As the result of T1's add is negative, the get() goes into the slow path
	83	* and observes refcnt being in the dead zone which makes the operation fail.
	84	*
	85	* Possible critical states:
	86	*
	87	* Context Counter References Operation
	88	* T1 0 1 init()
	89	* T2 1 2 get()
	90	* T1 0 1 put()
	91	* T2 -1 0 put() tries to mark dead
	92	* T1 0 1 get()
	93	* T2 0 1 put() mark dead fails
	94	* T1 -1 0 put() tries to mark dead
	95	* T1 DEAD 0 put() mark dead succeeds
	96	* T2 DEAD+1 0 get() fails and puts it back to DEAD
	97	*
	98	* Of course there are more complex scenarios, but the above illustrates
	99	* the working principle. The rest is left to the imagination of the
	100	* reader.
	101	*
	102	* Deconstruction race
	103	* ===================
	104	*
	105	* The release operation must be protected by prohibiting a grace period in
	106	* order to prevent a possible use after free:
	107	*
	108	* T1 T2
	109	* put() get()
	110	* // ref->refcnt = ONEREF
	111	* if (!atomic_add_negative(-1, &ref->refcnt))
	112	* return false; <- Not taken
	113	*
	114	* // ref->refcnt == NOREF
	115	* --> preemption
	116	* // Elevates ref->refcnt to ONEREF
	117	* if (!atomic_add_negative(1, &ref->refcnt))
	118	* return true; <- taken
	119	*
	120	* if (put(&p->ref)) { <-- Succeeds
	121	* remove_pointer(p);
	122	* kfree_rcu(p, rcu);
	123	* }
	124	*
	125	* RCU grace period ends, object is freed
	126	*
	127	* atomic_cmpxchg(&ref->refcnt, NOREF, DEAD); <- UAF
	128	*
	129	* This is prevented by disabling preemption around the put() operation as
	130	* that's in most kernel configurations cheaper than a rcu_read_lock() /
	131	* rcu_read_unlock() pair and in many cases even a NOOP. In any case it
	132	* prevents the grace period which keeps the object alive until all put()
	133	* operations complete.
	134	*
	135	* Saturation protection
	136	* =====================
	137	*
	138	* The reference count has a saturation limit RCUREF_MAXREF (INT_MAX).
	139	* Once this is exceedded the reference count becomes stale by setting it
	140	* to RCUREF_SATURATED, which will cause a memory leak, but it prevents
	141	* wrap arounds which obviously cause worse problems than a memory
	142	* leak. When saturation is reached a warning is emitted.
	143	*
	144	* Race conditions
	145	* ===============
	146	*
	147	* All reference count increment/decrement operations are unconditional and
	148	* only verified after the fact. This optimizes for the good case and takes
	149	* the occasional race vs. a dead or already saturated refcount into
	150	* account. The saturation and dead zones are large enough to accomodate
	151	* for that.
	152	*
	153	* Memory ordering
	154	* ===============
	155	*
	156	* Memory ordering rules are slightly relaxed wrt regular atomic_t functions
	157	* and provide only what is strictly required for refcounts.
	158	*
	159	* The increments are fully relaxed; these will not provide ordering. The
	160	* rationale is that whatever is used to obtain the object to increase the
	161	* reference count on will provide the ordering. For locked data
	162	* structures, its the lock acquire, for RCU/lockless data structures its
	163	* the dependent load.
	164	*
	165	* rcuref_get() provides a control dependency ordering future stores which
	166	* ensures that the object is not modified when acquiring a reference
	167	* fails.
	168	*
	169	* rcuref_put() provides release order, i.e. all prior loads and stores
	170	* will be issued before. It also provides a control dependency ordering
	171	* against the subsequent destruction of the object.
	172	*
	173	* If rcuref_put() successfully dropped the last reference and marked the
	174	* object DEAD it also provides acquire ordering.
	175	*/
	176
	177	#include <linux/export.h>
	178	#include <linux/rcuref.h>
	179
	180	/**
	181	* rcuref_get_slowpath - Slowpath of rcuref_get()
	182	* @ref: Pointer to the reference count
	183	*
	184	* Invoked when the reference count is outside of the valid zone.
	185	*
	186	* Return:
	187	* False if the reference count was already marked dead
	188	*
	189	* True if the reference count is saturated, which prevents the
	190	* object from being deconstructed ever.
	191	*/
	192	bool rcuref_get_slowpath(rcuref_t *ref)
	193	{
	194	unsigned int cnt = atomic_read(&ref->refcnt);
	195
	196	/*
	197	* If the reference count was already marked dead, undo the
	198	* increment so it stays in the middle of the dead zone and return
	199	* fail.
	200	*/
	201	if (cnt >= RCUREF_RELEASED) {
	202	atomic_set(&ref->refcnt, RCUREF_DEAD);
	203	return false;
	204	}
	205
	206	/*
	207	* If it was saturated, warn and mark it so. In case the increment
	208	* was already on a saturated value restore the saturation
	209	* marker. This keeps it in the middle of the saturation zone and
	210	* prevents the reference count from overflowing. This leaks the
	211	* object memory, but prevents the obvious reference count overflow
	212	* damage.
	213	*/
	214	if (WARN_ONCE(cnt > RCUREF_MAXREF, "rcuref saturated - leaking memory"))
	215	atomic_set(&ref->refcnt, RCUREF_SATURATED);
	216	return true;
	217	}
	218	EXPORT_SYMBOL_GPL(rcuref_get_slowpath);
	219
	220	/**
	221	* rcuref_put_slowpath - Slowpath of __rcuref_put()
	222	* @ref: Pointer to the reference count
	223	*
	224	* Invoked when the reference count is outside of the valid zone.
	225	*
	226	* Return:
	227	* True if this was the last reference with no future references
	228	* possible. This signals the caller that it can safely schedule the
	229	* object, which is protected by the reference counter, for
	230	* deconstruction.
	231	*
	232	* False if there are still active references or the put() raced
	233	* with a concurrent get()/put() pair. Caller is not allowed to
	234	* deconstruct the protected object.
	235	*/
	236	bool rcuref_put_slowpath(rcuref_t *ref)
	237	{
	238	unsigned int cnt = atomic_read(&ref->refcnt);
	239
	240	/* Did this drop the last reference? */
	241	if (likely(cnt == RCUREF_NOREF)) {
	242	/*
	243	* Carefully try to set the reference count to RCUREF_DEAD.
	244	*
	245	* This can fail if a concurrent get() operation has
	246	* elevated it again or the corresponding put() even marked
	247	* it dead already. Both are valid situations and do not
	248	* require a retry. If this fails the caller is not
	249	* allowed to deconstruct the object.
	250	*/
	251	if (atomic_cmpxchg_release(&ref->refcnt, RCUREF_NOREF, RCUREF_DEAD) != RCUREF_NOREF)
	252	return false;
	253
	254	/*
	255	* The caller can safely schedule the object for
	256	* deconstruction. Provide acquire ordering.
	257	*/
	258	smp_acquire__after_ctrl_dep();
	259	return true;
	260	}
	261
	262	/*
	263	* If the reference count was already in the dead zone, then this
	264	* put() operation is imbalanced. Warn, put the reference count back to
	265	* DEAD and tell the caller to not deconstruct the object.
	266	*/
	267	if (WARN_ONCE(cnt >= RCUREF_RELEASED, "rcuref - imbalanced put()")) {
	268	atomic_set(&ref->refcnt, RCUREF_DEAD);
	269	return false;
	270	}
	271
	272	/*
	273	* This is a put() operation on a saturated refcount. Restore the
	274	* mean saturation value and tell the caller to not deconstruct the
	275	* object.
	276	*/
	277	if (cnt > RCUREF_MAXREF)
	278	atomic_set(&ref->refcnt, RCUREF_SATURATED);
	279	return false;
	280	}
	281	EXPORT_SYMBOL_GPL(rcuref_put_slowpath);

scripts/atomic/atomics.tbl+1−1

@@ -33,7 +33,7 @@ try_cmpxchg B v p:old i:new · 32 unmodified lines
33	sub_and_test b i v	33	sub_and_test b i v
34	dec_and_test b v	34	dec_and_test b v
35	inc_and_test b v	35	inc_and_test b v
36	add_negative b i v	36	add_negative B i v
37	add_unless fb v i:a i:u	37	add_unless fb v i:a i:u
38	inc_not_zero b v	38	inc_not_zero b v
39	inc_unless_negative b v	39	inc_unless_negative b v

scripts/atomic/fallbacks/add_negative+5−6

 cat <<EOF
 /**
- * arch_${atomic}_add_negative - add and test if negative
+ * arch_${atomic}_add_negative${order} - Add and test if negative
  * @i: integer value to add
  * @v: pointer of type ${atomic}_t
  *
- * Atomically adds @i to @v and returns true
+ * Atomically adds @i to @v and returns true if the result is negative,
- * if the result is negative, or false when
+ * or false when the result is greater than or equal to zero.
- * result is greater than or equal to zero.
  */
 static __always_inline bool
-arch_${atomic}_add_negative(${int} i, ${atomic}_t *v)
+arch_${atomic}_add_negative${order}(${int} i, ${atomic}_t *v)
 {
-	return arch_${atomic}_add_return(i, v) < 0;
+	return arch_${atomic}_add_return${order}(i, v) < 0;
 }
 EOF

Merge branch 'locking/rcuref' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code