atomics: Provide atomic_add_negative() variants

atomic_add_negative() does not provide the relaxed/acquire/release variants. Provide them in preparation for a new scalable reference count algorithm.

Thomas Gleixner committed 3 years ago

Commit e5ab9ef1 parent fe15c265 files · +309 −18Browse files

Unified · Split

5 files

include/linux/atomic/atomic-arch-fallback.h +199 −9
include/linux/atomic/atomic-instrumented.h +67 −1
include/linux/atomic/atomic-long.h +37 −1
scripts/atomic/atomics.tbl +1 −1
scripts/atomic/fallbacks/add_negative +5 −6

include/linux/atomic/atomic-arch-fallback.h+199−9

@@ -1208,15 +1208,21 @@ arch_atomic_inc_and_test(atomic_t *v) · 1207 unmodified lines
1208	#define arch_atomic_inc_and_test arch_atomic_inc_and_test	1208	#define arch_atomic_inc_and_test arch_atomic_inc_and_test
1209	#endif	1209	#endif
1210		1210
		1211	#ifndef arch_atomic_add_negative_relaxed
		1212	#ifdef arch_atomic_add_negative
		1213	#define arch_atomic_add_negative_acquire arch_atomic_add_negative
		1214	#define arch_atomic_add_negative_release arch_atomic_add_negative
		1215	#define arch_atomic_add_negative_relaxed arch_atomic_add_negative
		1216	#endif /* arch_atomic_add_negative */
		1217
1211	#ifndef arch_atomic_add_negative	1218	#ifndef arch_atomic_add_negative
1212	/**	1219	/**
1213	* arch_atomic_add_negative - add and test if negative	1220	* arch_atomic_add_negative - Add and test if negative
1214	* @i: integer value to add	1221	* @i: integer value to add
1215	* @v: pointer of type atomic_t	1222	* @v: pointer of type atomic_t
1216	*	1223	*
1217	* Atomically adds @i to @v and returns true	1224	* Atomically adds @i to @v and returns true if the result is negative,
1218	* if the result is negative, or false when	1225	* or false when the result is greater than or equal to zero.
1219	* result is greater than or equal to zero.
1220	*/	1226	*/
1221	static __always_inline bool	1227	static __always_inline bool
1222	arch_atomic_add_negative(int i, atomic_t *v)	1228	arch_atomic_add_negative(int i, atomic_t *v)
@@ -1226,6 +1232,95 @@ arch_atomic_add_negative(int i, atomic_t *v) · 3 unmodified lines
1226	#define arch_atomic_add_negative arch_atomic_add_negative	1232	#define arch_atomic_add_negative arch_atomic_add_negative
1227	#endif	1233	#endif
1228		1234
		1235	#ifndef arch_atomic_add_negative_acquire
		1236	/**
		1237	* arch_atomic_add_negative_acquire - Add and test if negative
		1238	* @i: integer value to add
		1239	* @v: pointer of type atomic_t
		1240	*
		1241	* Atomically adds @i to @v and returns true if the result is negative,
		1242	* or false when the result is greater than or equal to zero.
		1243	*/
		1244	static __always_inline bool
		1245	arch_atomic_add_negative_acquire(int i, atomic_t *v)
		1246	{
		1247	return arch_atomic_add_return_acquire(i, v) < 0;
		1248	}
		1249	#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
		1250	#endif
		1251
		1252	#ifndef arch_atomic_add_negative_release
		1253	/**
		1254	* arch_atomic_add_negative_release - Add and test if negative
		1255	* @i: integer value to add
		1256	* @v: pointer of type atomic_t
		1257	*
		1258	* Atomically adds @i to @v and returns true if the result is negative,
		1259	* or false when the result is greater than or equal to zero.
		1260	*/
		1261	static __always_inline bool
		1262	arch_atomic_add_negative_release(int i, atomic_t *v)
		1263	{
		1264	return arch_atomic_add_return_release(i, v) < 0;
		1265	}
		1266	#define arch_atomic_add_negative_release arch_atomic_add_negative_release
		1267	#endif
		1268
		1269	#ifndef arch_atomic_add_negative_relaxed
		1270	/**
		1271	* arch_atomic_add_negative_relaxed - Add and test if negative
		1272	* @i: integer value to add
		1273	* @v: pointer of type atomic_t
		1274	*
		1275	* Atomically adds @i to @v and returns true if the result is negative,
		1276	* or false when the result is greater than or equal to zero.
		1277	*/
		1278	static __always_inline bool
		1279	arch_atomic_add_negative_relaxed(int i, atomic_t *v)
		1280	{
		1281	return arch_atomic_add_return_relaxed(i, v) < 0;
		1282	}
		1283	#define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed
		1284	#endif
		1285
		1286	#else /* arch_atomic_add_negative_relaxed */
		1287
		1288	#ifndef arch_atomic_add_negative_acquire
		1289	static __always_inline bool
		1290	arch_atomic_add_negative_acquire(int i, atomic_t *v)
		1291	{
		1292	bool ret = arch_atomic_add_negative_relaxed(i, v);
		1293	__atomic_acquire_fence();
		1294	return ret;
		1295	}
		1296	#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
		1297	#endif
		1298
		1299	#ifndef arch_atomic_add_negative_release
		1300	static __always_inline bool
		1301	arch_atomic_add_negative_release(int i, atomic_t *v)
		1302	{
		1303	__atomic_release_fence();
		1304	return arch_atomic_add_negative_relaxed(i, v);
		1305	}
		1306	#define arch_atomic_add_negative_release arch_atomic_add_negative_release
		1307	#endif
		1308
		1309	#ifndef arch_atomic_add_negative
		1310	static __always_inline bool
		1311	arch_atomic_add_negative(int i, atomic_t *v)
		1312	{
		1313	bool ret;
		1314	__atomic_pre_full_fence();
		1315	ret = arch_atomic_add_negative_relaxed(i, v);
		1316	__atomic_post_full_fence();
		1317	return ret;
		1318	}
		1319	#define arch_atomic_add_negative arch_atomic_add_negative
		1320	#endif
		1321
		1322	#endif /* arch_atomic_add_negative_relaxed */
		1323
1229	#ifndef arch_atomic_fetch_add_unless	1324	#ifndef arch_atomic_fetch_add_unless
1230	/**	1325	/**
1231	* arch_atomic_fetch_add_unless - add unless the number is already a given value	1326	* arch_atomic_fetch_add_unless - add unless the number is already a given value
@@ -2329,15 +2424,21 @@ arch_atomic64_inc_and_test(atomic64_t *v) · 1097 unmodified lines
2329	#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test	2424	#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2330	#endif	2425	#endif
2331		2426
		2427	#ifndef arch_atomic64_add_negative_relaxed
		2428	#ifdef arch_atomic64_add_negative
		2429	#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative
		2430	#define arch_atomic64_add_negative_release arch_atomic64_add_negative
		2431	#define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative
		2432	#endif /* arch_atomic64_add_negative */
		2433
2332	#ifndef arch_atomic64_add_negative	2434	#ifndef arch_atomic64_add_negative
2333	/**	2435	/**
2334	* arch_atomic64_add_negative - add and test if negative	2436	* arch_atomic64_add_negative - Add and test if negative
2335	* @i: integer value to add	2437	* @i: integer value to add
2336	* @v: pointer of type atomic64_t	2438	* @v: pointer of type atomic64_t
2337	*	2439	*
2338	* Atomically adds @i to @v and returns true	2440	* Atomically adds @i to @v and returns true if the result is negative,
2339	* if the result is negative, or false when	2441	* or false when the result is greater than or equal to zero.
2340	* result is greater than or equal to zero.
2341	*/	2442	*/
2342	static __always_inline bool	2443	static __always_inline bool
2343	arch_atomic64_add_negative(s64 i, atomic64_t *v)	2444	arch_atomic64_add_negative(s64 i, atomic64_t *v)
@@ -2347,6 +2448,95 @@ arch_atomic64_add_negative(s64 i, atomic64_t *v) · 3 unmodified lines
2347	#define arch_atomic64_add_negative arch_atomic64_add_negative	2448	#define arch_atomic64_add_negative arch_atomic64_add_negative
2348	#endif	2449	#endif
2349		2450
		2451	#ifndef arch_atomic64_add_negative_acquire
		2452	/**
		2453	* arch_atomic64_add_negative_acquire - Add and test if negative
		2454	* @i: integer value to add
		2455	* @v: pointer of type atomic64_t
		2456	*
		2457	* Atomically adds @i to @v and returns true if the result is negative,
		2458	* or false when the result is greater than or equal to zero.
		2459	*/
		2460	static __always_inline bool
		2461	arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
		2462	{
		2463	return arch_atomic64_add_return_acquire(i, v) < 0;
		2464	}
		2465	#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire
		2466	#endif
		2467
		2468	#ifndef arch_atomic64_add_negative_release
		2469	/**
		2470	* arch_atomic64_add_negative_release - Add and test if negative
		2471	* @i: integer value to add
		2472	* @v: pointer of type atomic64_t
		2473	*
		2474	* Atomically adds @i to @v and returns true if the result is negative,
		2475	* or false when the result is greater than or equal to zero.
		2476	*/
		2477	static __always_inline bool
		2478	arch_atomic64_add_negative_release(s64 i, atomic64_t *v)
		2479	{
		2480	return arch_atomic64_add_return_release(i, v) < 0;
		2481	}
		2482	#define arch_atomic64_add_negative_release arch_atomic64_add_negative_release
		2483	#endif
		2484
		2485	#ifndef arch_atomic64_add_negative_relaxed
		2486	/**
		2487	* arch_atomic64_add_negative_relaxed - Add and test if negative
		2488	* @i: integer value to add
		2489	* @v: pointer of type atomic64_t
		2490	*
		2491	* Atomically adds @i to @v and returns true if the result is negative,
		2492	* or false when the result is greater than or equal to zero.
		2493	*/
		2494	static __always_inline bool
		2495	arch_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
		2496	{
		2497	return arch_atomic64_add_return_relaxed(i, v) < 0;
		2498	}
		2499	#define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed
		2500	#endif
		2501
		2502	#else /* arch_atomic64_add_negative_relaxed */
		2503
		2504	#ifndef arch_atomic64_add_negative_acquire
		2505	static __always_inline bool
		2506	arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
		2507	{
		2508	bool ret = arch_atomic64_add_negative_relaxed(i, v);
		2509	__atomic_acquire_fence();
		2510	return ret;
		2511	}
		2512	#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire
		2513	#endif
		2514
		2515	#ifndef arch_atomic64_add_negative_release
		2516	static __always_inline bool
		2517	arch_atomic64_add_negative_release(s64 i, atomic64_t *v)
		2518	{
		2519	__atomic_release_fence();
		2520	return arch_atomic64_add_negative_relaxed(i, v);
		2521	}
		2522	#define arch_atomic64_add_negative_release arch_atomic64_add_negative_release
		2523	#endif
		2524
		2525	#ifndef arch_atomic64_add_negative
		2526	static __always_inline bool
		2527	arch_atomic64_add_negative(s64 i, atomic64_t *v)
		2528	{
		2529	bool ret;
		2530	__atomic_pre_full_fence();
		2531	ret = arch_atomic64_add_negative_relaxed(i, v);
		2532	__atomic_post_full_fence();
		2533	return ret;
		2534	}
		2535	#define arch_atomic64_add_negative arch_atomic64_add_negative
		2536	#endif
		2537
		2538	#endif /* arch_atomic64_add_negative_relaxed */
		2539
2350	#ifndef arch_atomic64_fetch_add_unless	2540	#ifndef arch_atomic64_fetch_add_unless
2351	/**	2541	/**
2352	* arch_atomic64_fetch_add_unless - add unless the number is already a given value	2542	* arch_atomic64_fetch_add_unless - add unless the number is already a given value
@@ -2456,4 +2646,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) · 103 unmodified lines
2456	#endif	2646	#endif
2457		2647
2458	#endif /* _LINUX_ATOMIC_FALLBACK_H */	2648	#endif /* _LINUX_ATOMIC_FALLBACK_H */
2459	// b5e87bdd5ede61470c29f7a7e4de781af3770f09	2649	// 00071fffa021cec66f6290d706d69c91df87bade

include/linux/atomic/atomic-instrumented.h+67−1

@@ -592,6 +592,28 @@ atomic_add_negative(int i, atomic_t *v) · 591 unmodified lines
592	return arch_atomic_add_negative(i, v);	592	return arch_atomic_add_negative(i, v);
593	}	593	}
594		594
		595	static __always_inline bool
		596	atomic_add_negative_acquire(int i, atomic_t *v)
		597	{
		598	instrument_atomic_read_write(v, sizeof(*v));
		599	return arch_atomic_add_negative_acquire(i, v);
		600	}
		601
		602	static __always_inline bool
		603	atomic_add_negative_release(int i, atomic_t *v)
		604	{
		605	kcsan_release();
		606	instrument_atomic_read_write(v, sizeof(*v));
		607	return arch_atomic_add_negative_release(i, v);
		608	}
		609
		610	static __always_inline bool
		611	atomic_add_negative_relaxed(int i, atomic_t *v)
		612	{
		613	instrument_atomic_read_write(v, sizeof(*v));
		614	return arch_atomic_add_negative_relaxed(i, v);
		615	}
		616
595	static __always_inline int	617	static __always_inline int
596	atomic_fetch_add_unless(atomic_t *v, int a, int u)	618	atomic_fetch_add_unless(atomic_t *v, int a, int u)
597	{	619	{
@@ -1211,6 +1233,28 @@ atomic64_add_negative(s64 i, atomic64_t *v) · 613 unmodified lines
1211	return arch_atomic64_add_negative(i, v);	1233	return arch_atomic64_add_negative(i, v);
1212	}	1234	}
1213		1235
		1236	static __always_inline bool
		1237	atomic64_add_negative_acquire(s64 i, atomic64_t *v)
		1238	{
		1239	instrument_atomic_read_write(v, sizeof(*v));
		1240	return arch_atomic64_add_negative_acquire(i, v);
		1241	}
		1242
		1243	static __always_inline bool
		1244	atomic64_add_negative_release(s64 i, atomic64_t *v)
		1245	{
		1246	kcsan_release();
		1247	instrument_atomic_read_write(v, sizeof(*v));
		1248	return arch_atomic64_add_negative_release(i, v);
		1249	}
		1250
		1251	static __always_inline bool
		1252	atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
		1253	{
		1254	instrument_atomic_read_write(v, sizeof(*v));
		1255	return arch_atomic64_add_negative_relaxed(i, v);
		1256	}
		1257
1214	static __always_inline s64	1258	static __always_inline s64
1215	atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)	1259	atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216	{	1260	{
@@ -1830,6 +1874,28 @@ atomic_long_add_negative(long i, atomic_long_t *v) · 613 unmodified lines
1830	return arch_atomic_long_add_negative(i, v);	1874	return arch_atomic_long_add_negative(i, v);
1831	}	1875	}
1832		1876
		1877	static __always_inline bool
		1878	atomic_long_add_negative_acquire(long i, atomic_long_t *v)
		1879	{
		1880	instrument_atomic_read_write(v, sizeof(*v));
		1881	return arch_atomic_long_add_negative_acquire(i, v);
		1882	}
		1883
		1884	static __always_inline bool
		1885	atomic_long_add_negative_release(long i, atomic_long_t *v)
		1886	{
		1887	kcsan_release();
		1888	instrument_atomic_read_write(v, sizeof(*v));
		1889	return arch_atomic_long_add_negative_release(i, v);
		1890	}
		1891
		1892	static __always_inline bool
		1893	atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
		1894	{
		1895	instrument_atomic_read_write(v, sizeof(*v));
		1896	return arch_atomic_long_add_negative_relaxed(i, v);
		1897	}
		1898
1833	static __always_inline long	1899	static __always_inline long
1834	atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)	1900	atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835	{	1901	{
@@ -2083,4 +2149,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) · 247 unmodified lines
2083	})	2149	})
2084		2150
2085	#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */	2151	#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086	// 764f741eb77a7ad565dc8d99ce2837d5542e8aee	2152	// 1b485de9cbaa4900de59e14ee2084357eaeb1c3a

include/linux/atomic/atomic-long.h+37−1

@@ -479,6 +479,24 @@ arch_atomic_long_add_negative(long i, atomic_long_t *v) · 478 unmodified lines
479	return arch_atomic64_add_negative(i, v);	479	return arch_atomic64_add_negative(i, v);
480	}	480	}
481		481
		482	static __always_inline bool
		483	arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
		484	{
		485	return arch_atomic64_add_negative_acquire(i, v);
		486	}
		487
		488	static __always_inline bool
		489	arch_atomic_long_add_negative_release(long i, atomic_long_t *v)
		490	{
		491	return arch_atomic64_add_negative_release(i, v);
		492	}
		493
		494	static __always_inline bool
		495	arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
		496	{
		497	return arch_atomic64_add_negative_relaxed(i, v);
		498	}
		499
482	static __always_inline long	500	static __always_inline long
483	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)	501	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
484	{	502	{
@@ -973,6 +991,24 @@ arch_atomic_long_add_negative(long i, atomic_long_t *v) · 488 unmodified lines
973	return arch_atomic_add_negative(i, v);	991	return arch_atomic_add_negative(i, v);
974	}	992	}
975		993
		994	static __always_inline bool
		995	arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
		996	{
		997	return arch_atomic_add_negative_acquire(i, v);
		998	}
		999
		1000	static __always_inline bool
		1001	arch_atomic_long_add_negative_release(long i, atomic_long_t *v)
		1002	{
		1003	return arch_atomic_add_negative_release(i, v);
		1004	}
		1005
		1006	static __always_inline bool
		1007	arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
		1008	{
		1009	return arch_atomic_add_negative_relaxed(i, v);
		1010	}
		1011
976	static __always_inline long	1012	static __always_inline long
977	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)	1013	arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
978	{	1014	{
@@ -1011,4 +1047,4 @@ arch_atomic_long_dec_if_positive(atomic_long_t *v) · 32 unmodified lines
1011		1047
1012	#endif /* CONFIG_64BIT */	1048	#endif /* CONFIG_64BIT */
1013	#endif /* _LINUX_ATOMIC_LONG_H */	1049	#endif /* _LINUX_ATOMIC_LONG_H */
1014	// e8f0e08ff072b74d180eabe2ad001282b38c2c88	1050	// a194c07d7d2f4b0e178d3c118c919775d5d65f50

scripts/atomic/atomics.tbl+1−1

@@ -33,7 +33,7 @@ try_cmpxchg B v p:old i:new · 32 unmodified lines
33	sub_and_test b i v	33	sub_and_test b i v
34	dec_and_test b v	34	dec_and_test b v
35	inc_and_test b v	35	inc_and_test b v
36	add_negative b i v	36	add_negative B i v
37	add_unless fb v i:a i:u	37	add_unless fb v i:a i:u
38	inc_not_zero b v	38	inc_not_zero b v
39	inc_unless_negative b v	39	inc_unless_negative b v

scripts/atomic/fallbacks/add_negative+5−6

 cat <<EOF
 /**
- * arch_${atomic}_add_negative - add and test if negative
+ * arch_${atomic}_add_negative${order} - Add and test if negative
  * @i: integer value to add
  * @v: pointer of type ${atomic}_t
  *
- * Atomically adds @i to @v and returns true
+ * Atomically adds @i to @v and returns true if the result is negative,
- * if the result is negative, or false when
+ * or false when the result is greater than or equal to zero.
- * result is greater than or equal to zero.
  */
 static __always_inline bool
-arch_${atomic}_add_negative(${int} i, ${atomic}_t *v)
+arch_${atomic}_add_negative${order}(${int} i, ${atomic}_t *v)
 {
-	return arch_${atomic}_add_return(i, v) < 0;
+	return arch_${atomic}_add_return${order}(i, v) < 0;
 }
 EOF

atomics: Provide atomic_add_negative() variants

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code