Merge tag 'locking-core-2024-03-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar: - Micro-optimize local_xchg() and the rtmutex code on x86 - Fix percpu-rwsem contention tracepoints - Simplify debugging Kconfig dependencies - Update/clarify the documentation of atomic primitives - Misc cleanups * tag 'locking-core-2024-03-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/rtmutex: Use try_cmpxchg_relaxed() in mark_rt_mutex_waiters() locking/x86: Implement local_xchg() using CMPXCHG without the LOCK prefix locking/percpu-rwsem: Trigger contention tracepoints only if contended locking/rwsem: Make DEBUG_RWSEMS and PREEMPT_RT mutually exclusive locking/rwsem: Clarify that RWSEM_READER_OWNED is just a hint locking/mutex: Simplify <linux/mutex.h> locking/qspinlock: Fix 'wait_early' set but not used warning locking/atomic: scripts: Clarify ordering of conditional atomics

Linus Torvalds committed 2 years ago

Commit a5b1a012 parents b040240 ce3576e17 files · +154 −49Browse files

Unified · Split

17 files

arch/x86/include/asm/local.h +14 −2
include/linux/atomic/atomic-arch-fallback.h +37 −9
include/linux/atomic/atomic-instrumented.h +55 −13
include/linux/atomic/atomic-long.h +19 −5
include/linux/mutex.h +2 −6
kernel/locking/percpu-rwsem.c +8 −3
kernel/locking/qspinlock_paravirt.h +1 −1
kernel/locking/rtmutex.c +5 −4
kernel/locking/rwsem.c +3 −3
lib/Kconfig.debug +2 −2
scripts/atomic/kerneldoc/add_unless +1 −0
scripts/atomic/kerneldoc/cmpxchg +1 −0
scripts/atomic/kerneldoc/dec_if_positive +1 −0
scripts/atomic/kerneldoc/dec_unless_positive +1 −0
scripts/atomic/kerneldoc/inc_not_zero +1 −0
scripts/atomic/kerneldoc/inc_unless_negative +1 −0
scripts/atomic/kerneldoc/try_cmpxchg +2 −1

arch/x86/include/asm/local.h+14−2

@@ -131,8 +131,20 @@ static inline bool local_try_cmpxchg(local_t l, long old, long new) · 130 unmodified lines
131	131	(typeof(l->a.counter) *) old, new);
132	132	}
133	133
134		−/* Always has a lock prefix */
135		−#define local_xchg(l, n) (xchg(&((l)->a.counter), (n)))
	134	+/*
	135	+ * Implement local_xchg using CMPXCHG instruction without the LOCK prefix.
	136	+ * XCHG is expensive due to the implied LOCK prefix. The processor
	137	+ * cannot prefetch cachelines if XCHG is used.
	138	+ */
	139	+static __always_inline long
	140	+local_xchg(local_t *l, long n)
	141	+{
	142	+ long c = local_read(l);
	143	+
	144	+ do { } while (!local_try_cmpxchg(l, &c, n));
	145	+
	146	+ return c;
	147	+}
136	148
137	149	/**
138	150	* local_add_unless - add unless the number is already a given value

include/linux/atomic/atomic-arch-fallback.h+37−9

@@ -2005,6 +2005,7 @@ raw_atomic_xchg_relaxed(atomic_t *v, int new) · 2004 unmodified lines
2005	2005	* @new: int value to assign
2006	2006	*
2007	2007	* If (@v == @old), atomically updates @v to @new with full ordering.
	2008	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2008	2009	*
2009	2010	* Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
2010	2011	*
@@ -2033,6 +2034,7 @@ raw_atomic_cmpxchg(atomic_t *v, int old, int new) · 22 unmodified lines
2033	2034	* @new: int value to assign
2034	2035	*
2035	2036	* If (@v == @old), atomically updates @v to @new with acquire ordering.
	2037	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2036	2038	*
2037	2039	* Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2038	2040	*
@@ -2061,6 +2063,7 @@ raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) · 22 unmodified lines
2061	2063	* @new: int value to assign
2062	2064	*
2063	2065	* If (@v == @old), atomically updates @v to @new with release ordering.
	2066	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2064	2067	*
2065	2068	* Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2066	2069	*
@@ -2088,6 +2091,7 @@ raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) · 21 unmodified lines
2088	2091	* @new: int value to assign
2089	2092	*
2090	2093	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
	2094	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2091	2095	*
2092	2096	* Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2093	2097	*
@@ -2112,7 +2116,8 @@ raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) · 18 unmodified lines
2112	2116	* @new: int value to assign
2113	2117	*
2114	2118	* If (@v == @old), atomically updates @v to @new with full ordering.
2115		− * Otherwise, updates @old to the current value of @v.
	2119	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2120	+ * and relaxed ordering is provided.
2116	2121	*
2117	2122	* Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2118	2123	*
@@ -2145,7 +2150,8 @@ raw_atomic_try_cmpxchg(atomic_t v, int old, int new) · 26 unmodified lines
2145	2150	* @new: int value to assign
2146	2151	*
2147	2152	* If (@v == @old), atomically updates @v to @new with acquire ordering.
2148		− * Otherwise, updates @old to the current value of @v.
	2153	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2154	+ * and relaxed ordering is provided.
2149	2155	*
2150	2156	* Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2151	2157	*
@@ -2178,7 +2184,8 @@ raw_atomic_try_cmpxchg_acquire(atomic_t v, int old, int new) · 26 unmodified lines
2178	2184	* @new: int value to assign
2179	2185	*
2180	2186	* If (@v == @old), atomically updates @v to @new with release ordering.
2181		− * Otherwise, updates @old to the current value of @v.
	2187	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2188	+ * and relaxed ordering is provided.
2182	2189	*
2183	2190	* Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2184	2191	*
@@ -2210,7 +2217,8 @@ raw_atomic_try_cmpxchg_release(atomic_t v, int old, int new) · 25 unmodified lines
2210	2217	* @new: int value to assign
2211	2218	*
2212	2219	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
2213		− * Otherwise, updates @old to the current value of @v.
	2220	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2221	+ * and relaxed ordering is provided.
2214	2222	*
2215	2223	* Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2216	2224	*
@@ -2403,6 +2411,7 @@ raw_atomic_add_negative_relaxed(int i, atomic_t *v) · 186 unmodified lines
2403	2411	* @u: int value to compare with
2404	2412	*
2405	2413	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	2414	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2406	2415	*
2407	2416	* Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
2408	2417	*
@@ -2432,6 +2441,7 @@ raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) · 23 unmodified lines
2432	2441	* @u: int value to compare with
2433	2442	*
2434	2443	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	2444	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2435	2445	*
2436	2446	* Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
2437	2447	*
@@ -2452,6 +2462,7 @@ raw_atomic_add_unless(atomic_t *v, int a, int u) · 14 unmodified lines
2452	2462	* @v: pointer to atomic_t
2453	2463	*
2454	2464	* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	2465	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2455	2466	*
2456	2467	* Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
2457	2468	*
@@ -2472,6 +2483,7 @@ raw_atomic_inc_not_zero(atomic_t *v) · 14 unmodified lines
2472	2483	* @v: pointer to atomic_t
2473	2484	*
2474	2485	* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	2486	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2475	2487	*
2476	2488	* Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2477	2489	*
@@ -2499,6 +2511,7 @@ raw_atomic_inc_unless_negative(atomic_t *v) · 21 unmodified lines
2499	2511	* @v: pointer to atomic_t
2500	2512	*
2501	2513	* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	2514	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2502	2515	*
2503	2516	* Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2504	2517	*
@@ -2526,6 +2539,7 @@ raw_atomic_dec_unless_positive(atomic_t *v) · 21 unmodified lines
2526	2539	* @v: pointer to atomic_t
2527	2540	*
2528	2541	* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	2542	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2529	2543	*
2530	2544	* Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2531	2545	*
@@ -4117,6 +4131,7 @@ raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) · 1585 unmodified lines
4117	4131	* @new: s64 value to assign
4118	4132	*
4119	4133	* If (@v == @old), atomically updates @v to @new with full ordering.
	4134	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4120	4135	*
4121	4136	* Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
4122	4137	*
@@ -4145,6 +4160,7 @@ raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) · 22 unmodified lines
4145	4160	* @new: s64 value to assign
4146	4161	*
4147	4162	* If (@v == @old), atomically updates @v to @new with acquire ordering.
	4163	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4148	4164	*
4149	4165	* Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
4150	4166	*
@@ -4173,6 +4189,7 @@ raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) · 22 unmodified lines
4173	4189	* @new: s64 value to assign
4174	4190	*
4175	4191	* If (@v == @old), atomically updates @v to @new with release ordering.
	4192	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4176	4193	*
4177	4194	* Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
4178	4195	*
@@ -4200,6 +4217,7 @@ raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) · 21 unmodified lines
4200	4217	* @new: s64 value to assign
4201	4218	*
4202	4219	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
	4220	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4203	4221	*
4204	4222	* Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
4205	4223	*
@@ -4224,7 +4242,8 @@ raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) · 18 unmodified lines
4224	4242	* @new: s64 value to assign
4225	4243	*
4226	4244	* If (@v == @old), atomically updates @v to @new with full ordering.
4227		− * Otherwise, updates @old to the current value of @v.
	4245	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4246	+ * and relaxed ordering is provided.
4228	4247	*
4229	4248	* Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
4230	4249	*
@@ -4257,7 +4276,8 @@ raw_atomic64_try_cmpxchg(atomic64_t v, s64 old, s64 new) · 26 unmodified lines
4257	4276	* @new: s64 value to assign
4258	4277	*
4259	4278	* If (@v == @old), atomically updates @v to @new with acquire ordering.
4260		− * Otherwise, updates @old to the current value of @v.
	4279	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4280	+ * and relaxed ordering is provided.
4261	4281	*
4262	4282	* Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
4263	4283	*
@@ -4290,7 +4310,8 @@ raw_atomic64_try_cmpxchg_acquire(atomic64_t v, s64 old, s64 new) · 26 unmodified lines
4290	4310	* @new: s64 value to assign
4291	4311	*
4292	4312	* If (@v == @old), atomically updates @v to @new with release ordering.
4293		− * Otherwise, updates @old to the current value of @v.
	4313	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4314	+ * and relaxed ordering is provided.
4294	4315	*
4295	4316	* Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
4296	4317	*
@@ -4322,7 +4343,8 @@ raw_atomic64_try_cmpxchg_release(atomic64_t v, s64 old, s64 new) · 25 unmodified lines
4322	4343	* @new: s64 value to assign
4323	4344	*
4324	4345	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
4325		− * Otherwise, updates @old to the current value of @v.
	4346	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4347	+ * and relaxed ordering is provided.
4326	4348	*
4327	4349	* Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
4328	4350	*
@@ -4515,6 +4537,7 @@ raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) · 186 unmodified lines
4515	4537	* @u: s64 value to compare with
4516	4538	*
4517	4539	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	4540	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4518	4541	*
4519	4542	* Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
4520	4543	*
@@ -4544,6 +4567,7 @@ raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) · 23 unmodified lines
4544	4567	* @u: s64 value to compare with
4545	4568	*
4546	4569	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	4570	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4547	4571	*
4548	4572	* Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
4549	4573	*
@@ -4564,6 +4588,7 @@ raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) · 14 unmodified lines
4564	4588	* @v: pointer to atomic64_t
4565	4589	*
4566	4590	* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	4591	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4567	4592	*
4568	4593	* Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
4569	4594	*
@@ -4584,6 +4609,7 @@ raw_atomic64_inc_not_zero(atomic64_t *v) · 14 unmodified lines
4584	4609	* @v: pointer to atomic64_t
4585	4610	*
4586	4611	* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	4612	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4587	4613	*
4588	4614	* Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
4589	4615	*
@@ -4611,6 +4637,7 @@ raw_atomic64_inc_unless_negative(atomic64_t *v) · 21 unmodified lines
4611	4637	* @v: pointer to atomic64_t
4612	4638	*
4613	4639	* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	4640	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4614	4641	*
4615	4642	* Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
4616	4643	*
@@ -4638,6 +4665,7 @@ raw_atomic64_dec_unless_positive(atomic64_t *v) · 21 unmodified lines
4638	4665	* @v: pointer to atomic64_t
4639	4666	*
4640	4667	* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	4668	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4641	4669	*
4642	4670	* Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
4643	4671	*
@@ -4662,4 +4690,4 @@ raw_atomic64_dec_if_positive(atomic64_t *v) · 18 unmodified lines
4662	4690	}
4663	4691
4664	4692	#endif /* _LINUX_ATOMIC_FALLBACK_H */
4665		−// eec048affea735b8464f58e6d96992101f8f85f1
	4693	+// 14850c0b0db20c62fdc78ccd1d42b98b88d76331

include/linux/atomic/atomic-instrumented.h+55−13

@@ -1182,6 +1182,7 @@ atomic_xchg_relaxed(atomic_t *v, int new) · 1181 unmodified lines
1182	1182	* @new: int value to assign
1183	1183	*
1184	1184	* If (@v == @old), atomically updates @v to @new with full ordering.
	1185	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1185	1186	*
1186	1187	* Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
1187	1188	*
@@ -1202,6 +1203,7 @@ atomic_cmpxchg(atomic_t *v, int old, int new) · 14 unmodified lines
1202	1203	* @new: int value to assign
1203	1204	*
1204	1205	* If (@v == @old), atomically updates @v to @new with acquire ordering.
	1206	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1205	1207	*
1206	1208	* Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
1207	1209	*
@@ -1221,6 +1223,7 @@ atomic_cmpxchg_acquire(atomic_t *v, int old, int new) · 13 unmodified lines
1221	1223	* @new: int value to assign
1222	1224	*
1223	1225	* If (@v == @old), atomically updates @v to @new with release ordering.
	1226	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1224	1227	*
1225	1228	* Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
1226	1229	*
@@ -1241,6 +1244,7 @@ atomic_cmpxchg_release(atomic_t *v, int old, int new) · 14 unmodified lines
1241	1244	* @new: int value to assign
1242	1245	*
1243	1246	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
	1247	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1244	1248	*
1245	1249	* Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
1246	1250	*
@@ -1260,7 +1264,8 @@ atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) · 13 unmodified lines
1260	1264	* @new: int value to assign
1261	1265	*
1262	1266	* If (@v == @old), atomically updates @v to @new with full ordering.
1263		− * Otherwise, updates @old to the current value of @v.
	1267	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1268	+ * and relaxed ordering is provided.
1264	1269	*
1265	1270	* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
1266	1271	*
@@ -1282,7 +1287,8 @@ atomic_try_cmpxchg(atomic_t v, int old, int new) · 15 unmodified lines
1282	1287	* @new: int value to assign
1283	1288	*
1284	1289	* If (@v == @old), atomically updates @v to @new with acquire ordering.
1285		− * Otherwise, updates @old to the current value of @v.
	1290	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1291	+ * and relaxed ordering is provided.
1286	1292	*
1287	1293	* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
1288	1294	*
@@ -1303,7 +1309,8 @@ atomic_try_cmpxchg_acquire(atomic_t v, int old, int new) · 14 unmodified lines
1303	1309	* @new: int value to assign
1304	1310	*
1305	1311	* If (@v == @old), atomically updates @v to @new with release ordering.
1306		− * Otherwise, updates @old to the current value of @v.
	1312	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1313	+ * and relaxed ordering is provided.
1307	1314	*
1308	1315	* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
1309	1316	*
@@ -1325,7 +1332,8 @@ atomic_try_cmpxchg_release(atomic_t v, int old, int new) · 15 unmodified lines
1325	1332	* @new: int value to assign
1326	1333	*
1327	1334	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
1328		− * Otherwise, updates @old to the current value of @v.
	1335	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1336	+ * and relaxed ordering is provided.
1329	1337	*
1330	1338	* Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
1331	1339	*
@@ -1475,6 +1483,7 @@ atomic_add_negative_relaxed(int i, atomic_t *v) · 143 unmodified lines
1475	1483	* @u: int value to compare with
1476	1484	*
1477	1485	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1486	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1478	1487	*
1479	1488	* Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
1480	1489	*
@@ -1495,6 +1504,7 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) · 14 unmodified lines
1495	1504	* @u: int value to compare with
1496	1505	*
1497	1506	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1507	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1498	1508	*
1499	1509	* Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
1500	1510	*
@@ -1513,6 +1523,7 @@ atomic_add_unless(atomic_t *v, int a, int u) · 12 unmodified lines
1513	1523	* @v: pointer to atomic_t
1514	1524	*
1515	1525	* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	1526	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1516	1527	*
1517	1528	* Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
1518	1529	*
@@ -1531,6 +1542,7 @@ atomic_inc_not_zero(atomic_t *v) · 12 unmodified lines
1531	1542	* @v: pointer to atomic_t
1532	1543	*
1533	1544	* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	1545	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1534	1546	*
1535	1547	* Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
1536	1548	*
@@ -1549,6 +1561,7 @@ atomic_inc_unless_negative(atomic_t *v) · 12 unmodified lines
1549	1561	* @v: pointer to atomic_t
1550	1562	*
1551	1563	* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	1564	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1552	1565	*
1553	1566	* Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
1554	1567	*
@@ -1567,6 +1580,7 @@ atomic_dec_unless_positive(atomic_t *v) · 12 unmodified lines
1567	1580	* @v: pointer to atomic_t
1568	1581	*
1569	1582	* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	1583	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1570	1584	*
1571	1585	* Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
1572	1586	*
@@ -2746,6 +2760,7 @@ atomic64_xchg_relaxed(atomic64_t *v, s64 new) · 1173 unmodified lines
2746	2760	* @new: s64 value to assign
2747	2761	*
2748	2762	* If (@v == @old), atomically updates @v to @new with full ordering.
	2763	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2749	2764	*
2750	2765	* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
2751	2766	*
@@ -2766,6 +2781,7 @@ atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) · 14 unmodified lines
2766	2781	* @new: s64 value to assign
2767	2782	*
2768	2783	* If (@v == @old), atomically updates @v to @new with acquire ordering.
	2784	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2769	2785	*
2770	2786	* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
2771	2787	*
@@ -2785,6 +2801,7 @@ atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) · 13 unmodified lines
2785	2801	* @new: s64 value to assign
2786	2802	*
2787	2803	* If (@v == @old), atomically updates @v to @new with release ordering.
	2804	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2788	2805	*
2789	2806	* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
2790	2807	*
@@ -2805,6 +2822,7 @@ atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) · 14 unmodified lines
2805	2822	* @new: s64 value to assign
2806	2823	*
2807	2824	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
	2825	+ * Otherwise, @v is not modified and relaxed ordering is provided.
2808	2826	*
2809	2827	* Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
2810	2828	*
@@ -2824,7 +2842,8 @@ atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) · 13 unmodified lines
2824	2842	* @new: s64 value to assign
2825	2843	*
2826	2844	* If (@v == @old), atomically updates @v to @new with full ordering.
2827		− * Otherwise, updates @old to the current value of @v.
	2845	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2846	+ * and relaxed ordering is provided.
2828	2847	*
2829	2848	* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
2830	2849	*
@@ -2846,7 +2865,8 @@ atomic64_try_cmpxchg(atomic64_t v, s64 old, s64 new) · 15 unmodified lines
2846	2865	* @new: s64 value to assign
2847	2866	*
2848	2867	* If (@v == @old), atomically updates @v to @new with acquire ordering.
2849		− * Otherwise, updates @old to the current value of @v.
	2868	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2869	+ * and relaxed ordering is provided.
2850	2870	*
2851	2871	* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
2852	2872	*
@@ -2867,7 +2887,8 @@ atomic64_try_cmpxchg_acquire(atomic64_t v, s64 old, s64 new) · 14 unmodified lines
2867	2887	* @new: s64 value to assign
2868	2888	*
2869	2889	* If (@v == @old), atomically updates @v to @new with release ordering.
2870		− * Otherwise, updates @old to the current value of @v.
	2890	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2891	+ * and relaxed ordering is provided.
2871	2892	*
2872	2893	* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
2873	2894	*
@@ -2889,7 +2910,8 @@ atomic64_try_cmpxchg_release(atomic64_t v, s64 old, s64 new) · 15 unmodified lines
2889	2910	* @new: s64 value to assign
2890	2911	*
2891	2912	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
2892		− * Otherwise, updates @old to the current value of @v.
	2913	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	2914	+ * and relaxed ordering is provided.
2893	2915	*
2894	2916	* Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
2895	2917	*
@@ -3039,6 +3061,7 @@ atomic64_add_negative_relaxed(s64 i, atomic64_t *v) · 143 unmodified lines
3039	3061	* @u: s64 value to compare with
3040	3062	*
3041	3063	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	3064	+ * Otherwise, @v is not modified and relaxed ordering is provided.
3042	3065	*
3043	3066	* Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
3044	3067	*
@@ -3059,6 +3082,7 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) · 14 unmodified lines
3059	3082	* @u: s64 value to compare with
3060	3083	*
3061	3084	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	3085	+ * Otherwise, @v is not modified and relaxed ordering is provided.
3062	3086	*
3063	3087	* Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
3064	3088	*
@@ -3077,6 +3101,7 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) · 12 unmodified lines
3077	3101	* @v: pointer to atomic64_t
3078	3102	*
3079	3103	* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	3104	+ * Otherwise, @v is not modified and relaxed ordering is provided.
3080	3105	*
3081	3106	* Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
3082	3107	*
@@ -3095,6 +3120,7 @@ atomic64_inc_not_zero(atomic64_t *v) · 12 unmodified lines
3095	3120	* @v: pointer to atomic64_t
3096	3121	*
3097	3122	* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	3123	+ * Otherwise, @v is not modified and relaxed ordering is provided.
3098	3124	*
3099	3125	* Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
3100	3126	*
@@ -3113,6 +3139,7 @@ atomic64_inc_unless_negative(atomic64_t *v) · 12 unmodified lines
3113	3139	* @v: pointer to atomic64_t
3114	3140	*
3115	3141	* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	3142	+ * Otherwise, @v is not modified and relaxed ordering is provided.
3116	3143	*
3117	3144	* Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
3118	3145	*
@@ -3131,6 +3158,7 @@ atomic64_dec_unless_positive(atomic64_t *v) · 12 unmodified lines
3131	3158	* @v: pointer to atomic64_t
3132	3159	*
3133	3160	* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	3161	+ * Otherwise, @v is not modified and relaxed ordering is provided.
3134	3162	*
3135	3163	* Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
3136	3164	*
@@ -4310,6 +4338,7 @@ atomic_long_xchg_relaxed(atomic_long_t *v, long new) · 1173 unmodified lines
4310	4338	* @new: long value to assign
4311	4339	*
4312	4340	* If (@v == @old), atomically updates @v to @new with full ordering.
	4341	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4313	4342	*
4314	4343	* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
4315	4344	*
@@ -4330,6 +4359,7 @@ atomic_long_cmpxchg(atomic_long_t *v, long old, long new) · 14 unmodified lines
4330	4359	* @new: long value to assign
4331	4360	*
4332	4361	* If (@v == @old), atomically updates @v to @new with acquire ordering.
	4362	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4333	4363	*
4334	4364	* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
4335	4365	*
@@ -4349,6 +4379,7 @@ atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) · 13 unmodified lines
4349	4379	* @new: long value to assign
4350	4380	*
4351	4381	* If (@v == @old), atomically updates @v to @new with release ordering.
	4382	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4352	4383	*
4353	4384	* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
4354	4385	*
@@ -4369,6 +4400,7 @@ atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) · 14 unmodified lines
4369	4400	* @new: long value to assign
4370	4401	*
4371	4402	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
	4403	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4372	4404	*
4373	4405	* Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
4374	4406	*
@@ -4388,7 +4420,8 @@ atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) · 13 unmodified lines
4388	4420	* @new: long value to assign
4389	4421	*
4390	4422	* If (@v == @old), atomically updates @v to @new with full ordering.
4391		− * Otherwise, updates @old to the current value of @v.
	4423	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4424	+ * and relaxed ordering is provided.
4392	4425	*
4393	4426	* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
4394	4427	*
@@ -4410,7 +4443,8 @@ atomic_long_try_cmpxchg(atomic_long_t v, long old, long new) · 15 unmodified lines
4410	4443	* @new: long value to assign
4411	4444	*
4412	4445	* If (@v == @old), atomically updates @v to @new with acquire ordering.
4413		− * Otherwise, updates @old to the current value of @v.
	4446	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4447	+ * and relaxed ordering is provided.
4414	4448	*
4415	4449	* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
4416	4450	*
@@ -4431,7 +4465,8 @@ atomic_long_try_cmpxchg_acquire(atomic_long_t v, long old, long new) · 14 unmodified lines
4431	4465	* @new: long value to assign
4432	4466	*
4433	4467	* If (@v == @old), atomically updates @v to @new with release ordering.
4434		− * Otherwise, updates @old to the current value of @v.
	4468	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4469	+ * and relaxed ordering is provided.
4435	4470	*
4436	4471	* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
4437	4472	*
@@ -4453,7 +4488,8 @@ atomic_long_try_cmpxchg_release(atomic_long_t v, long old, long new) · 15 unmodified lines
4453	4488	* @new: long value to assign
4454	4489	*
4455	4490	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
4456		− * Otherwise, updates @old to the current value of @v.
	4491	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	4492	+ * and relaxed ordering is provided.
4457	4493	*
4458	4494	* Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
4459	4495	*
@@ -4603,6 +4639,7 @@ atomic_long_add_negative_relaxed(long i, atomic_long_t *v) · 143 unmodified lines
4603	4639	* @u: long value to compare with
4604	4640	*
4605	4641	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	4642	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4606	4643	*
4607	4644	* Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
4608	4645	*
@@ -4623,6 +4660,7 @@ atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) · 14 unmodified lines
4623	4660	* @u: long value to compare with
4624	4661	*
4625	4662	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	4663	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4626	4664	*
4627	4665	* Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
4628	4666	*
@@ -4641,6 +4679,7 @@ atomic_long_add_unless(atomic_long_t *v, long a, long u) · 12 unmodified lines
4641	4679	* @v: pointer to atomic_long_t
4642	4680	*
4643	4681	* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	4682	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4644	4683	*
4645	4684	* Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
4646	4685	*
@@ -4659,6 +4698,7 @@ atomic_long_inc_not_zero(atomic_long_t *v) · 12 unmodified lines
4659	4698	* @v: pointer to atomic_long_t
4660	4699	*
4661	4700	* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	4701	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4662	4702	*
4663	4703	* Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
4664	4704	*
@@ -4677,6 +4717,7 @@ atomic_long_inc_unless_negative(atomic_long_t *v) · 12 unmodified lines
4677	4717	* @v: pointer to atomic_long_t
4678	4718	*
4679	4719	* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	4720	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4680	4721	*
4681	4722	* Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.
4682	4723	*
@@ -4695,6 +4736,7 @@ atomic_long_dec_unless_positive(atomic_long_t *v) · 12 unmodified lines
4695	4736	* @v: pointer to atomic_long_t
4696	4737	*
4697	4738	* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	4739	+ * Otherwise, @v is not modified and relaxed ordering is provided.
4698	4740	*
4699	4741	* Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there.
4700	4742	*
@@ -5008,4 +5050,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) · 307 unmodified lines
5008	5050
5009	5051
5010	5052	#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
5011		−// 2cc4bc990fef44d3836ec108f11b610f3f438184
	5053	+// ce5b65e0f1f8a276268b667194581d24bed219d4

include/linux/atomic/atomic-long.h+19−5

@@ -1352,6 +1352,7 @@ raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new) · 1351 unmodified lines
1352	1352	* @new: long value to assign
1353	1353	*
1354	1354	* If (@v == @old), atomically updates @v to @new with full ordering.
	1355	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1355	1356	*
1356	1357	* Safe to use in noinstr code; prefer atomic_long_cmpxchg() elsewhere.
1357	1358	*
@@ -1374,6 +1375,7 @@ raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) · 16 unmodified lines
1374	1375	* @new: long value to assign
1375	1376	*
1376	1377	* If (@v == @old), atomically updates @v to @new with acquire ordering.
	1378	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1377	1379	*
1378	1380	* Safe to use in noinstr code; prefer atomic_long_cmpxchg_acquire() elsewhere.
1379	1381	*
@@ -1396,6 +1398,7 @@ raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) · 16 unmodified lines
1396	1398	* @new: long value to assign
1397	1399	*
1398	1400	* If (@v == @old), atomically updates @v to @new with release ordering.
	1401	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1399	1402	*
1400	1403	* Safe to use in noinstr code; prefer atomic_long_cmpxchg_release() elsewhere.
1401	1404	*
@@ -1418,6 +1421,7 @@ raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) · 16 unmodified lines
1418	1421	* @new: long value to assign
1419	1422	*
1420	1423	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
	1424	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1421	1425	*
1422	1426	* Safe to use in noinstr code; prefer atomic_long_cmpxchg_relaxed() elsewhere.
1423	1427	*
@@ -1440,7 +1444,8 @@ raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) · 16 unmodified lines
1440	1444	* @new: long value to assign
1441	1445	*
1442	1446	* If (@v == @old), atomically updates @v to @new with full ordering.
1443		− * Otherwise, updates @old to the current value of @v.
	1447	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1448	+ * and relaxed ordering is provided.
1444	1449	*
1445	1450	* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg() elsewhere.
1446	1451	*
@@ -1463,7 +1468,8 @@ raw_atomic_long_try_cmpxchg(atomic_long_t v, long old, long new) · 16 unmodified lines
1463	1468	* @new: long value to assign
1464	1469	*
1465	1470	* If (@v == @old), atomically updates @v to @new with acquire ordering.
1466		− * Otherwise, updates @old to the current value of @v.
	1471	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1472	+ * and relaxed ordering is provided.
1467	1473	*
1468	1474	* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_acquire() elsewhere.
1469	1475	*
@@ -1486,7 +1492,8 @@ raw_atomic_long_try_cmpxchg_acquire(atomic_long_t v, long old, long new) · 16 unmodified lines
1486	1492	* @new: long value to assign
1487	1493	*
1488	1494	* If (@v == @old), atomically updates @v to @new with release ordering.
1489		− * Otherwise, updates @old to the current value of @v.
	1495	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1496	+ * and relaxed ordering is provided.
1490	1497	*
1491	1498	* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_release() elsewhere.
1492	1499	*
@@ -1509,7 +1516,8 @@ raw_atomic_long_try_cmpxchg_release(atomic_long_t v, long old, long new) · 16 unmodified lines
1509	1516	* @new: long value to assign
1510	1517	*
1511	1518	* If (@v == @old), atomically updates @v to @new with relaxed ordering.
1512		− * Otherwise, updates @old to the current value of @v.
	1519	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	1520	+ * and relaxed ordering is provided.
1513	1521	*
1514	1522	* Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_relaxed() elsewhere.
1515	1523	*
@@ -1677,6 +1685,7 @@ raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) · 161 unmodified lines
1677	1685	* @u: long value to compare with
1678	1686	*
1679	1687	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1688	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1680	1689	*
1681	1690	* Safe to use in noinstr code; prefer atomic_long_fetch_add_unless() elsewhere.
1682	1691	*
@@ -1699,6 +1708,7 @@ raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) · 16 unmodified lines
1699	1708	* @u: long value to compare with
1700	1709	*
1701	1710	* If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1711	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1702	1712	*
1703	1713	* Safe to use in noinstr code; prefer atomic_long_add_unless() elsewhere.
1704	1714	*
@@ -1719,6 +1729,7 @@ raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) · 14 unmodified lines
1719	1729	* @v: pointer to atomic_long_t
1720	1730	*
1721	1731	* If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	1732	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1722	1733	*
1723	1734	* Safe to use in noinstr code; prefer atomic_long_inc_not_zero() elsewhere.
1724	1735	*
@@ -1739,6 +1750,7 @@ raw_atomic_long_inc_not_zero(atomic_long_t *v) · 14 unmodified lines
1739	1750	* @v: pointer to atomic_long_t
1740	1751	*
1741	1752	* If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	1753	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1742	1754	*
1743	1755	* Safe to use in noinstr code; prefer atomic_long_inc_unless_negative() elsewhere.
1744	1756	*
@@ -1759,6 +1771,7 @@ raw_atomic_long_inc_unless_negative(atomic_long_t *v) · 14 unmodified lines
1759	1771	* @v: pointer to atomic_long_t
1760	1772	*
1761	1773	* If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	1774	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1762	1775	*
1763	1776	* Safe to use in noinstr code; prefer atomic_long_dec_unless_positive() elsewhere.
1764	1777	*
@@ -1779,6 +1792,7 @@ raw_atomic_long_dec_unless_positive(atomic_long_t *v) · 14 unmodified lines
1779	1792	* @v: pointer to atomic_long_t
1780	1793	*
1781	1794	* If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	1795	+ * Otherwise, @v is not modified and relaxed ordering is provided.
1782	1796	*
1783	1797	* Safe to use in noinstr code; prefer atomic_long_dec_if_positive() elsewhere.
1784	1798	*
@@ -1795,4 +1809,4 @@ raw_atomic_long_dec_if_positive(atomic_long_t *v) · 10 unmodified lines
1795	1809	}
1796	1810
1797	1811	#endif /* _LINUX_ATOMIC_LONG_H */
1798		−// 4ef23f98c73cff96d239896175fd26b10b88899e
	1812	+// 1c4a26fc77f345342953770ebe3c4d08e7ce2f9a

include/linux/mutex.h+2−6

@@ -32,11 +32,9 @@ · 31 unmodified lines
32	32	# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
33	33	#endif
34	34
35		−#ifndef CONFIG_PREEMPT_RT
36		−
37	35	#ifdef CONFIG_DEBUG_MUTEXES
38	36
39		−#define __DEBUG_MUTEX_INITIALIZER(lockname) \
	37	+# define __DEBUG_MUTEX_INITIALIZER(lockname) \
40	38	, .magic = &lockname
41	39
42	40	extern void mutex_destroy(struct mutex *lock);
@@ -49,6 +47,7 @@ static inline void mutex_destroy(struct mutex *lock) {} · 6 unmodified lines
49	47
50	48	#endif
51	49
	50	+#ifndef CONFIG_PREEMPT_RT
52	51	/**
53	52	* mutex_init - initialize the mutex
54	53	* @mutex: the mutex to be initialized
@@ -101,9 +100,6 @@ extern bool mutex_is_locked(struct mutex *lock); · 46 unmodified lines
101	100
102	101	extern void __mutex_rt_init(struct mutex lock, const char name,
103	102	struct lock_class_key *key);
104		−extern int mutex_trylock(struct mutex *lock);
105		−
106		−static inline void mutex_destroy(struct mutex *lock) { }
107	103
108	104	#define mutex_is_locked(l) rt_mutex_base_is_locked(&(l)->rtmutex)
109	105

kernel/locking/percpu-rwsem.c+8−3

@@ -223,9 +223,10 @@ static bool readers_active_check(struct percpu_rw_semaphore *sem) · 222 unmodified lines
223	223
224	224	void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
225	225	{
	226	+ bool contended = false;
	227	+
226	228	might_sleep();
227	229	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
228		− trace_contention_begin(sem, LCB_F_PERCPU \| LCB_F_WRITE);
229	230
230	231	/* Notify readers to take the slow path. */
231	232	rcu_sync_enter(&sem->rss);
@@ -234,8 +235,11 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem) · 2 unmodified lines
234	235	* Try set sem->block; this provides writer-writer exclusion.
235	236	* Having sem->block set makes new readers block.
236	237	*/
237		− if (!__percpu_down_write_trylock(sem))
	238	+ if (!__percpu_down_write_trylock(sem)) {
	239	+ trace_contention_begin(sem, LCB_F_PERCPU \| LCB_F_WRITE);
238	240	percpu_rwsem_wait(sem, /* .reader = */ false);
	241	+ contended = true;
	242	+ }
239	243
240	244	/* smp_mb() implied by __percpu_down_write_trylock() on success -- D matches A */
241	245
@@ -247,7 +251,8 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem) · 5 unmodified lines
247	251
248	252	/* Wait for all active readers to complete. */
249	253	rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);
250		− trace_contention_end(sem, 0);
	254	+ if (contended)
	255	+ trace_contention_end(sem, 0);
251	256	}
252	257	EXPORT_SYMBOL_GPL(percpu_down_write);
253	258

kernel/locking/qspinlock_paravirt.h+1−1

@@ -294,8 +294,8 @@ static void pv_wait_node(struct mcs_spinlock node, struct mcs_spinlock prev) · 293 unmodified lines
294	294	{
295	295	struct pv_node pn = (struct pv_node )node;
296	296	struct pv_node pp = (struct pv_node )prev;
	297	+ bool __maybe_unused wait_early;
297	298	int loop;
298		− bool wait_early;
299	299
300	300	for (;;) {
301	301	for (wait_early = false, loop = SPIN_THRESHOLD; loop; loop--) {

kernel/locking/rtmutex.c+5−4

@@ -237,12 +237,13 @@ static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock, · 236 unmodified lines
237	237	*/
238	238	static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
239	239	{
240		− unsigned long owner, p = (unsigned long ) &lock->owner;
	240	+ unsigned long p = (unsigned long ) &lock->owner;
	241	+ unsigned long owner, new;
241	242
	243	+ owner = READ_ONCE(*p);
242	244	do {
243		− owner = *p;
244		− } while (cmpxchg_relaxed(p, owner,
245		− owner \| RT_MUTEX_HAS_WAITERS) != owner);
	245	+ new = owner \| RT_MUTEX_HAS_WAITERS;
	246	+ } while (!try_cmpxchg_relaxed(p, &owner, new));
246	247
247	248	/*
248	249	* The cmpxchg loop above is relaxed to avoid back-to-back ACQUIRE

kernel/locking/rwsem.c+3−3

@@ -35,7 +35,7 @@ · 34 unmodified lines
35	35	/*
36	36	* The least significant 2 bits of the owner value has the following
37	37	* meanings when set.
38		− * - Bit 0: RWSEM_READER_OWNED - The rwsem is owned by readers
	38	+ * - Bit 0: RWSEM_READER_OWNED - rwsem may be owned by readers (just a hint)
39	39	* - Bit 1: RWSEM_NONSPINNABLE - Cannot spin on a reader-owned lock
40	40	*
41	41	* When the rwsem is reader-owned and a spinning writer has timed out,
@@ -1002,8 +1002,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat · 960 unmodified lines
1002	1002
1003	1003	/*
1004	1004	* To prevent a constant stream of readers from starving a sleeping
1005		− * waiter, don't attempt optimistic lock stealing if the lock is
1006		− * currently owned by readers.
	1005	+ * writer, don't attempt optimistic lock stealing if the lock is
	1006	+ * very likely owned by readers.
1007	1007	*/
1008	1008	if ((atomic_long_read(&sem->owner) & RWSEM_READER_OWNED) &&
1009	1009	(rcnt > 1) && !(count & RWSEM_WRITER_LOCKED))

lib/Kconfig.debug+2−2

@@ -1303,7 +1303,7 @@ config PROVE_LOCKING · 1302 unmodified lines
1303	1303	select DEBUG_SPINLOCK
1304	1304	select DEBUG_MUTEXES if !PREEMPT_RT
1305	1305	select DEBUG_RT_MUTEXES if RT_MUTEXES
1306		− select DEBUG_RWSEMS
	1306	+ select DEBUG_RWSEMS if !PREEMPT_RT
1307	1307	select DEBUG_WW_MUTEX_SLOWPATH
1308	1308	select DEBUG_LOCK_ALLOC
1309	1309	select PREEMPT_COUNT if !ARCH_NO_PREEMPT
@@ -1426,7 +1426,7 @@ config DEBUG_WW_MUTEX_SLOWPATH · 116 unmodified lines
1426	1426
1427	1427	config DEBUG_RWSEMS
1428	1428	bool "RW Semaphore debugging: basic checks"
1429		− depends on DEBUG_KERNEL
	1429	+ depends on DEBUG_KERNEL && !PREEMPT_RT
1430	1430	help
1431	1431	This debugging feature allows mismatched rw semaphore locks
1432	1432	and unlocks to be detected and reported.

scripts/atomic/kerneldoc/add_unless+1−0

@@ -10,6 +10,7 @@ cat <<EOF · 9 unmodified lines
10	10	* @u: ${int} value to compare with
11	11	*
12	12	* If (@v != @u), atomically updates @v to (@v + @a) with ${desc_order} ordering.
	13	+ * Otherwise, @v is not modified and relaxed ordering is provided.
13	14	*
14	15	* ${desc_noinstr}
15	16	*

scripts/atomic/kerneldoc/cmpxchg+1−0

@@ -6,6 +6,7 @@ cat <<EOF · 5 unmodified lines
6	6	* @new: ${int} value to assign
7	7	*
8	8	* If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
	9	+ * Otherwise, @v is not modified and relaxed ordering is provided.
9	10	*
10	11	* ${desc_noinstr}
11	12	*

scripts/atomic/kerneldoc/dec_if_positive+1−0

@@ -4,6 +4,7 @@ cat <<EOF · 3 unmodified lines
4	4	* @v: pointer to ${atomic}_t
5	5	*
6	6	* If (@v > 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
	7	+ * Otherwise, @v is not modified and relaxed ordering is provided.
7	8	*
8	9	* ${desc_noinstr}
9	10	*

scripts/atomic/kerneldoc/dec_unless_positive+1−0

@@ -4,6 +4,7 @@ cat <<EOF · 3 unmodified lines
4	4	* @v: pointer to ${atomic}_t
5	5	*
6	6	* If (@v <= 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
	7	+ * Otherwise, @v is not modified and relaxed ordering is provided.
7	8	*
8	9	* ${desc_noinstr}
9	10	*

scripts/atomic/kerneldoc/inc_not_zero+1−0

@@ -4,6 +4,7 @@ cat <<EOF · 3 unmodified lines
4	4	* @v: pointer to ${atomic}_t
5	5	*
6	6	* If (@v != 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
	7	+ * Otherwise, @v is not modified and relaxed ordering is provided.
7	8	*
8	9	* ${desc_noinstr}
9	10	*

scripts/atomic/kerneldoc/inc_unless_negative+1−0

@@ -4,6 +4,7 @@ cat <<EOF · 3 unmodified lines
4	4	* @v: pointer to ${atomic}_t
5	5	*
6	6	* If (@v >= 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
	7	+ * Otherwise, @v is not modified and relaxed ordering is provided.
7	8	*
8	9	* ${desc_noinstr}
9	10	*

scripts/atomic/kerneldoc/try_cmpxchg+2−1

@@ -6,7 +6,8 @@ cat <<EOF · 5 unmodified lines
6	6	* @new: ${int} value to assign
7	7	*
8	8	* If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
9		− * Otherwise, updates @old to the current value of @v.
	9	+ * Otherwise, @v is not modified, @old is updated to the current value of @v,
	10	+ * and relaxed ordering is provided.
10	11	*
11	12	* ${desc_noinstr}
12	13	*

Merge tag 'locking-core-2024-03-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code