History

Merge tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull atomics rework from Thomas Gleixner: "Peter Zijlstras rework of atomics and fallbacks. This solves two problems: 1) Compilers uninline small atomic_* static inline functions which can expose them to instrumentation. 2) The instrumentation of atomic primitives was done at the architecture level while composites or fallbacks were provided at the generic level. As a result there are no uninstrumented variants of the fallbacks. Both issues were in the way of fully isolating fragile entry code pathes and especially the text poke int3 handler which is prone to an endless recursion problem when anything in that code path is about to be instrumented. This was always a problem, but got elevated due to the new batch mode updates of tracing. The solution is to mark the functions __always_inline and to flip the fallback and instrumentation so the non-instrumented variants are at the architecture level and the instrumentation is done in generic code. The latter introduces another fallback variant which will go away once all architectures have been moved over to arch_atomic_*" * tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/atomics: Flip fallbacks and instrumentation asm-generic/atomic: Use __always_inline for fallback wrappers

Linus Torvalds committed 6 years ago

Commit 9716e572 parents b1a6274 37f817328 files · +2594 −269Browse files

Unified · Split

28 files
  1. arch/arm64/include/asm/atomic.h +3 −3
  2. arch/x86/include/asm/atomic.h +13 −4
  3. arch/x86/include/asm/atomic64_32.h +9 −0
  4. arch/x86/include/asm/atomic64_64.h +12 −3
  5. include/linux/atomic-arch-fallback.h +2291 −0
  6. include/linux/atomic-fallback.h +171 −175
  7. include/linux/atomic.h +11 −0
  8. scripts/atomic/fallbacks/acquire +3 −3
  9. scripts/atomic/fallbacks/add_negative +4 −4
  10. scripts/atomic/fallbacks/add_unless +4 −4
  11. scripts/atomic/fallbacks/andnot +3 −3
  12. scripts/atomic/fallbacks/dec +3 −3
  13. scripts/atomic/fallbacks/dec_and_test +4 −4
  14. scripts/atomic/fallbacks/dec_if_positive +4 −4
  15. scripts/atomic/fallbacks/dec_unless_positive +4 −4
  16. scripts/atomic/fallbacks/fence +3 −3
  17. scripts/atomic/fallbacks/fetch_add_unless +5 −5
  18. scripts/atomic/fallbacks/inc +3 −3
  19. scripts/atomic/fallbacks/inc_and_test +4 −4
  20. scripts/atomic/fallbacks/inc_not_zero +4 −4
  21. scripts/atomic/fallbacks/inc_unless_negative +4 −4
  22. scripts/atomic/fallbacks/read_acquire +2 −2
  23. scripts/atomic/fallbacks/release +3 −3
  24. scripts/atomic/fallbacks/set_release +2 −2
  25. scripts/atomic/fallbacks/sub_and_test +4 −4
  26. scripts/atomic/fallbacks/try_cmpxchg +3 −3
  27. scripts/atomic/gen-atomic-fallback.sh +15 −16
  28. scripts/atomic/gen-atomics.sh +3 −2
arch/arm64/include/asm/atomic.h+3−3
@@ -101,8 +101,8 @@ static inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 100 unmodified lines
101101
102102 #define ATOMIC_INIT(i) { (i) }
103103
104−#define arch_atomic_read(v) READ_ONCE((v)->counter)
105−#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
104+#define arch_atomic_read(v) __READ_ONCE((v)->counter)
105+#define arch_atomic_set(v, i) __WRITE_ONCE(((v)->counter), (i))
106106
107107 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
108108 #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
@@ -225,6 +225,6 @@ static inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 116 unmodified lines
225225
226226 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
227227
228−#include <asm-generic/atomic-instrumented.h>
228+#define ARCH_ATOMIC
229229
230230 #endif /* __ASM_ATOMIC_H */
arch/x86/include/asm/atomic.h+13−4
@@ -28,7 +28,7 @@ static __always_inline int arch_atomic_read(const atomic_t *v) · 27 unmodified lines
2828 * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here,
2929 * it's non-inlined function that increases binary size and stack usage.
3030 */
31− return READ_ONCE((v)->counter);
31+ return __READ_ONCE((v)->counter);
3232 }
3333
3434 /**
@@ -40,7 +40,7 @@ static __always_inline int arch_atomic_read(const atomic_t *v) · 5 unmodified lines
4040 */
4141 static __always_inline void arch_atomic_set(atomic_t *v, int i)
4242 {
43− WRITE_ONCE(v->counter, i);
43+ __WRITE_ONCE(v->counter, i);
4444 }
4545
4646 /**
@@ -166,6 +166,7 @@ static __always_inline int arch_atomic_add_return(int i, atomic_t *v) · 119 unmodified lines
166166 {
167167 return i + xadd(&v->counter, i);
168168 }
169+#define arch_atomic_add_return arch_atomic_add_return
169170
170171 /**
171172 * arch_atomic_sub_return - subtract integer and return
@@ -178,32 +179,37 @@ static __always_inline int arch_atomic_sub_return(int i, atomic_t *v) · 6 unmodified lines
178179 {
179180 return arch_atomic_add_return(-i, v);
180181 }
182+#define arch_atomic_sub_return arch_atomic_sub_return
181183
182184 static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)
183185 {
184186 return xadd(&v->counter, i);
185187 }
188+#define arch_atomic_fetch_add arch_atomic_fetch_add
186189
187190 static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v)
188191 {
189192 return xadd(&v->counter, -i);
190193 }
194+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
191195
192196 static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
193197 {
194198 return arch_cmpxchg(&v->counter, old, new);
195199 }
200+#define arch_atomic_cmpxchg arch_atomic_cmpxchg
196201
197−#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
198202 static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
199203 {
200204 return try_cmpxchg(&v->counter, old, new);
201205 }
206+#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
202207
203208 static inline int arch_atomic_xchg(atomic_t *v, int new)
204209 {
205210 return arch_xchg(&v->counter, new);
206211 }
212+#define arch_atomic_xchg arch_atomic_xchg
207213
208214 static inline void arch_atomic_and(int i, atomic_t *v)
209215 {
@@ -221,6 +227,7 @@ static inline int arch_atomic_fetch_and(int i, atomic_t *v) · 11 unmodified lines
221227
222228 return val;
223229 }
230+#define arch_atomic_fetch_and arch_atomic_fetch_and
224231
225232 static inline void arch_atomic_or(int i, atomic_t *v)
226233 {
@@ -238,6 +245,7 @@ static inline int arch_atomic_fetch_or(int i, atomic_t *v) · 11 unmodified lines
238245
239246 return val;
240247 }
248+#define arch_atomic_fetch_or arch_atomic_fetch_or
241249
242250 static inline void arch_atomic_xor(int i, atomic_t *v)
243251 {
@@ -255,6 +263,7 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) · 11 unmodified lines
255263
256264 return val;
257265 }
266+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
258267
259268 #ifdef CONFIG_X86_32
260269 # include <asm/atomic64_32.h>
@@ -262,6 +271,6 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) · 1 unmodified line
262271 # include <asm/atomic64_64.h>
263272 #endif
264273
265−#include <asm-generic/atomic-instrumented.h>
274+#define ARCH_ATOMIC
266275
267276 #endif /* _ASM_X86_ATOMIC_H */
arch/x86/include/asm/atomic64_32.h+9−0
@@ -75,6 +75,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) · 74 unmodified lines
7575 {
7676 return arch_cmpxchg64(&v->counter, o, n);
7777 }
78+#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
7879
7980 /**
8081 * arch_atomic64_xchg - xchg atomic64 variable
@@ -94,6 +95,7 @@ static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) · 13 unmodified lines
9495 : "memory");
9596 return o;
9697 }
98+#define arch_atomic64_xchg arch_atomic64_xchg
9799
98100 /**
99101 * arch_atomic64_set - set atomic64 variable
@@ -138,6 +140,7 @@ static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) · 38 unmodified lines
138140 ASM_NO_INPUT_CLOBBER("memory"));
139141 return i;
140142 }
143+#define arch_atomic64_add_return arch_atomic64_add_return
141144
142145 /*
143146 * Other variants with different arithmetic operators:
@@ -149,6 +152,7 @@ static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) · 5 unmodified lines
149152 ASM_NO_INPUT_CLOBBER("memory"));
150153 return i;
151154 }
155+#define arch_atomic64_sub_return arch_atomic64_sub_return
152156
153157 static inline s64 arch_atomic64_inc_return(atomic64_t *v)
154158 {
@@ -242,6 +246,7 @@ static inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) · 87 unmodified lines
242246 "S" (v) : "memory");
243247 return (int)a;
244248 }
249+#define arch_atomic64_add_unless arch_atomic64_add_unless
245250
246251 static inline int arch_atomic64_inc_not_zero(atomic64_t *v)
247252 {
@@ -281,6 +286,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) · 33 unmodified lines
281286
282287 return old;
283288 }
289+#define arch_atomic64_fetch_and arch_atomic64_fetch_and
284290
285291 static inline void arch_atomic64_or(s64 i, atomic64_t *v)
286292 {
@@ -299,6 +305,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) · 12 unmodified lines
299305
300306 return old;
301307 }
308+#define arch_atomic64_fetch_or arch_atomic64_fetch_or
302309
303310 static inline void arch_atomic64_xor(s64 i, atomic64_t *v)
304311 {
@@ -317,6 +324,7 @@ static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) · 12 unmodified lines
317324
318325 return old;
319326 }
327+#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
320328
321329 static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
322330 {
@@ -327,6 +335,7 @@ static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) · 4 unmodified lines
327335
328336 return old;
329337 }
338+#define arch_atomic64_fetch_add arch_atomic64_fetch_add
330339
331340 #define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v))
332341
arch/x86/include/asm/atomic64_64.h+12−3
@@ -19,7 +19,7 @@ · 18 unmodified lines
1919 */
2020 static inline s64 arch_atomic64_read(const atomic64_t *v)
2121 {
22− return READ_ONCE((v)->counter);
22+ return __READ_ONCE((v)->counter);
2323 }
2424
2525 /**
@@ -31,7 +31,7 @@ static inline s64 arch_atomic64_read(const atomic64_t *v) · 5 unmodified lines
3131 */
3232 static inline void arch_atomic64_set(atomic64_t *v, s64 i)
3333 {
34− WRITE_ONCE(v->counter, i);
34+ __WRITE_ONCE(v->counter, i);
3535 }
3636
3737 /**
@@ -159,37 +159,43 @@ static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) · 121 unmodified lines
159159 {
160160 return i + xadd(&v->counter, i);
161161 }
162+#define arch_atomic64_add_return arch_atomic64_add_return
162163
163164 static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
164165 {
165166 return arch_atomic64_add_return(-i, v);
166167 }
168+#define arch_atomic64_sub_return arch_atomic64_sub_return
167169
168170 static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
169171 {
170172 return xadd(&v->counter, i);
171173 }
174+#define arch_atomic64_fetch_add arch_atomic64_fetch_add
172175
173176 static inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
174177 {
175178 return xadd(&v->counter, -i);
176179 }
180+#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
177181
178182 static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
179183 {
180184 return arch_cmpxchg(&v->counter, old, new);
181185 }
186+#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
182187
183−#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
184188 static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
185189 {
186190 return try_cmpxchg(&v->counter, old, new);
187191 }
192+#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
188193
189194 static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)
190195 {
191196 return arch_xchg(&v->counter, new);
192197 }
198+#define arch_atomic64_xchg arch_atomic64_xchg
193199
194200 static inline void arch_atomic64_and(s64 i, atomic64_t *v)
195201 {
@@ -207,6 +213,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) · 11 unmodified lines
207213 } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
208214 return val;
209215 }
216+#define arch_atomic64_fetch_and arch_atomic64_fetch_and
210217
211218 static inline void arch_atomic64_or(s64 i, atomic64_t *v)
212219 {
@@ -224,6 +231,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) · 11 unmodified lines
224231 } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
225232 return val;
226233 }
234+#define arch_atomic64_fetch_or arch_atomic64_fetch_or
227235
228236 static inline void arch_atomic64_xor(s64 i, atomic64_t *v)
229237 {
@@ -241,5 +249,6 @@ static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) · 11 unmodified lines
241249 } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
242250 return val;
243251 }
252+#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
244253
245254 #endif /* _ASM_X86_ATOMIC64_64_H */
include/linux/atomic-arch-fallback.h+2291−0
@@ -0,0 +1,2291 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+// Generated by scripts/atomic/gen-atomic-fallback.sh
4+// DO NOT MODIFY THIS FILE DIRECTLY
5+
6+#ifndef _LINUX_ATOMIC_FALLBACK_H
7+#define _LINUX_ATOMIC_FALLBACK_H
8+
9+#include <linux/compiler.h>
10+
11+#ifndef arch_xchg_relaxed
12+#define arch_xchg_relaxed arch_xchg
13+#define arch_xchg_acquire arch_xchg
14+#define arch_xchg_release arch_xchg
15+#else /* arch_xchg_relaxed */
16+
17+#ifndef arch_xchg_acquire
18+#define arch_xchg_acquire(...) \
19+ __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20+#endif
21+
22+#ifndef arch_xchg_release
23+#define arch_xchg_release(...) \
24+ __atomic_op_release(arch_xchg, __VA_ARGS__)
25+#endif
26+
27+#ifndef arch_xchg
28+#define arch_xchg(...) \
29+ __atomic_op_fence(arch_xchg, __VA_ARGS__)
30+#endif
31+
32+#endif /* arch_xchg_relaxed */
33+
34+#ifndef arch_cmpxchg_relaxed
35+#define arch_cmpxchg_relaxed arch_cmpxchg
36+#define arch_cmpxchg_acquire arch_cmpxchg
37+#define arch_cmpxchg_release arch_cmpxchg
38+#else /* arch_cmpxchg_relaxed */
39+
40+#ifndef arch_cmpxchg_acquire
41+#define arch_cmpxchg_acquire(...) \
42+ __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
43+#endif
44+
45+#ifndef arch_cmpxchg_release
46+#define arch_cmpxchg_release(...) \
47+ __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
48+#endif
49+
50+#ifndef arch_cmpxchg
51+#define arch_cmpxchg(...) \
52+ __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53+#endif
54+
55+#endif /* arch_cmpxchg_relaxed */
56+
57+#ifndef arch_cmpxchg64_relaxed
58+#define arch_cmpxchg64_relaxed arch_cmpxchg64
59+#define arch_cmpxchg64_acquire arch_cmpxchg64
60+#define arch_cmpxchg64_release arch_cmpxchg64
61+#else /* arch_cmpxchg64_relaxed */
62+
63+#ifndef arch_cmpxchg64_acquire
64+#define arch_cmpxchg64_acquire(...) \
65+ __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
66+#endif
67+
68+#ifndef arch_cmpxchg64_release
69+#define arch_cmpxchg64_release(...) \
70+ __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
71+#endif
72+
73+#ifndef arch_cmpxchg64
74+#define arch_cmpxchg64(...) \
75+ __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
76+#endif
77+
78+#endif /* arch_cmpxchg64_relaxed */
79+
80+#ifndef arch_atomic_read_acquire
81+static __always_inline int
82+arch_atomic_read_acquire(const atomic_t *v)
83+{
84+ return smp_load_acquire(&(v)->counter);
85+}
86+#define arch_atomic_read_acquire arch_atomic_read_acquire
87+#endif
88+
89+#ifndef arch_atomic_set_release
90+static __always_inline void
91+arch_atomic_set_release(atomic_t *v, int i)
92+{
93+ smp_store_release(&(v)->counter, i);
94+}
95+#define arch_atomic_set_release arch_atomic_set_release
96+#endif
97+
98+#ifndef arch_atomic_add_return_relaxed
99+#define arch_atomic_add_return_acquire arch_atomic_add_return
100+#define arch_atomic_add_return_release arch_atomic_add_return
101+#define arch_atomic_add_return_relaxed arch_atomic_add_return
102+#else /* arch_atomic_add_return_relaxed */
103+
104+#ifndef arch_atomic_add_return_acquire
105+static __always_inline int
106+arch_atomic_add_return_acquire(int i, atomic_t *v)
107+{
108+ int ret = arch_atomic_add_return_relaxed(i, v);
109+ __atomic_acquire_fence();
110+ return ret;
111+}
112+#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
113+#endif
114+
115+#ifndef arch_atomic_add_return_release
116+static __always_inline int
117+arch_atomic_add_return_release(int i, atomic_t *v)
118+{
119+ __atomic_release_fence();
120+ return arch_atomic_add_return_relaxed(i, v);
121+}
122+#define arch_atomic_add_return_release arch_atomic_add_return_release
123+#endif
124+
125+#ifndef arch_atomic_add_return
126+static __always_inline int
127+arch_atomic_add_return(int i, atomic_t *v)
128+{
129+ int ret;
130+ __atomic_pre_full_fence();
131+ ret = arch_atomic_add_return_relaxed(i, v);
132+ __atomic_post_full_fence();
133+ return ret;
134+}
135+#define arch_atomic_add_return arch_atomic_add_return
136+#endif
137+
138+#endif /* arch_atomic_add_return_relaxed */
139+
140+#ifndef arch_atomic_fetch_add_relaxed
141+#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
142+#define arch_atomic_fetch_add_release arch_atomic_fetch_add
143+#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
144+#else /* arch_atomic_fetch_add_relaxed */
145+
146+#ifndef arch_atomic_fetch_add_acquire
147+static __always_inline int
148+arch_atomic_fetch_add_acquire(int i, atomic_t *v)
149+{
150+ int ret = arch_atomic_fetch_add_relaxed(i, v);
151+ __atomic_acquire_fence();
152+ return ret;
153+}
154+#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
155+#endif
156+
157+#ifndef arch_atomic_fetch_add_release
158+static __always_inline int
159+arch_atomic_fetch_add_release(int i, atomic_t *v)
160+{
161+ __atomic_release_fence();
162+ return arch_atomic_fetch_add_relaxed(i, v);
163+}
164+#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
165+#endif
166+
167+#ifndef arch_atomic_fetch_add
168+static __always_inline int
169+arch_atomic_fetch_add(int i, atomic_t *v)
170+{
171+ int ret;
172+ __atomic_pre_full_fence();
173+ ret = arch_atomic_fetch_add_relaxed(i, v);
174+ __atomic_post_full_fence();
175+ return ret;
176+}
177+#define arch_atomic_fetch_add arch_atomic_fetch_add
178+#endif
179+
180+#endif /* arch_atomic_fetch_add_relaxed */
181+
182+#ifndef arch_atomic_sub_return_relaxed
183+#define arch_atomic_sub_return_acquire arch_atomic_sub_return
184+#define arch_atomic_sub_return_release arch_atomic_sub_return
185+#define arch_atomic_sub_return_relaxed arch_atomic_sub_return
186+#else /* arch_atomic_sub_return_relaxed */
187+
188+#ifndef arch_atomic_sub_return_acquire
189+static __always_inline int
190+arch_atomic_sub_return_acquire(int i, atomic_t *v)
191+{
192+ int ret = arch_atomic_sub_return_relaxed(i, v);
193+ __atomic_acquire_fence();
194+ return ret;
195+}
196+#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
197+#endif
198+
199+#ifndef arch_atomic_sub_return_release
200+static __always_inline int
201+arch_atomic_sub_return_release(int i, atomic_t *v)
202+{
203+ __atomic_release_fence();
204+ return arch_atomic_sub_return_relaxed(i, v);
205+}
206+#define arch_atomic_sub_return_release arch_atomic_sub_return_release
207+#endif
208+
209+#ifndef arch_atomic_sub_return
210+static __always_inline int
211+arch_atomic_sub_return(int i, atomic_t *v)
212+{
213+ int ret;
214+ __atomic_pre_full_fence();
215+ ret = arch_atomic_sub_return_relaxed(i, v);
216+ __atomic_post_full_fence();
217+ return ret;
218+}
219+#define arch_atomic_sub_return arch_atomic_sub_return
220+#endif
221+
222+#endif /* arch_atomic_sub_return_relaxed */
223+
224+#ifndef arch_atomic_fetch_sub_relaxed
225+#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
226+#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
227+#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
228+#else /* arch_atomic_fetch_sub_relaxed */
229+
230+#ifndef arch_atomic_fetch_sub_acquire
231+static __always_inline int
232+arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
233+{
234+ int ret = arch_atomic_fetch_sub_relaxed(i, v);
235+ __atomic_acquire_fence();
236+ return ret;
237+}
238+#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
239+#endif
240+
241+#ifndef arch_atomic_fetch_sub_release
242+static __always_inline int
243+arch_atomic_fetch_sub_release(int i, atomic_t *v)
244+{
245+ __atomic_release_fence();
246+ return arch_atomic_fetch_sub_relaxed(i, v);
247+}
248+#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
249+#endif
250+
251+#ifndef arch_atomic_fetch_sub
252+static __always_inline int
253+arch_atomic_fetch_sub(int i, atomic_t *v)
254+{
255+ int ret;
256+ __atomic_pre_full_fence();
257+ ret = arch_atomic_fetch_sub_relaxed(i, v);
258+ __atomic_post_full_fence();
259+ return ret;
260+}
261+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
262+#endif
263+
264+#endif /* arch_atomic_fetch_sub_relaxed */
265+
266+#ifndef arch_atomic_inc
267+static __always_inline void
268+arch_atomic_inc(atomic_t *v)
269+{
270+ arch_atomic_add(1, v);
271+}
272+#define arch_atomic_inc arch_atomic_inc
273+#endif
274+
275+#ifndef arch_atomic_inc_return_relaxed
276+#ifdef arch_atomic_inc_return
277+#define arch_atomic_inc_return_acquire arch_atomic_inc_return
278+#define arch_atomic_inc_return_release arch_atomic_inc_return
279+#define arch_atomic_inc_return_relaxed arch_atomic_inc_return
280+#endif /* arch_atomic_inc_return */
281+
282+#ifndef arch_atomic_inc_return
283+static __always_inline int
284+arch_atomic_inc_return(atomic_t *v)
285+{
286+ return arch_atomic_add_return(1, v);
287+}
288+#define arch_atomic_inc_return arch_atomic_inc_return
289+#endif
290+
291+#ifndef arch_atomic_inc_return_acquire
292+static __always_inline int
293+arch_atomic_inc_return_acquire(atomic_t *v)
294+{
295+ return arch_atomic_add_return_acquire(1, v);
296+}
297+#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
298+#endif
299+
300+#ifndef arch_atomic_inc_return_release
301+static __always_inline int
302+arch_atomic_inc_return_release(atomic_t *v)
303+{
304+ return arch_atomic_add_return_release(1, v);
305+}
306+#define arch_atomic_inc_return_release arch_atomic_inc_return_release
307+#endif
308+
309+#ifndef arch_atomic_inc_return_relaxed
310+static __always_inline int
311+arch_atomic_inc_return_relaxed(atomic_t *v)
312+{
313+ return arch_atomic_add_return_relaxed(1, v);
314+}
315+#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
316+#endif
317+
318+#else /* arch_atomic_inc_return_relaxed */
319+
320+#ifndef arch_atomic_inc_return_acquire
321+static __always_inline int
322+arch_atomic_inc_return_acquire(atomic_t *v)
323+{
324+ int ret = arch_atomic_inc_return_relaxed(v);
325+ __atomic_acquire_fence();
326+ return ret;
327+}
328+#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
329+#endif
330+
331+#ifndef arch_atomic_inc_return_release
332+static __always_inline int
333+arch_atomic_inc_return_release(atomic_t *v)
334+{
335+ __atomic_release_fence();
336+ return arch_atomic_inc_return_relaxed(v);
337+}
338+#define arch_atomic_inc_return_release arch_atomic_inc_return_release
339+#endif
340+
341+#ifndef arch_atomic_inc_return
342+static __always_inline int
343+arch_atomic_inc_return(atomic_t *v)
344+{
345+ int ret;
346+ __atomic_pre_full_fence();
347+ ret = arch_atomic_inc_return_relaxed(v);
348+ __atomic_post_full_fence();
349+ return ret;
350+}
351+#define arch_atomic_inc_return arch_atomic_inc_return
352+#endif
353+
354+#endif /* arch_atomic_inc_return_relaxed */
355+
356+#ifndef arch_atomic_fetch_inc_relaxed
357+#ifdef arch_atomic_fetch_inc
358+#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
359+#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
360+#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
361+#endif /* arch_atomic_fetch_inc */
362+
363+#ifndef arch_atomic_fetch_inc
364+static __always_inline int
365+arch_atomic_fetch_inc(atomic_t *v)
366+{
367+ return arch_atomic_fetch_add(1, v);
368+}
369+#define arch_atomic_fetch_inc arch_atomic_fetch_inc
370+#endif
371+
372+#ifndef arch_atomic_fetch_inc_acquire
373+static __always_inline int
374+arch_atomic_fetch_inc_acquire(atomic_t *v)
375+{
376+ return arch_atomic_fetch_add_acquire(1, v);
377+}
378+#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
379+#endif
380+
381+#ifndef arch_atomic_fetch_inc_release
382+static __always_inline int
383+arch_atomic_fetch_inc_release(atomic_t *v)
384+{
385+ return arch_atomic_fetch_add_release(1, v);
386+}
387+#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
388+#endif
389+
390+#ifndef arch_atomic_fetch_inc_relaxed
391+static __always_inline int
392+arch_atomic_fetch_inc_relaxed(atomic_t *v)
393+{
394+ return arch_atomic_fetch_add_relaxed(1, v);
395+}
396+#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
397+#endif
398+
399+#else /* arch_atomic_fetch_inc_relaxed */
400+
401+#ifndef arch_atomic_fetch_inc_acquire
402+static __always_inline int
403+arch_atomic_fetch_inc_acquire(atomic_t *v)
404+{
405+ int ret = arch_atomic_fetch_inc_relaxed(v);
406+ __atomic_acquire_fence();
407+ return ret;
408+}
409+#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
410+#endif
411+
412+#ifndef arch_atomic_fetch_inc_release
413+static __always_inline int
414+arch_atomic_fetch_inc_release(atomic_t *v)
415+{
416+ __atomic_release_fence();
417+ return arch_atomic_fetch_inc_relaxed(v);
418+}
419+#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
420+#endif
421+
422+#ifndef arch_atomic_fetch_inc
423+static __always_inline int
424+arch_atomic_fetch_inc(atomic_t *v)
425+{
426+ int ret;
427+ __atomic_pre_full_fence();
428+ ret = arch_atomic_fetch_inc_relaxed(v);
429+ __atomic_post_full_fence();
430+ return ret;
431+}
432+#define arch_atomic_fetch_inc arch_atomic_fetch_inc
433+#endif
434+
435+#endif /* arch_atomic_fetch_inc_relaxed */
436+
437+#ifndef arch_atomic_dec
438+static __always_inline void
439+arch_atomic_dec(atomic_t *v)
440+{
441+ arch_atomic_sub(1, v);
442+}
443+#define arch_atomic_dec arch_atomic_dec
444+#endif
445+
446+#ifndef arch_atomic_dec_return_relaxed
447+#ifdef arch_atomic_dec_return
448+#define arch_atomic_dec_return_acquire arch_atomic_dec_return
449+#define arch_atomic_dec_return_release arch_atomic_dec_return
450+#define arch_atomic_dec_return_relaxed arch_atomic_dec_return
451+#endif /* arch_atomic_dec_return */
452+
453+#ifndef arch_atomic_dec_return
454+static __always_inline int
455+arch_atomic_dec_return(atomic_t *v)
456+{
457+ return arch_atomic_sub_return(1, v);
458+}
459+#define arch_atomic_dec_return arch_atomic_dec_return
460+#endif
461+
462+#ifndef arch_atomic_dec_return_acquire
463+static __always_inline int
464+arch_atomic_dec_return_acquire(atomic_t *v)
465+{
466+ return arch_atomic_sub_return_acquire(1, v);
467+}
468+#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
469+#endif
470+
471+#ifndef arch_atomic_dec_return_release
472+static __always_inline int
473+arch_atomic_dec_return_release(atomic_t *v)
474+{
475+ return arch_atomic_sub_return_release(1, v);
476+}
477+#define arch_atomic_dec_return_release arch_atomic_dec_return_release
478+#endif
479+
480+#ifndef arch_atomic_dec_return_relaxed
481+static __always_inline int
482+arch_atomic_dec_return_relaxed(atomic_t *v)
483+{
484+ return arch_atomic_sub_return_relaxed(1, v);
485+}
486+#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
487+#endif
488+
489+#else /* arch_atomic_dec_return_relaxed */
490+
491+#ifndef arch_atomic_dec_return_acquire
492+static __always_inline int
493+arch_atomic_dec_return_acquire(atomic_t *v)
494+{
495+ int ret = arch_atomic_dec_return_relaxed(v);
496+ __atomic_acquire_fence();
497+ return ret;
498+}
499+#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
500+#endif
501+
502+#ifndef arch_atomic_dec_return_release
503+static __always_inline int
504+arch_atomic_dec_return_release(atomic_t *v)
505+{
506+ __atomic_release_fence();
507+ return arch_atomic_dec_return_relaxed(v);
508+}
509+#define arch_atomic_dec_return_release arch_atomic_dec_return_release
510+#endif
511+
512+#ifndef arch_atomic_dec_return
513+static __always_inline int
514+arch_atomic_dec_return(atomic_t *v)
515+{
516+ int ret;
517+ __atomic_pre_full_fence();
518+ ret = arch_atomic_dec_return_relaxed(v);
519+ __atomic_post_full_fence();
520+ return ret;
521+}
522+#define arch_atomic_dec_return arch_atomic_dec_return
523+#endif
524+
525+#endif /* arch_atomic_dec_return_relaxed */
526+
527+#ifndef arch_atomic_fetch_dec_relaxed
528+#ifdef arch_atomic_fetch_dec
529+#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
530+#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
531+#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
532+#endif /* arch_atomic_fetch_dec */
533+
534+#ifndef arch_atomic_fetch_dec
535+static __always_inline int
536+arch_atomic_fetch_dec(atomic_t *v)
537+{
538+ return arch_atomic_fetch_sub(1, v);
539+}
540+#define arch_atomic_fetch_dec arch_atomic_fetch_dec
541+#endif
542+
543+#ifndef arch_atomic_fetch_dec_acquire
544+static __always_inline int
545+arch_atomic_fetch_dec_acquire(atomic_t *v)
546+{
547+ return arch_atomic_fetch_sub_acquire(1, v);
548+}
549+#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
550+#endif
551+
552+#ifndef arch_atomic_fetch_dec_release
553+static __always_inline int
554+arch_atomic_fetch_dec_release(atomic_t *v)
555+{
556+ return arch_atomic_fetch_sub_release(1, v);
557+}
558+#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
559+#endif
560+
561+#ifndef arch_atomic_fetch_dec_relaxed
562+static __always_inline int
563+arch_atomic_fetch_dec_relaxed(atomic_t *v)
564+{
565+ return arch_atomic_fetch_sub_relaxed(1, v);
566+}
567+#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
568+#endif
569+
570+#else /* arch_atomic_fetch_dec_relaxed */
571+
572+#ifndef arch_atomic_fetch_dec_acquire
573+static __always_inline int
574+arch_atomic_fetch_dec_acquire(atomic_t *v)
575+{
576+ int ret = arch_atomic_fetch_dec_relaxed(v);
577+ __atomic_acquire_fence();
578+ return ret;
579+}
580+#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
581+#endif
582+
583+#ifndef arch_atomic_fetch_dec_release
584+static __always_inline int
585+arch_atomic_fetch_dec_release(atomic_t *v)
586+{
587+ __atomic_release_fence();
588+ return arch_atomic_fetch_dec_relaxed(v);
589+}
590+#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
591+#endif
592+
593+#ifndef arch_atomic_fetch_dec
594+static __always_inline int
595+arch_atomic_fetch_dec(atomic_t *v)
596+{
597+ int ret;
598+ __atomic_pre_full_fence();
599+ ret = arch_atomic_fetch_dec_relaxed(v);
600+ __atomic_post_full_fence();
601+ return ret;
602+}
603+#define arch_atomic_fetch_dec arch_atomic_fetch_dec
604+#endif
605+
606+#endif /* arch_atomic_fetch_dec_relaxed */
607+
608+#ifndef arch_atomic_fetch_and_relaxed
609+#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
610+#define arch_atomic_fetch_and_release arch_atomic_fetch_and
611+#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
612+#else /* arch_atomic_fetch_and_relaxed */
613+
614+#ifndef arch_atomic_fetch_and_acquire
615+static __always_inline int
616+arch_atomic_fetch_and_acquire(int i, atomic_t *v)
617+{
618+ int ret = arch_atomic_fetch_and_relaxed(i, v);
619+ __atomic_acquire_fence();
620+ return ret;
621+}
622+#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
623+#endif
624+
625+#ifndef arch_atomic_fetch_and_release
626+static __always_inline int
627+arch_atomic_fetch_and_release(int i, atomic_t *v)
628+{
629+ __atomic_release_fence();
630+ return arch_atomic_fetch_and_relaxed(i, v);
631+}
632+#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
633+#endif
634+
635+#ifndef arch_atomic_fetch_and
636+static __always_inline int
637+arch_atomic_fetch_and(int i, atomic_t *v)
638+{
639+ int ret;
640+ __atomic_pre_full_fence();
641+ ret = arch_atomic_fetch_and_relaxed(i, v);
642+ __atomic_post_full_fence();
643+ return ret;
644+}
645+#define arch_atomic_fetch_and arch_atomic_fetch_and
646+#endif
647+
648+#endif /* arch_atomic_fetch_and_relaxed */
649+
650+#ifndef arch_atomic_andnot
651+static __always_inline void
652+arch_atomic_andnot(int i, atomic_t *v)
653+{
654+ arch_atomic_and(~i, v);
655+}
656+#define arch_atomic_andnot arch_atomic_andnot
657+#endif
658+
659+#ifndef arch_atomic_fetch_andnot_relaxed
660+#ifdef arch_atomic_fetch_andnot
661+#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
662+#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
663+#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
664+#endif /* arch_atomic_fetch_andnot */
665+
666+#ifndef arch_atomic_fetch_andnot
667+static __always_inline int
668+arch_atomic_fetch_andnot(int i, atomic_t *v)
669+{
670+ return arch_atomic_fetch_and(~i, v);
671+}
672+#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
673+#endif
674+
675+#ifndef arch_atomic_fetch_andnot_acquire
676+static __always_inline int
677+arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
678+{
679+ return arch_atomic_fetch_and_acquire(~i, v);
680+}
681+#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
682+#endif
683+
684+#ifndef arch_atomic_fetch_andnot_release
685+static __always_inline int
686+arch_atomic_fetch_andnot_release(int i, atomic_t *v)
687+{
688+ return arch_atomic_fetch_and_release(~i, v);
689+}
690+#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
691+#endif
692+
693+#ifndef arch_atomic_fetch_andnot_relaxed
694+static __always_inline int
695+arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
696+{
697+ return arch_atomic_fetch_and_relaxed(~i, v);
698+}
699+#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
700+#endif
701+
702+#else /* arch_atomic_fetch_andnot_relaxed */
703+
704+#ifndef arch_atomic_fetch_andnot_acquire
705+static __always_inline int
706+arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
707+{
708+ int ret = arch_atomic_fetch_andnot_relaxed(i, v);
709+ __atomic_acquire_fence();
710+ return ret;
711+}
712+#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
713+#endif
714+
715+#ifndef arch_atomic_fetch_andnot_release
716+static __always_inline int
717+arch_atomic_fetch_andnot_release(int i, atomic_t *v)
718+{
719+ __atomic_release_fence();
720+ return arch_atomic_fetch_andnot_relaxed(i, v);
721+}
722+#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
723+#endif
724+
725+#ifndef arch_atomic_fetch_andnot
726+static __always_inline int
727+arch_atomic_fetch_andnot(int i, atomic_t *v)
728+{
729+ int ret;
730+ __atomic_pre_full_fence();
731+ ret = arch_atomic_fetch_andnot_relaxed(i, v);
732+ __atomic_post_full_fence();
733+ return ret;
734+}
735+#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
736+#endif
737+
738+#endif /* arch_atomic_fetch_andnot_relaxed */
739+
740+#ifndef arch_atomic_fetch_or_relaxed
741+#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
742+#define arch_atomic_fetch_or_release arch_atomic_fetch_or
743+#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
744+#else /* arch_atomic_fetch_or_relaxed */
745+
746+#ifndef arch_atomic_fetch_or_acquire
747+static __always_inline int
748+arch_atomic_fetch_or_acquire(int i, atomic_t *v)
749+{
750+ int ret = arch_atomic_fetch_or_relaxed(i, v);
751+ __atomic_acquire_fence();
752+ return ret;
753+}
754+#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
755+#endif
756+
757+#ifndef arch_atomic_fetch_or_release
758+static __always_inline int
759+arch_atomic_fetch_or_release(int i, atomic_t *v)
760+{
761+ __atomic_release_fence();
762+ return arch_atomic_fetch_or_relaxed(i, v);
763+}
764+#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
765+#endif
766+
767+#ifndef arch_atomic_fetch_or
768+static __always_inline int
769+arch_atomic_fetch_or(int i, atomic_t *v)
770+{
771+ int ret;
772+ __atomic_pre_full_fence();
773+ ret = arch_atomic_fetch_or_relaxed(i, v);
774+ __atomic_post_full_fence();
775+ return ret;
776+}
777+#define arch_atomic_fetch_or arch_atomic_fetch_or
778+#endif
779+
780+#endif /* arch_atomic_fetch_or_relaxed */
781+
782+#ifndef arch_atomic_fetch_xor_relaxed
783+#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
784+#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
785+#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
786+#else /* arch_atomic_fetch_xor_relaxed */
787+
788+#ifndef arch_atomic_fetch_xor_acquire
789+static __always_inline int
790+arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
791+{
792+ int ret = arch_atomic_fetch_xor_relaxed(i, v);
793+ __atomic_acquire_fence();
794+ return ret;
795+}
796+#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
797+#endif
798+
799+#ifndef arch_atomic_fetch_xor_release
800+static __always_inline int
801+arch_atomic_fetch_xor_release(int i, atomic_t *v)
802+{
803+ __atomic_release_fence();
804+ return arch_atomic_fetch_xor_relaxed(i, v);
805+}
806+#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
807+#endif
808+
809+#ifndef arch_atomic_fetch_xor
810+static __always_inline int
811+arch_atomic_fetch_xor(int i, atomic_t *v)
812+{
813+ int ret;
814+ __atomic_pre_full_fence();
815+ ret = arch_atomic_fetch_xor_relaxed(i, v);
816+ __atomic_post_full_fence();
817+ return ret;
818+}
819+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
820+#endif
821+
822+#endif /* arch_atomic_fetch_xor_relaxed */
823+
824+#ifndef arch_atomic_xchg_relaxed
825+#define arch_atomic_xchg_acquire arch_atomic_xchg
826+#define arch_atomic_xchg_release arch_atomic_xchg
827+#define arch_atomic_xchg_relaxed arch_atomic_xchg
828+#else /* arch_atomic_xchg_relaxed */
829+
830+#ifndef arch_atomic_xchg_acquire
831+static __always_inline int
832+arch_atomic_xchg_acquire(atomic_t *v, int i)
833+{
834+ int ret = arch_atomic_xchg_relaxed(v, i);
835+ __atomic_acquire_fence();
836+ return ret;
837+}
838+#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
839+#endif
840+
841+#ifndef arch_atomic_xchg_release
842+static __always_inline int
843+arch_atomic_xchg_release(atomic_t *v, int i)
844+{
845+ __atomic_release_fence();
846+ return arch_atomic_xchg_relaxed(v, i);
847+}
848+#define arch_atomic_xchg_release arch_atomic_xchg_release
849+#endif
850+
851+#ifndef arch_atomic_xchg
852+static __always_inline int
853+arch_atomic_xchg(atomic_t *v, int i)
854+{
855+ int ret;
856+ __atomic_pre_full_fence();
857+ ret = arch_atomic_xchg_relaxed(v, i);
858+ __atomic_post_full_fence();
859+ return ret;
860+}
861+#define arch_atomic_xchg arch_atomic_xchg
862+#endif
863+
864+#endif /* arch_atomic_xchg_relaxed */
865+
866+#ifndef arch_atomic_cmpxchg_relaxed
867+#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
868+#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
869+#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
870+#else /* arch_atomic_cmpxchg_relaxed */
871+
872+#ifndef arch_atomic_cmpxchg_acquire
873+static __always_inline int
874+arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
875+{
876+ int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
877+ __atomic_acquire_fence();
878+ return ret;
879+}
880+#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
881+#endif
882+
883+#ifndef arch_atomic_cmpxchg_release
884+static __always_inline int
885+arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
886+{
887+ __atomic_release_fence();
888+ return arch_atomic_cmpxchg_relaxed(v, old, new);
889+}
890+#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
891+#endif
892+
893+#ifndef arch_atomic_cmpxchg
894+static __always_inline int
895+arch_atomic_cmpxchg(atomic_t *v, int old, int new)
896+{
897+ int ret;
898+ __atomic_pre_full_fence();
899+ ret = arch_atomic_cmpxchg_relaxed(v, old, new);
900+ __atomic_post_full_fence();
901+ return ret;
902+}
903+#define arch_atomic_cmpxchg arch_atomic_cmpxchg
904+#endif
905+
906+#endif /* arch_atomic_cmpxchg_relaxed */
907+
908+#ifndef arch_atomic_try_cmpxchg_relaxed
909+#ifdef arch_atomic_try_cmpxchg
910+#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
911+#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
912+#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
913+#endif /* arch_atomic_try_cmpxchg */
914+
915+#ifndef arch_atomic_try_cmpxchg
916+static __always_inline bool
917+arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
918+{
919+ int r, o = *old;
920+ r = arch_atomic_cmpxchg(v, o, new);
921+ if (unlikely(r != o))
922+ *old = r;
923+ return likely(r == o);
924+}
925+#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
926+#endif
927+
928+#ifndef arch_atomic_try_cmpxchg_acquire
929+static __always_inline bool
930+arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
931+{
932+ int r, o = *old;
933+ r = arch_atomic_cmpxchg_acquire(v, o, new);
934+ if (unlikely(r != o))
935+ *old = r;
936+ return likely(r == o);
937+}
938+#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
939+#endif
940+
941+#ifndef arch_atomic_try_cmpxchg_release
942+static __always_inline bool
943+arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
944+{
945+ int r, o = *old;
946+ r = arch_atomic_cmpxchg_release(v, o, new);
947+ if (unlikely(r != o))
948+ *old = r;
949+ return likely(r == o);
950+}
951+#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
952+#endif
953+
954+#ifndef arch_atomic_try_cmpxchg_relaxed
955+static __always_inline bool
956+arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
957+{
958+ int r, o = *old;
959+ r = arch_atomic_cmpxchg_relaxed(v, o, new);
960+ if (unlikely(r != o))
961+ *old = r;
962+ return likely(r == o);
963+}
964+#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
965+#endif
966+
967+#else /* arch_atomic_try_cmpxchg_relaxed */
968+
969+#ifndef arch_atomic_try_cmpxchg_acquire
970+static __always_inline bool
971+arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
972+{
973+ bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
974+ __atomic_acquire_fence();
975+ return ret;
976+}
977+#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
978+#endif
979+
980+#ifndef arch_atomic_try_cmpxchg_release
981+static __always_inline bool
982+arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
983+{
984+ __atomic_release_fence();
985+ return arch_atomic_try_cmpxchg_relaxed(v, old, new);
986+}
987+#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
988+#endif
989+
990+#ifndef arch_atomic_try_cmpxchg
991+static __always_inline bool
992+arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
993+{
994+ bool ret;
995+ __atomic_pre_full_fence();
996+ ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
997+ __atomic_post_full_fence();
998+ return ret;
999+}
1000+#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1001+#endif
1002+
1003+#endif /* arch_atomic_try_cmpxchg_relaxed */
1004+
1005+#ifndef arch_atomic_sub_and_test
1006+/**
1007+ * arch_atomic_sub_and_test - subtract value from variable and test result
1008+ * @i: integer value to subtract
1009+ * @v: pointer of type atomic_t
1010+ *
1011+ * Atomically subtracts @i from @v and returns
1012+ * true if the result is zero, or false for all
1013+ * other cases.
1014+ */
1015+static __always_inline bool
1016+arch_atomic_sub_and_test(int i, atomic_t *v)
1017+{
1018+ return arch_atomic_sub_return(i, v) == 0;
1019+}
1020+#define arch_atomic_sub_and_test arch_atomic_sub_and_test
1021+#endif
1022+
1023+#ifndef arch_atomic_dec_and_test
1024+/**
1025+ * arch_atomic_dec_and_test - decrement and test
1026+ * @v: pointer of type atomic_t
1027+ *
1028+ * Atomically decrements @v by 1 and
1029+ * returns true if the result is 0, or false for all other
1030+ * cases.
1031+ */
1032+static __always_inline bool
1033+arch_atomic_dec_and_test(atomic_t *v)
1034+{
1035+ return arch_atomic_dec_return(v) == 0;
1036+}
1037+#define arch_atomic_dec_and_test arch_atomic_dec_and_test
1038+#endif
1039+
1040+#ifndef arch_atomic_inc_and_test
1041+/**
1042+ * arch_atomic_inc_and_test - increment and test
1043+ * @v: pointer of type atomic_t
1044+ *
1045+ * Atomically increments @v by 1
1046+ * and returns true if the result is zero, or false for all
1047+ * other cases.
1048+ */
1049+static __always_inline bool
1050+arch_atomic_inc_and_test(atomic_t *v)
1051+{
1052+ return arch_atomic_inc_return(v) == 0;
1053+}
1054+#define arch_atomic_inc_and_test arch_atomic_inc_and_test
1055+#endif
1056+
1057+#ifndef arch_atomic_add_negative
1058+/**
1059+ * arch_atomic_add_negative - add and test if negative
1060+ * @i: integer value to add
1061+ * @v: pointer of type atomic_t
1062+ *
1063+ * Atomically adds @i to @v and returns true
1064+ * if the result is negative, or false when
1065+ * result is greater than or equal to zero.
1066+ */
1067+static __always_inline bool
1068+arch_atomic_add_negative(int i, atomic_t *v)
1069+{
1070+ return arch_atomic_add_return(i, v) < 0;
1071+}
1072+#define arch_atomic_add_negative arch_atomic_add_negative
1073+#endif
1074+
1075+#ifndef arch_atomic_fetch_add_unless
1076+/**
1077+ * arch_atomic_fetch_add_unless - add unless the number is already a given value
1078+ * @v: pointer of type atomic_t
1079+ * @a: the amount to add to v...
1080+ * @u: ...unless v is equal to u.
1081+ *
1082+ * Atomically adds @a to @v, so long as @v was not already @u.
1083+ * Returns original value of @v
1084+ */
1085+static __always_inline int
1086+arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1087+{
1088+ int c = arch_atomic_read(v);
1089+
1090+ do {
1091+ if (unlikely(c == u))
1092+ break;
1093+ } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1094+
1095+ return c;
1096+}
1097+#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1098+#endif
1099+
1100+#ifndef arch_atomic_add_unless
1101+/**
1102+ * arch_atomic_add_unless - add unless the number is already a given value
1103+ * @v: pointer of type atomic_t
1104+ * @a: the amount to add to v...
1105+ * @u: ...unless v is equal to u.
1106+ *
1107+ * Atomically adds @a to @v, if @v was not already @u.
1108+ * Returns true if the addition was done.
1109+ */
1110+static __always_inline bool
1111+arch_atomic_add_unless(atomic_t *v, int a, int u)
1112+{
1113+ return arch_atomic_fetch_add_unless(v, a, u) != u;
1114+}
1115+#define arch_atomic_add_unless arch_atomic_add_unless
1116+#endif
1117+
1118+#ifndef arch_atomic_inc_not_zero
1119+/**
1120+ * arch_atomic_inc_not_zero - increment unless the number is zero
1121+ * @v: pointer of type atomic_t
1122+ *
1123+ * Atomically increments @v by 1, if @v is non-zero.
1124+ * Returns true if the increment was done.
1125+ */
1126+static __always_inline bool
1127+arch_atomic_inc_not_zero(atomic_t *v)
1128+{
1129+ return arch_atomic_add_unless(v, 1, 0);
1130+}
1131+#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1132+#endif
1133+
1134+#ifndef arch_atomic_inc_unless_negative
1135+static __always_inline bool
1136+arch_atomic_inc_unless_negative(atomic_t *v)
1137+{
1138+ int c = arch_atomic_read(v);
1139+
1140+ do {
1141+ if (unlikely(c < 0))
1142+ return false;
1143+ } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1144+
1145+ return true;
1146+}
1147+#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1148+#endif
1149+
1150+#ifndef arch_atomic_dec_unless_positive
1151+static __always_inline bool
1152+arch_atomic_dec_unless_positive(atomic_t *v)
1153+{
1154+ int c = arch_atomic_read(v);
1155+
1156+ do {
1157+ if (unlikely(c > 0))
1158+ return false;
1159+ } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1160+
1161+ return true;
1162+}
1163+#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1164+#endif
1165+
1166+#ifndef arch_atomic_dec_if_positive
1167+static __always_inline int
1168+arch_atomic_dec_if_positive(atomic_t *v)
1169+{
1170+ int dec, c = arch_atomic_read(v);
1171+
1172+ do {
1173+ dec = c - 1;
1174+ if (unlikely(dec < 0))
1175+ break;
1176+ } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1177+
1178+ return dec;
1179+}
1180+#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1181+#endif
1182+
1183+#ifdef CONFIG_GENERIC_ATOMIC64
1184+#include <asm-generic/atomic64.h>
1185+#endif
1186+
1187+#ifndef arch_atomic64_read_acquire
1188+static __always_inline s64
1189+arch_atomic64_read_acquire(const atomic64_t *v)
1190+{
1191+ return smp_load_acquire(&(v)->counter);
1192+}
1193+#define arch_atomic64_read_acquire arch_atomic64_read_acquire
1194+#endif
1195+
1196+#ifndef arch_atomic64_set_release
1197+static __always_inline void
1198+arch_atomic64_set_release(atomic64_t *v, s64 i)
1199+{
1200+ smp_store_release(&(v)->counter, i);
1201+}
1202+#define arch_atomic64_set_release arch_atomic64_set_release
1203+#endif
1204+
1205+#ifndef arch_atomic64_add_return_relaxed
1206+#define arch_atomic64_add_return_acquire arch_atomic64_add_return
1207+#define arch_atomic64_add_return_release arch_atomic64_add_return
1208+#define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1209+#else /* arch_atomic64_add_return_relaxed */
1210+
1211+#ifndef arch_atomic64_add_return_acquire
1212+static __always_inline s64
1213+arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1214+{
1215+ s64 ret = arch_atomic64_add_return_relaxed(i, v);
1216+ __atomic_acquire_fence();
1217+ return ret;
1218+}
1219+#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1220+#endif
1221+
1222+#ifndef arch_atomic64_add_return_release
1223+static __always_inline s64
1224+arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1225+{
1226+ __atomic_release_fence();
1227+ return arch_atomic64_add_return_relaxed(i, v);
1228+}
1229+#define arch_atomic64_add_return_release arch_atomic64_add_return_release
1230+#endif
1231+
1232+#ifndef arch_atomic64_add_return
1233+static __always_inline s64
1234+arch_atomic64_add_return(s64 i, atomic64_t *v)
1235+{
1236+ s64 ret;
1237+ __atomic_pre_full_fence();
1238+ ret = arch_atomic64_add_return_relaxed(i, v);
1239+ __atomic_post_full_fence();
1240+ return ret;
1241+}
1242+#define arch_atomic64_add_return arch_atomic64_add_return
1243+#endif
1244+
1245+#endif /* arch_atomic64_add_return_relaxed */
1246+
1247+#ifndef arch_atomic64_fetch_add_relaxed
1248+#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1249+#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1250+#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1251+#else /* arch_atomic64_fetch_add_relaxed */
1252+
1253+#ifndef arch_atomic64_fetch_add_acquire
1254+static __always_inline s64
1255+arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1256+{
1257+ s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1258+ __atomic_acquire_fence();
1259+ return ret;
1260+}
1261+#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1262+#endif
1263+
1264+#ifndef arch_atomic64_fetch_add_release
1265+static __always_inline s64
1266+arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1267+{
1268+ __atomic_release_fence();
1269+ return arch_atomic64_fetch_add_relaxed(i, v);
1270+}
1271+#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1272+#endif
1273+
1274+#ifndef arch_atomic64_fetch_add
1275+static __always_inline s64
1276+arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1277+{
1278+ s64 ret;
1279+ __atomic_pre_full_fence();
1280+ ret = arch_atomic64_fetch_add_relaxed(i, v);
1281+ __atomic_post_full_fence();
1282+ return ret;
1283+}
1284+#define arch_atomic64_fetch_add arch_atomic64_fetch_add
1285+#endif
1286+
1287+#endif /* arch_atomic64_fetch_add_relaxed */
1288+
1289+#ifndef arch_atomic64_sub_return_relaxed
1290+#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1291+#define arch_atomic64_sub_return_release arch_atomic64_sub_return
1292+#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1293+#else /* arch_atomic64_sub_return_relaxed */
1294+
1295+#ifndef arch_atomic64_sub_return_acquire
1296+static __always_inline s64
1297+arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1298+{
1299+ s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1300+ __atomic_acquire_fence();
1301+ return ret;
1302+}
1303+#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1304+#endif
1305+
1306+#ifndef arch_atomic64_sub_return_release
1307+static __always_inline s64
1308+arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1309+{
1310+ __atomic_release_fence();
1311+ return arch_atomic64_sub_return_relaxed(i, v);
1312+}
1313+#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1314+#endif
1315+
1316+#ifndef arch_atomic64_sub_return
1317+static __always_inline s64
1318+arch_atomic64_sub_return(s64 i, atomic64_t *v)
1319+{
1320+ s64 ret;
1321+ __atomic_pre_full_fence();
1322+ ret = arch_atomic64_sub_return_relaxed(i, v);
1323+ __atomic_post_full_fence();
1324+ return ret;
1325+}
1326+#define arch_atomic64_sub_return arch_atomic64_sub_return
1327+#endif
1328+
1329+#endif /* arch_atomic64_sub_return_relaxed */
1330+
1331+#ifndef arch_atomic64_fetch_sub_relaxed
1332+#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1333+#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1334+#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1335+#else /* arch_atomic64_fetch_sub_relaxed */
1336+
1337+#ifndef arch_atomic64_fetch_sub_acquire
1338+static __always_inline s64
1339+arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1340+{
1341+ s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1342+ __atomic_acquire_fence();
1343+ return ret;
1344+}
1345+#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1346+#endif
1347+
1348+#ifndef arch_atomic64_fetch_sub_release
1349+static __always_inline s64
1350+arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1351+{
1352+ __atomic_release_fence();
1353+ return arch_atomic64_fetch_sub_relaxed(i, v);
1354+}
1355+#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1356+#endif
1357+
1358+#ifndef arch_atomic64_fetch_sub
1359+static __always_inline s64
1360+arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1361+{
1362+ s64 ret;
1363+ __atomic_pre_full_fence();
1364+ ret = arch_atomic64_fetch_sub_relaxed(i, v);
1365+ __atomic_post_full_fence();
1366+ return ret;
1367+}
1368+#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1369+#endif
1370+
1371+#endif /* arch_atomic64_fetch_sub_relaxed */
1372+
1373+#ifndef arch_atomic64_inc
1374+static __always_inline void
1375+arch_atomic64_inc(atomic64_t *v)
1376+{
1377+ arch_atomic64_add(1, v);
1378+}
1379+#define arch_atomic64_inc arch_atomic64_inc
1380+#endif
1381+
1382+#ifndef arch_atomic64_inc_return_relaxed
1383+#ifdef arch_atomic64_inc_return
1384+#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1385+#define arch_atomic64_inc_return_release arch_atomic64_inc_return
1386+#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1387+#endif /* arch_atomic64_inc_return */
1388+
1389+#ifndef arch_atomic64_inc_return
1390+static __always_inline s64
1391+arch_atomic64_inc_return(atomic64_t *v)
1392+{
1393+ return arch_atomic64_add_return(1, v);
1394+}
1395+#define arch_atomic64_inc_return arch_atomic64_inc_return
1396+#endif
1397+
1398+#ifndef arch_atomic64_inc_return_acquire
1399+static __always_inline s64
1400+arch_atomic64_inc_return_acquire(atomic64_t *v)
1401+{
1402+ return arch_atomic64_add_return_acquire(1, v);
1403+}
1404+#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1405+#endif
1406+
1407+#ifndef arch_atomic64_inc_return_release
1408+static __always_inline s64
1409+arch_atomic64_inc_return_release(atomic64_t *v)
1410+{
1411+ return arch_atomic64_add_return_release(1, v);
1412+}
1413+#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1414+#endif
1415+
1416+#ifndef arch_atomic64_inc_return_relaxed
1417+static __always_inline s64
1418+arch_atomic64_inc_return_relaxed(atomic64_t *v)
1419+{
1420+ return arch_atomic64_add_return_relaxed(1, v);
1421+}
1422+#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1423+#endif
1424+
1425+#else /* arch_atomic64_inc_return_relaxed */
1426+
1427+#ifndef arch_atomic64_inc_return_acquire
1428+static __always_inline s64
1429+arch_atomic64_inc_return_acquire(atomic64_t *v)
1430+{
1431+ s64 ret = arch_atomic64_inc_return_relaxed(v);
1432+ __atomic_acquire_fence();
1433+ return ret;
1434+}
1435+#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1436+#endif
1437+
1438+#ifndef arch_atomic64_inc_return_release
1439+static __always_inline s64
1440+arch_atomic64_inc_return_release(atomic64_t *v)
1441+{
1442+ __atomic_release_fence();
1443+ return arch_atomic64_inc_return_relaxed(v);
1444+}
1445+#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1446+#endif
1447+
1448+#ifndef arch_atomic64_inc_return
1449+static __always_inline s64
1450+arch_atomic64_inc_return(atomic64_t *v)
1451+{
1452+ s64 ret;
1453+ __atomic_pre_full_fence();
1454+ ret = arch_atomic64_inc_return_relaxed(v);
1455+ __atomic_post_full_fence();
1456+ return ret;
1457+}
1458+#define arch_atomic64_inc_return arch_atomic64_inc_return
1459+#endif
1460+
1461+#endif /* arch_atomic64_inc_return_relaxed */
1462+
1463+#ifndef arch_atomic64_fetch_inc_relaxed
1464+#ifdef arch_atomic64_fetch_inc
1465+#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1466+#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1467+#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1468+#endif /* arch_atomic64_fetch_inc */
1469+
1470+#ifndef arch_atomic64_fetch_inc
1471+static __always_inline s64
1472+arch_atomic64_fetch_inc(atomic64_t *v)
1473+{
1474+ return arch_atomic64_fetch_add(1, v);
1475+}
1476+#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1477+#endif
1478+
1479+#ifndef arch_atomic64_fetch_inc_acquire
1480+static __always_inline s64
1481+arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1482+{
1483+ return arch_atomic64_fetch_add_acquire(1, v);
1484+}
1485+#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1486+#endif
1487+
1488+#ifndef arch_atomic64_fetch_inc_release
1489+static __always_inline s64
1490+arch_atomic64_fetch_inc_release(atomic64_t *v)
1491+{
1492+ return arch_atomic64_fetch_add_release(1, v);
1493+}
1494+#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1495+#endif
1496+
1497+#ifndef arch_atomic64_fetch_inc_relaxed
1498+static __always_inline s64
1499+arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1500+{
1501+ return arch_atomic64_fetch_add_relaxed(1, v);
1502+}
1503+#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1504+#endif
1505+
1506+#else /* arch_atomic64_fetch_inc_relaxed */
1507+
1508+#ifndef arch_atomic64_fetch_inc_acquire
1509+static __always_inline s64
1510+arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1511+{
1512+ s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1513+ __atomic_acquire_fence();
1514+ return ret;
1515+}
1516+#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1517+#endif
1518+
1519+#ifndef arch_atomic64_fetch_inc_release
1520+static __always_inline s64
1521+arch_atomic64_fetch_inc_release(atomic64_t *v)
1522+{
1523+ __atomic_release_fence();
1524+ return arch_atomic64_fetch_inc_relaxed(v);
1525+}
1526+#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1527+#endif
1528+
1529+#ifndef arch_atomic64_fetch_inc
1530+static __always_inline s64
1531+arch_atomic64_fetch_inc(atomic64_t *v)
1532+{
1533+ s64 ret;
1534+ __atomic_pre_full_fence();
1535+ ret = arch_atomic64_fetch_inc_relaxed(v);
1536+ __atomic_post_full_fence();
1537+ return ret;
1538+}
1539+#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1540+#endif
1541+
1542+#endif /* arch_atomic64_fetch_inc_relaxed */
1543+
1544+#ifndef arch_atomic64_dec
1545+static __always_inline void
1546+arch_atomic64_dec(atomic64_t *v)
1547+{
1548+ arch_atomic64_sub(1, v);
1549+}
1550+#define arch_atomic64_dec arch_atomic64_dec
1551+#endif
1552+
1553+#ifndef arch_atomic64_dec_return_relaxed
1554+#ifdef arch_atomic64_dec_return
1555+#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1556+#define arch_atomic64_dec_return_release arch_atomic64_dec_return
1557+#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1558+#endif /* arch_atomic64_dec_return */
1559+
1560+#ifndef arch_atomic64_dec_return
1561+static __always_inline s64
1562+arch_atomic64_dec_return(atomic64_t *v)
1563+{
1564+ return arch_atomic64_sub_return(1, v);
1565+}
1566+#define arch_atomic64_dec_return arch_atomic64_dec_return
1567+#endif
1568+
1569+#ifndef arch_atomic64_dec_return_acquire
1570+static __always_inline s64
1571+arch_atomic64_dec_return_acquire(atomic64_t *v)
1572+{
1573+ return arch_atomic64_sub_return_acquire(1, v);
1574+}
1575+#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1576+#endif
1577+
1578+#ifndef arch_atomic64_dec_return_release
1579+static __always_inline s64
1580+arch_atomic64_dec_return_release(atomic64_t *v)
1581+{
1582+ return arch_atomic64_sub_return_release(1, v);
1583+}
1584+#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1585+#endif
1586+
1587+#ifndef arch_atomic64_dec_return_relaxed
1588+static __always_inline s64
1589+arch_atomic64_dec_return_relaxed(atomic64_t *v)
1590+{
1591+ return arch_atomic64_sub_return_relaxed(1, v);
1592+}
1593+#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1594+#endif
1595+
1596+#else /* arch_atomic64_dec_return_relaxed */
1597+
1598+#ifndef arch_atomic64_dec_return_acquire
1599+static __always_inline s64
1600+arch_atomic64_dec_return_acquire(atomic64_t *v)
1601+{
1602+ s64 ret = arch_atomic64_dec_return_relaxed(v);
1603+ __atomic_acquire_fence();
1604+ return ret;
1605+}
1606+#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1607+#endif
1608+
1609+#ifndef arch_atomic64_dec_return_release
1610+static __always_inline s64
1611+arch_atomic64_dec_return_release(atomic64_t *v)
1612+{
1613+ __atomic_release_fence();
1614+ return arch_atomic64_dec_return_relaxed(v);
1615+}
1616+#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1617+#endif
1618+
1619+#ifndef arch_atomic64_dec_return
1620+static __always_inline s64
1621+arch_atomic64_dec_return(atomic64_t *v)
1622+{
1623+ s64 ret;
1624+ __atomic_pre_full_fence();
1625+ ret = arch_atomic64_dec_return_relaxed(v);
1626+ __atomic_post_full_fence();
1627+ return ret;
1628+}
1629+#define arch_atomic64_dec_return arch_atomic64_dec_return
1630+#endif
1631+
1632+#endif /* arch_atomic64_dec_return_relaxed */
1633+
1634+#ifndef arch_atomic64_fetch_dec_relaxed
1635+#ifdef arch_atomic64_fetch_dec
1636+#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1637+#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1638+#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1639+#endif /* arch_atomic64_fetch_dec */
1640+
1641+#ifndef arch_atomic64_fetch_dec
1642+static __always_inline s64
1643+arch_atomic64_fetch_dec(atomic64_t *v)
1644+{
1645+ return arch_atomic64_fetch_sub(1, v);
1646+}
1647+#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1648+#endif
1649+
1650+#ifndef arch_atomic64_fetch_dec_acquire
1651+static __always_inline s64
1652+arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1653+{
1654+ return arch_atomic64_fetch_sub_acquire(1, v);
1655+}
1656+#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1657+#endif
1658+
1659+#ifndef arch_atomic64_fetch_dec_release
1660+static __always_inline s64
1661+arch_atomic64_fetch_dec_release(atomic64_t *v)
1662+{
1663+ return arch_atomic64_fetch_sub_release(1, v);
1664+}
1665+#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1666+#endif
1667+
1668+#ifndef arch_atomic64_fetch_dec_relaxed
1669+static __always_inline s64
1670+arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1671+{
1672+ return arch_atomic64_fetch_sub_relaxed(1, v);
1673+}
1674+#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1675+#endif
1676+
1677+#else /* arch_atomic64_fetch_dec_relaxed */
1678+
1679+#ifndef arch_atomic64_fetch_dec_acquire
1680+static __always_inline s64
1681+arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1682+{
1683+ s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1684+ __atomic_acquire_fence();
1685+ return ret;
1686+}
1687+#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1688+#endif
1689+
1690+#ifndef arch_atomic64_fetch_dec_release
1691+static __always_inline s64
1692+arch_atomic64_fetch_dec_release(atomic64_t *v)
1693+{
1694+ __atomic_release_fence();
1695+ return arch_atomic64_fetch_dec_relaxed(v);
1696+}
1697+#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1698+#endif
1699+
1700+#ifndef arch_atomic64_fetch_dec
1701+static __always_inline s64
1702+arch_atomic64_fetch_dec(atomic64_t *v)
1703+{
1704+ s64 ret;
1705+ __atomic_pre_full_fence();
1706+ ret = arch_atomic64_fetch_dec_relaxed(v);
1707+ __atomic_post_full_fence();
1708+ return ret;
1709+}
1710+#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1711+#endif
1712+
1713+#endif /* arch_atomic64_fetch_dec_relaxed */
1714+
1715+#ifndef arch_atomic64_fetch_and_relaxed
1716+#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1717+#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1718+#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1719+#else /* arch_atomic64_fetch_and_relaxed */
1720+
1721+#ifndef arch_atomic64_fetch_and_acquire
1722+static __always_inline s64
1723+arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1724+{
1725+ s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1726+ __atomic_acquire_fence();
1727+ return ret;
1728+}
1729+#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1730+#endif
1731+
1732+#ifndef arch_atomic64_fetch_and_release
1733+static __always_inline s64
1734+arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1735+{
1736+ __atomic_release_fence();
1737+ return arch_atomic64_fetch_and_relaxed(i, v);
1738+}
1739+#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1740+#endif
1741+
1742+#ifndef arch_atomic64_fetch_and
1743+static __always_inline s64
1744+arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1745+{
1746+ s64 ret;
1747+ __atomic_pre_full_fence();
1748+ ret = arch_atomic64_fetch_and_relaxed(i, v);
1749+ __atomic_post_full_fence();
1750+ return ret;
1751+}
1752+#define arch_atomic64_fetch_and arch_atomic64_fetch_and
1753+#endif
1754+
1755+#endif /* arch_atomic64_fetch_and_relaxed */
1756+
1757+#ifndef arch_atomic64_andnot
1758+static __always_inline void
1759+arch_atomic64_andnot(s64 i, atomic64_t *v)
1760+{
1761+ arch_atomic64_and(~i, v);
1762+}
1763+#define arch_atomic64_andnot arch_atomic64_andnot
1764+#endif
1765+
1766+#ifndef arch_atomic64_fetch_andnot_relaxed
1767+#ifdef arch_atomic64_fetch_andnot
1768+#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1769+#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1770+#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1771+#endif /* arch_atomic64_fetch_andnot */
1772+
1773+#ifndef arch_atomic64_fetch_andnot
1774+static __always_inline s64
1775+arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1776+{
1777+ return arch_atomic64_fetch_and(~i, v);
1778+}
1779+#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1780+#endif
1781+
1782+#ifndef arch_atomic64_fetch_andnot_acquire
1783+static __always_inline s64
1784+arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1785+{
1786+ return arch_atomic64_fetch_and_acquire(~i, v);
1787+}
1788+#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1789+#endif
1790+
1791+#ifndef arch_atomic64_fetch_andnot_release
1792+static __always_inline s64
1793+arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1794+{
1795+ return arch_atomic64_fetch_and_release(~i, v);
1796+}
1797+#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1798+#endif
1799+
1800+#ifndef arch_atomic64_fetch_andnot_relaxed
1801+static __always_inline s64
1802+arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1803+{
1804+ return arch_atomic64_fetch_and_relaxed(~i, v);
1805+}
1806+#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1807+#endif
1808+
1809+#else /* arch_atomic64_fetch_andnot_relaxed */
1810+
1811+#ifndef arch_atomic64_fetch_andnot_acquire
1812+static __always_inline s64
1813+arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1814+{
1815+ s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1816+ __atomic_acquire_fence();
1817+ return ret;
1818+}
1819+#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1820+#endif
1821+
1822+#ifndef arch_atomic64_fetch_andnot_release
1823+static __always_inline s64
1824+arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1825+{
1826+ __atomic_release_fence();
1827+ return arch_atomic64_fetch_andnot_relaxed(i, v);
1828+}
1829+#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1830+#endif
1831+
1832+#ifndef arch_atomic64_fetch_andnot
1833+static __always_inline s64
1834+arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1835+{
1836+ s64 ret;
1837+ __atomic_pre_full_fence();
1838+ ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1839+ __atomic_post_full_fence();
1840+ return ret;
1841+}
1842+#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1843+#endif
1844+
1845+#endif /* arch_atomic64_fetch_andnot_relaxed */
1846+
1847+#ifndef arch_atomic64_fetch_or_relaxed
1848+#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
1849+#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
1850+#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
1851+#else /* arch_atomic64_fetch_or_relaxed */
1852+
1853+#ifndef arch_atomic64_fetch_or_acquire
1854+static __always_inline s64
1855+arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1856+{
1857+ s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
1858+ __atomic_acquire_fence();
1859+ return ret;
1860+}
1861+#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
1862+#endif
1863+
1864+#ifndef arch_atomic64_fetch_or_release
1865+static __always_inline s64
1866+arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
1867+{
1868+ __atomic_release_fence();
1869+ return arch_atomic64_fetch_or_relaxed(i, v);
1870+}
1871+#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
1872+#endif
1873+
1874+#ifndef arch_atomic64_fetch_or
1875+static __always_inline s64
1876+arch_atomic64_fetch_or(s64 i, atomic64_t *v)
1877+{
1878+ s64 ret;
1879+ __atomic_pre_full_fence();
1880+ ret = arch_atomic64_fetch_or_relaxed(i, v);
1881+ __atomic_post_full_fence();
1882+ return ret;
1883+}
1884+#define arch_atomic64_fetch_or arch_atomic64_fetch_or
1885+#endif
1886+
1887+#endif /* arch_atomic64_fetch_or_relaxed */
1888+
1889+#ifndef arch_atomic64_fetch_xor_relaxed
1890+#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
1891+#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
1892+#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
1893+#else /* arch_atomic64_fetch_xor_relaxed */
1894+
1895+#ifndef arch_atomic64_fetch_xor_acquire
1896+static __always_inline s64
1897+arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1898+{
1899+ s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1900+ __atomic_acquire_fence();
1901+ return ret;
1902+}
1903+#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
1904+#endif
1905+
1906+#ifndef arch_atomic64_fetch_xor_release
1907+static __always_inline s64
1908+arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1909+{
1910+ __atomic_release_fence();
1911+ return arch_atomic64_fetch_xor_relaxed(i, v);
1912+}
1913+#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
1914+#endif
1915+
1916+#ifndef arch_atomic64_fetch_xor
1917+static __always_inline s64
1918+arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
1919+{
1920+ s64 ret;
1921+ __atomic_pre_full_fence();
1922+ ret = arch_atomic64_fetch_xor_relaxed(i, v);
1923+ __atomic_post_full_fence();
1924+ return ret;
1925+}
1926+#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
1927+#endif
1928+
1929+#endif /* arch_atomic64_fetch_xor_relaxed */
1930+
1931+#ifndef arch_atomic64_xchg_relaxed
1932+#define arch_atomic64_xchg_acquire arch_atomic64_xchg
1933+#define arch_atomic64_xchg_release arch_atomic64_xchg
1934+#define arch_atomic64_xchg_relaxed arch_atomic64_xchg
1935+#else /* arch_atomic64_xchg_relaxed */
1936+
1937+#ifndef arch_atomic64_xchg_acquire
1938+static __always_inline s64
1939+arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
1940+{
1941+ s64 ret = arch_atomic64_xchg_relaxed(v, i);
1942+ __atomic_acquire_fence();
1943+ return ret;
1944+}
1945+#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
1946+#endif
1947+
1948+#ifndef arch_atomic64_xchg_release
1949+static __always_inline s64
1950+arch_atomic64_xchg_release(atomic64_t *v, s64 i)
1951+{
1952+ __atomic_release_fence();
1953+ return arch_atomic64_xchg_relaxed(v, i);
1954+}
1955+#define arch_atomic64_xchg_release arch_atomic64_xchg_release
1956+#endif
1957+
1958+#ifndef arch_atomic64_xchg
1959+static __always_inline s64
1960+arch_atomic64_xchg(atomic64_t *v, s64 i)
1961+{
1962+ s64 ret;
1963+ __atomic_pre_full_fence();
1964+ ret = arch_atomic64_xchg_relaxed(v, i);
1965+ __atomic_post_full_fence();
1966+ return ret;
1967+}
1968+#define arch_atomic64_xchg arch_atomic64_xchg
1969+#endif
1970+
1971+#endif /* arch_atomic64_xchg_relaxed */
1972+
1973+#ifndef arch_atomic64_cmpxchg_relaxed
1974+#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
1975+#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
1976+#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
1977+#else /* arch_atomic64_cmpxchg_relaxed */
1978+
1979+#ifndef arch_atomic64_cmpxchg_acquire
1980+static __always_inline s64
1981+arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1982+{
1983+ s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
1984+ __atomic_acquire_fence();
1985+ return ret;
1986+}
1987+#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
1988+#endif
1989+
1990+#ifndef arch_atomic64_cmpxchg_release
1991+static __always_inline s64
1992+arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1993+{
1994+ __atomic_release_fence();
1995+ return arch_atomic64_cmpxchg_relaxed(v, old, new);
1996+}
1997+#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
1998+#endif
1999+
2000+#ifndef arch_atomic64_cmpxchg
2001+static __always_inline s64
2002+arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2003+{
2004+ s64 ret;
2005+ __atomic_pre_full_fence();
2006+ ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2007+ __atomic_post_full_fence();
2008+ return ret;
2009+}
2010+#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2011+#endif
2012+
2013+#endif /* arch_atomic64_cmpxchg_relaxed */
2014+
2015+#ifndef arch_atomic64_try_cmpxchg_relaxed
2016+#ifdef arch_atomic64_try_cmpxchg
2017+#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2018+#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2019+#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2020+#endif /* arch_atomic64_try_cmpxchg */
2021+
2022+#ifndef arch_atomic64_try_cmpxchg
2023+static __always_inline bool
2024+arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2025+{
2026+ s64 r, o = *old;
2027+ r = arch_atomic64_cmpxchg(v, o, new);
2028+ if (unlikely(r != o))
2029+ *old = r;
2030+ return likely(r == o);
2031+}
2032+#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2033+#endif
2034+
2035+#ifndef arch_atomic64_try_cmpxchg_acquire
2036+static __always_inline bool
2037+arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2038+{
2039+ s64 r, o = *old;
2040+ r = arch_atomic64_cmpxchg_acquire(v, o, new);
2041+ if (unlikely(r != o))
2042+ *old = r;
2043+ return likely(r == o);
2044+}
2045+#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2046+#endif
2047+
2048+#ifndef arch_atomic64_try_cmpxchg_release
2049+static __always_inline bool
2050+arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2051+{
2052+ s64 r, o = *old;
2053+ r = arch_atomic64_cmpxchg_release(v, o, new);
2054+ if (unlikely(r != o))
2055+ *old = r;
2056+ return likely(r == o);
2057+}
2058+#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2059+#endif
2060+
2061+#ifndef arch_atomic64_try_cmpxchg_relaxed
2062+static __always_inline bool
2063+arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2064+{
2065+ s64 r, o = *old;
2066+ r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2067+ if (unlikely(r != o))
2068+ *old = r;
2069+ return likely(r == o);
2070+}
2071+#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2072+#endif
2073+
2074+#else /* arch_atomic64_try_cmpxchg_relaxed */
2075+
2076+#ifndef arch_atomic64_try_cmpxchg_acquire
2077+static __always_inline bool
2078+arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2079+{
2080+ bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2081+ __atomic_acquire_fence();
2082+ return ret;
2083+}
2084+#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2085+#endif
2086+
2087+#ifndef arch_atomic64_try_cmpxchg_release
2088+static __always_inline bool
2089+arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2090+{
2091+ __atomic_release_fence();
2092+ return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2093+}
2094+#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2095+#endif
2096+
2097+#ifndef arch_atomic64_try_cmpxchg
2098+static __always_inline bool
2099+arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2100+{
2101+ bool ret;
2102+ __atomic_pre_full_fence();
2103+ ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2104+ __atomic_post_full_fence();
2105+ return ret;
2106+}
2107+#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2108+#endif
2109+
2110+#endif /* arch_atomic64_try_cmpxchg_relaxed */
2111+
2112+#ifndef arch_atomic64_sub_and_test
2113+/**
2114+ * arch_atomic64_sub_and_test - subtract value from variable and test result
2115+ * @i: integer value to subtract
2116+ * @v: pointer of type atomic64_t
2117+ *
2118+ * Atomically subtracts @i from @v and returns
2119+ * true if the result is zero, or false for all
2120+ * other cases.
2121+ */
2122+static __always_inline bool
2123+arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2124+{
2125+ return arch_atomic64_sub_return(i, v) == 0;
2126+}
2127+#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2128+#endif
2129+
2130+#ifndef arch_atomic64_dec_and_test
2131+/**
2132+ * arch_atomic64_dec_and_test - decrement and test
2133+ * @v: pointer of type atomic64_t
2134+ *
2135+ * Atomically decrements @v by 1 and
2136+ * returns true if the result is 0, or false for all other
2137+ * cases.
2138+ */
2139+static __always_inline bool
2140+arch_atomic64_dec_and_test(atomic64_t *v)
2141+{
2142+ return arch_atomic64_dec_return(v) == 0;
2143+}
2144+#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2145+#endif
2146+
2147+#ifndef arch_atomic64_inc_and_test
2148+/**
2149+ * arch_atomic64_inc_and_test - increment and test
2150+ * @v: pointer of type atomic64_t
2151+ *
2152+ * Atomically increments @v by 1
2153+ * and returns true if the result is zero, or false for all
2154+ * other cases.
2155+ */
2156+static __always_inline bool
2157+arch_atomic64_inc_and_test(atomic64_t *v)
2158+{
2159+ return arch_atomic64_inc_return(v) == 0;
2160+}
2161+#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2162+#endif
2163+
2164+#ifndef arch_atomic64_add_negative
2165+/**
2166+ * arch_atomic64_add_negative - add and test if negative
2167+ * @i: integer value to add
2168+ * @v: pointer of type atomic64_t
2169+ *
2170+ * Atomically adds @i to @v and returns true
2171+ * if the result is negative, or false when
2172+ * result is greater than or equal to zero.
2173+ */
2174+static __always_inline bool
2175+arch_atomic64_add_negative(s64 i, atomic64_t *v)
2176+{
2177+ return arch_atomic64_add_return(i, v) < 0;
2178+}
2179+#define arch_atomic64_add_negative arch_atomic64_add_negative
2180+#endif
2181+
2182+#ifndef arch_atomic64_fetch_add_unless
2183+/**
2184+ * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2185+ * @v: pointer of type atomic64_t
2186+ * @a: the amount to add to v...
2187+ * @u: ...unless v is equal to u.
2188+ *
2189+ * Atomically adds @a to @v, so long as @v was not already @u.
2190+ * Returns original value of @v
2191+ */
2192+static __always_inline s64
2193+arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2194+{
2195+ s64 c = arch_atomic64_read(v);
2196+
2197+ do {
2198+ if (unlikely(c == u))
2199+ break;
2200+ } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2201+
2202+ return c;
2203+}
2204+#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2205+#endif
2206+
2207+#ifndef arch_atomic64_add_unless
2208+/**
2209+ * arch_atomic64_add_unless - add unless the number is already a given value
2210+ * @v: pointer of type atomic64_t
2211+ * @a: the amount to add to v...
2212+ * @u: ...unless v is equal to u.
2213+ *
2214+ * Atomically adds @a to @v, if @v was not already @u.
2215+ * Returns true if the addition was done.
2216+ */
2217+static __always_inline bool
2218+arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2219+{
2220+ return arch_atomic64_fetch_add_unless(v, a, u) != u;
2221+}
2222+#define arch_atomic64_add_unless arch_atomic64_add_unless
2223+#endif
2224+
2225+#ifndef arch_atomic64_inc_not_zero
2226+/**
2227+ * arch_atomic64_inc_not_zero - increment unless the number is zero
2228+ * @v: pointer of type atomic64_t
2229+ *
2230+ * Atomically increments @v by 1, if @v is non-zero.
2231+ * Returns true if the increment was done.
2232+ */
2233+static __always_inline bool
2234+arch_atomic64_inc_not_zero(atomic64_t *v)
2235+{
2236+ return arch_atomic64_add_unless(v, 1, 0);
2237+}
2238+#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2239+#endif
2240+
2241+#ifndef arch_atomic64_inc_unless_negative
2242+static __always_inline bool
2243+arch_atomic64_inc_unless_negative(atomic64_t *v)
2244+{
2245+ s64 c = arch_atomic64_read(v);
2246+
2247+ do {
2248+ if (unlikely(c < 0))
2249+ return false;
2250+ } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2251+
2252+ return true;
2253+}
2254+#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2255+#endif
2256+
2257+#ifndef arch_atomic64_dec_unless_positive
2258+static __always_inline bool
2259+arch_atomic64_dec_unless_positive(atomic64_t *v)
2260+{
2261+ s64 c = arch_atomic64_read(v);
2262+
2263+ do {
2264+ if (unlikely(c > 0))
2265+ return false;
2266+ } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2267+
2268+ return true;
2269+}
2270+#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2271+#endif
2272+
2273+#ifndef arch_atomic64_dec_if_positive
2274+static __always_inline s64
2275+arch_atomic64_dec_if_positive(atomic64_t *v)
2276+{
2277+ s64 dec, c = arch_atomic64_read(v);
2278+
2279+ do {
2280+ dec = c - 1;
2281+ if (unlikely(dec < 0))
2282+ break;
2283+ } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2284+
2285+ return dec;
2286+}
2287+#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2288+#endif
2289+
2290+#endif /* _LINUX_ATOMIC_FALLBACK_H */
2291+// 90cd26cfd69d2250303d654955a0cc12620fb91b
include/linux/atomic-fallback.h+171−175
@@ -6,6 +6,8 @@ · 5 unmodified lines
66 #ifndef _LINUX_ATOMIC_FALLBACK_H
77 #define _LINUX_ATOMIC_FALLBACK_H
88
9+#include <linux/compiler.h>
10+
911 #ifndef xchg_relaxed
1012 #define xchg_relaxed xchg
1113 #define xchg_acquire xchg
@@ -76,7 +78,7 @@ · 64 unmodified lines
7678 #endif /* cmpxchg64_relaxed */
7779
7880 #ifndef atomic_read_acquire
79−static inline int
81+static __always_inline int
8082 atomic_read_acquire(const atomic_t *v)
8183 {
8284 return smp_load_acquire(&(v)->counter);
@@ -85,7 +87,7 @@ atomic_read_acquire(const atomic_t *v) · 2 unmodified lines
8587 #endif
8688
8789 #ifndef atomic_set_release
88−static inline void
90+static __always_inline void
8991 atomic_set_release(atomic_t *v, int i)
9092 {
9193 smp_store_release(&(v)->counter, i);
@@ -100,7 +102,7 @@ atomic_set_release(atomic_t *v, int i) · 8 unmodified lines
100102 #else /* atomic_add_return_relaxed */
101103
102104 #ifndef atomic_add_return_acquire
103−static inline int
105+static __always_inline int
104106 atomic_add_return_acquire(int i, atomic_t *v)
105107 {
106108 int ret = atomic_add_return_relaxed(i, v);
@@ -111,7 +113,7 @@ atomic_add_return_acquire(int i, atomic_t *v) · 4 unmodified lines
111113 #endif
112114
113115 #ifndef atomic_add_return_release
114−static inline int
116+static __always_inline int
115117 atomic_add_return_release(int i, atomic_t *v)
116118 {
117119 __atomic_release_fence();
@@ -121,7 +123,7 @@ atomic_add_return_release(int i, atomic_t *v) · 3 unmodified lines
121123 #endif
122124
123125 #ifndef atomic_add_return
124−static inline int
126+static __always_inline int
125127 atomic_add_return(int i, atomic_t *v)
126128 {
127129 int ret;
@@ -142,7 +144,7 @@ atomic_add_return(int i, atomic_t *v) · 14 unmodified lines
142144 #else /* atomic_fetch_add_relaxed */
143145
144146 #ifndef atomic_fetch_add_acquire
145−static inline int
147+static __always_inline int
146148 atomic_fetch_add_acquire(int i, atomic_t *v)
147149 {
148150 int ret = atomic_fetch_add_relaxed(i, v);
@@ -153,7 +155,7 @@ atomic_fetch_add_acquire(int i, atomic_t *v) · 4 unmodified lines
153155 #endif
154156
155157 #ifndef atomic_fetch_add_release
156−static inline int
158+static __always_inline int
157159 atomic_fetch_add_release(int i, atomic_t *v)
158160 {
159161 __atomic_release_fence();
@@ -163,7 +165,7 @@ atomic_fetch_add_release(int i, atomic_t *v) · 3 unmodified lines
163165 #endif
164166
165167 #ifndef atomic_fetch_add
166−static inline int
168+static __always_inline int
167169 atomic_fetch_add(int i, atomic_t *v)
168170 {
169171 int ret;
@@ -184,7 +186,7 @@ atomic_fetch_add(int i, atomic_t *v) · 14 unmodified lines
184186 #else /* atomic_sub_return_relaxed */
185187
186188 #ifndef atomic_sub_return_acquire
187−static inline int
189+static __always_inline int
188190 atomic_sub_return_acquire(int i, atomic_t *v)
189191 {
190192 int ret = atomic_sub_return_relaxed(i, v);
@@ -195,7 +197,7 @@ atomic_sub_return_acquire(int i, atomic_t *v) · 4 unmodified lines
195197 #endif
196198
197199 #ifndef atomic_sub_return_release
198−static inline int
200+static __always_inline int
199201 atomic_sub_return_release(int i, atomic_t *v)
200202 {
201203 __atomic_release_fence();
@@ -205,7 +207,7 @@ atomic_sub_return_release(int i, atomic_t *v) · 3 unmodified lines
205207 #endif
206208
207209 #ifndef atomic_sub_return
208−static inline int
210+static __always_inline int
209211 atomic_sub_return(int i, atomic_t *v)
210212 {
211213 int ret;
@@ -226,7 +228,7 @@ atomic_sub_return(int i, atomic_t *v) · 14 unmodified lines
226228 #else /* atomic_fetch_sub_relaxed */
227229
228230 #ifndef atomic_fetch_sub_acquire
229−static inline int
231+static __always_inline int
230232 atomic_fetch_sub_acquire(int i, atomic_t *v)
231233 {
232234 int ret = atomic_fetch_sub_relaxed(i, v);
@@ -237,7 +239,7 @@ atomic_fetch_sub_acquire(int i, atomic_t *v) · 4 unmodified lines
237239 #endif
238240
239241 #ifndef atomic_fetch_sub_release
240−static inline int
242+static __always_inline int
241243 atomic_fetch_sub_release(int i, atomic_t *v)
242244 {
243245 __atomic_release_fence();
@@ -247,7 +249,7 @@ atomic_fetch_sub_release(int i, atomic_t *v) · 3 unmodified lines
247249 #endif
248250
249251 #ifndef atomic_fetch_sub
250−static inline int
252+static __always_inline int
251253 atomic_fetch_sub(int i, atomic_t *v)
252254 {
253255 int ret;
@@ -262,7 +264,7 @@ atomic_fetch_sub(int i, atomic_t *v) · 8 unmodified lines
262264 #endif /* atomic_fetch_sub_relaxed */
263265
264266 #ifndef atomic_inc
265−static inline void
267+static __always_inline void
266268 atomic_inc(atomic_t *v)
267269 {
268270 atomic_add(1, v);
@@ -278,7 +280,7 @@ atomic_inc(atomic_t *v) · 9 unmodified lines
278280 #endif /* atomic_inc_return */
279281
280282 #ifndef atomic_inc_return
281−static inline int
283+static __always_inline int
282284 atomic_inc_return(atomic_t *v)
283285 {
284286 return atomic_add_return(1, v);
@@ -287,7 +289,7 @@ atomic_inc_return(atomic_t *v) · 2 unmodified lines
287289 #endif
288290
289291 #ifndef atomic_inc_return_acquire
290−static inline int
292+static __always_inline int
291293 atomic_inc_return_acquire(atomic_t *v)
292294 {
293295 return atomic_add_return_acquire(1, v);
@@ -296,7 +298,7 @@ atomic_inc_return_acquire(atomic_t *v) · 2 unmodified lines
296298 #endif
297299
298300 #ifndef atomic_inc_return_release
299−static inline int
301+static __always_inline int
300302 atomic_inc_return_release(atomic_t *v)
301303 {
302304 return atomic_add_return_release(1, v);
@@ -305,7 +307,7 @@ atomic_inc_return_release(atomic_t *v) · 2 unmodified lines
305307 #endif
306308
307309 #ifndef atomic_inc_return_relaxed
308−static inline int
310+static __always_inline int
309311 atomic_inc_return_relaxed(atomic_t *v)
310312 {
311313 return atomic_add_return_relaxed(1, v);
@@ -316,7 +318,7 @@ atomic_inc_return_relaxed(atomic_t *v) · 4 unmodified lines
316318 #else /* atomic_inc_return_relaxed */
317319
318320 #ifndef atomic_inc_return_acquire
319−static inline int
321+static __always_inline int
320322 atomic_inc_return_acquire(atomic_t *v)
321323 {
322324 int ret = atomic_inc_return_relaxed(v);
@@ -327,7 +329,7 @@ atomic_inc_return_acquire(atomic_t *v) · 4 unmodified lines
327329 #endif
328330
329331 #ifndef atomic_inc_return_release
330−static inline int
332+static __always_inline int
331333 atomic_inc_return_release(atomic_t *v)
332334 {
333335 __atomic_release_fence();
@@ -337,7 +339,7 @@ atomic_inc_return_release(atomic_t *v) · 3 unmodified lines
337339 #endif
338340
339341 #ifndef atomic_inc_return
340−static inline int
342+static __always_inline int
341343 atomic_inc_return(atomic_t *v)
342344 {
343345 int ret;
@@ -359,7 +361,7 @@ atomic_inc_return(atomic_t *v) · 15 unmodified lines
359361 #endif /* atomic_fetch_inc */
360362
361363 #ifndef atomic_fetch_inc
362−static inline int
364+static __always_inline int
363365 atomic_fetch_inc(atomic_t *v)
364366 {
365367 return atomic_fetch_add(1, v);
@@ -368,7 +370,7 @@ atomic_fetch_inc(atomic_t *v) · 2 unmodified lines
368370 #endif
369371
370372 #ifndef atomic_fetch_inc_acquire
371−static inline int
373+static __always_inline int
372374 atomic_fetch_inc_acquire(atomic_t *v)
373375 {
374376 return atomic_fetch_add_acquire(1, v);
@@ -377,7 +379,7 @@ atomic_fetch_inc_acquire(atomic_t *v) · 2 unmodified lines
377379 #endif
378380
379381 #ifndef atomic_fetch_inc_release
380−static inline int
382+static __always_inline int
381383 atomic_fetch_inc_release(atomic_t *v)
382384 {
383385 return atomic_fetch_add_release(1, v);
@@ -386,7 +388,7 @@ atomic_fetch_inc_release(atomic_t *v) · 2 unmodified lines
386388 #endif
387389
388390 #ifndef atomic_fetch_inc_relaxed
389−static inline int
391+static __always_inline int
390392 atomic_fetch_inc_relaxed(atomic_t *v)
391393 {
392394 return atomic_fetch_add_relaxed(1, v);
@@ -397,7 +399,7 @@ atomic_fetch_inc_relaxed(atomic_t *v) · 4 unmodified lines
397399 #else /* atomic_fetch_inc_relaxed */
398400
399401 #ifndef atomic_fetch_inc_acquire
400−static inline int
402+static __always_inline int
401403 atomic_fetch_inc_acquire(atomic_t *v)
402404 {
403405 int ret = atomic_fetch_inc_relaxed(v);
@@ -408,7 +410,7 @@ atomic_fetch_inc_acquire(atomic_t *v) · 4 unmodified lines
408410 #endif
409411
410412 #ifndef atomic_fetch_inc_release
411−static inline int
413+static __always_inline int
412414 atomic_fetch_inc_release(atomic_t *v)
413415 {
414416 __atomic_release_fence();
@@ -418,7 +420,7 @@ atomic_fetch_inc_release(atomic_t *v) · 3 unmodified lines
418420 #endif
419421
420422 #ifndef atomic_fetch_inc
421−static inline int
423+static __always_inline int
422424 atomic_fetch_inc(atomic_t *v)
423425 {
424426 int ret;
@@ -433,7 +435,7 @@ atomic_fetch_inc(atomic_t *v) · 8 unmodified lines
433435 #endif /* atomic_fetch_inc_relaxed */
434436
435437 #ifndef atomic_dec
436−static inline void
438+static __always_inline void
437439 atomic_dec(atomic_t *v)
438440 {
439441 atomic_sub(1, v);
@@ -449,7 +451,7 @@ atomic_dec(atomic_t *v) · 9 unmodified lines
449451 #endif /* atomic_dec_return */
450452
451453 #ifndef atomic_dec_return
452−static inline int
454+static __always_inline int
453455 atomic_dec_return(atomic_t *v)
454456 {
455457 return atomic_sub_return(1, v);
@@ -458,7 +460,7 @@ atomic_dec_return(atomic_t *v) · 2 unmodified lines
458460 #endif
459461
460462 #ifndef atomic_dec_return_acquire
461−static inline int
463+static __always_inline int
462464 atomic_dec_return_acquire(atomic_t *v)
463465 {
464466 return atomic_sub_return_acquire(1, v);
@@ -467,7 +469,7 @@ atomic_dec_return_acquire(atomic_t *v) · 2 unmodified lines
467469 #endif
468470
469471 #ifndef atomic_dec_return_release
470−static inline int
472+static __always_inline int
471473 atomic_dec_return_release(atomic_t *v)
472474 {
473475 return atomic_sub_return_release(1, v);
@@ -476,7 +478,7 @@ atomic_dec_return_release(atomic_t *v) · 2 unmodified lines
476478 #endif
477479
478480 #ifndef atomic_dec_return_relaxed
479−static inline int
481+static __always_inline int
480482 atomic_dec_return_relaxed(atomic_t *v)
481483 {
482484 return atomic_sub_return_relaxed(1, v);
@@ -487,7 +489,7 @@ atomic_dec_return_relaxed(atomic_t *v) · 4 unmodified lines
487489 #else /* atomic_dec_return_relaxed */
488490
489491 #ifndef atomic_dec_return_acquire
490−static inline int
492+static __always_inline int
491493 atomic_dec_return_acquire(atomic_t *v)
492494 {
493495 int ret = atomic_dec_return_relaxed(v);
@@ -498,7 +500,7 @@ atomic_dec_return_acquire(atomic_t *v) · 4 unmodified lines
498500 #endif
499501
500502 #ifndef atomic_dec_return_release
501−static inline int
503+static __always_inline int
502504 atomic_dec_return_release(atomic_t *v)
503505 {
504506 __atomic_release_fence();
@@ -508,7 +510,7 @@ atomic_dec_return_release(atomic_t *v) · 3 unmodified lines
508510 #endif
509511
510512 #ifndef atomic_dec_return
511−static inline int
513+static __always_inline int
512514 atomic_dec_return(atomic_t *v)
513515 {
514516 int ret;
@@ -530,7 +532,7 @@ atomic_dec_return(atomic_t *v) · 15 unmodified lines
530532 #endif /* atomic_fetch_dec */
531533
532534 #ifndef atomic_fetch_dec
533−static inline int
535+static __always_inline int
534536 atomic_fetch_dec(atomic_t *v)
535537 {
536538 return atomic_fetch_sub(1, v);
@@ -539,7 +541,7 @@ atomic_fetch_dec(atomic_t *v) · 2 unmodified lines
539541 #endif
540542
541543 #ifndef atomic_fetch_dec_acquire
542−static inline int
544+static __always_inline int
543545 atomic_fetch_dec_acquire(atomic_t *v)
544546 {
545547 return atomic_fetch_sub_acquire(1, v);
@@ -548,7 +550,7 @@ atomic_fetch_dec_acquire(atomic_t *v) · 2 unmodified lines
548550 #endif
549551
550552 #ifndef atomic_fetch_dec_release
551−static inline int
553+static __always_inline int
552554 atomic_fetch_dec_release(atomic_t *v)
553555 {
554556 return atomic_fetch_sub_release(1, v);
@@ -557,7 +559,7 @@ atomic_fetch_dec_release(atomic_t *v) · 2 unmodified lines
557559 #endif
558560
559561 #ifndef atomic_fetch_dec_relaxed
560−static inline int
562+static __always_inline int
561563 atomic_fetch_dec_relaxed(atomic_t *v)
562564 {
563565 return atomic_fetch_sub_relaxed(1, v);
@@ -568,7 +570,7 @@ atomic_fetch_dec_relaxed(atomic_t *v) · 4 unmodified lines
568570 #else /* atomic_fetch_dec_relaxed */
569571
570572 #ifndef atomic_fetch_dec_acquire
571−static inline int
573+static __always_inline int
572574 atomic_fetch_dec_acquire(atomic_t *v)
573575 {
574576 int ret = atomic_fetch_dec_relaxed(v);
@@ -579,7 +581,7 @@ atomic_fetch_dec_acquire(atomic_t *v) · 4 unmodified lines
579581 #endif
580582
581583 #ifndef atomic_fetch_dec_release
582−static inline int
584+static __always_inline int
583585 atomic_fetch_dec_release(atomic_t *v)
584586 {
585587 __atomic_release_fence();
@@ -589,7 +591,7 @@ atomic_fetch_dec_release(atomic_t *v) · 3 unmodified lines
589591 #endif
590592
591593 #ifndef atomic_fetch_dec
592−static inline int
594+static __always_inline int
593595 atomic_fetch_dec(atomic_t *v)
594596 {
595597 int ret;
@@ -610,7 +612,7 @@ atomic_fetch_dec(atomic_t *v) · 14 unmodified lines
610612 #else /* atomic_fetch_and_relaxed */
611613
612614 #ifndef atomic_fetch_and_acquire
613−static inline int
615+static __always_inline int
614616 atomic_fetch_and_acquire(int i, atomic_t *v)
615617 {
616618 int ret = atomic_fetch_and_relaxed(i, v);
@@ -621,7 +623,7 @@ atomic_fetch_and_acquire(int i, atomic_t *v) · 4 unmodified lines
621623 #endif
622624
623625 #ifndef atomic_fetch_and_release
624−static inline int
626+static __always_inline int
625627 atomic_fetch_and_release(int i, atomic_t *v)
626628 {
627629 __atomic_release_fence();
@@ -631,7 +633,7 @@ atomic_fetch_and_release(int i, atomic_t *v) · 3 unmodified lines
631633 #endif
632634
633635 #ifndef atomic_fetch_and
634−static inline int
636+static __always_inline int
635637 atomic_fetch_and(int i, atomic_t *v)
636638 {
637639 int ret;
@@ -646,7 +648,7 @@ atomic_fetch_and(int i, atomic_t *v) · 8 unmodified lines
646648 #endif /* atomic_fetch_and_relaxed */
647649
648650 #ifndef atomic_andnot
649−static inline void
651+static __always_inline void
650652 atomic_andnot(int i, atomic_t *v)
651653 {
652654 atomic_and(~i, v);
@@ -662,7 +664,7 @@ atomic_andnot(int i, atomic_t *v) · 9 unmodified lines
662664 #endif /* atomic_fetch_andnot */
663665
664666 #ifndef atomic_fetch_andnot
665−static inline int
667+static __always_inline int
666668 atomic_fetch_andnot(int i, atomic_t *v)
667669 {
668670 return atomic_fetch_and(~i, v);
@@ -671,7 +673,7 @@ atomic_fetch_andnot(int i, atomic_t *v) · 2 unmodified lines
671673 #endif
672674
673675 #ifndef atomic_fetch_andnot_acquire
674−static inline int
676+static __always_inline int
675677 atomic_fetch_andnot_acquire(int i, atomic_t *v)
676678 {
677679 return atomic_fetch_and_acquire(~i, v);
@@ -680,7 +682,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) · 2 unmodified lines
680682 #endif
681683
682684 #ifndef atomic_fetch_andnot_release
683−static inline int
685+static __always_inline int
684686 atomic_fetch_andnot_release(int i, atomic_t *v)
685687 {
686688 return atomic_fetch_and_release(~i, v);
@@ -689,7 +691,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) · 2 unmodified lines
689691 #endif
690692
691693 #ifndef atomic_fetch_andnot_relaxed
692−static inline int
694+static __always_inline int
693695 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
694696 {
695697 return atomic_fetch_and_relaxed(~i, v);
@@ -700,7 +702,7 @@ atomic_fetch_andnot_relaxed(int i, atomic_t *v) · 4 unmodified lines
700702 #else /* atomic_fetch_andnot_relaxed */
701703
702704 #ifndef atomic_fetch_andnot_acquire
703−static inline int
705+static __always_inline int
704706 atomic_fetch_andnot_acquire(int i, atomic_t *v)
705707 {
706708 int ret = atomic_fetch_andnot_relaxed(i, v);
@@ -711,7 +713,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) · 4 unmodified lines
711713 #endif
712714
713715 #ifndef atomic_fetch_andnot_release
714−static inline int
716+static __always_inline int
715717 atomic_fetch_andnot_release(int i, atomic_t *v)
716718 {
717719 __atomic_release_fence();
@@ -721,7 +723,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) · 3 unmodified lines
721723 #endif
722724
723725 #ifndef atomic_fetch_andnot
724−static inline int
726+static __always_inline int
725727 atomic_fetch_andnot(int i, atomic_t *v)
726728 {
727729 int ret;
@@ -742,7 +744,7 @@ atomic_fetch_andnot(int i, atomic_t *v) · 14 unmodified lines
742744 #else /* atomic_fetch_or_relaxed */
743745
744746 #ifndef atomic_fetch_or_acquire
745−static inline int
747+static __always_inline int
746748 atomic_fetch_or_acquire(int i, atomic_t *v)
747749 {
748750 int ret = atomic_fetch_or_relaxed(i, v);
@@ -753,7 +755,7 @@ atomic_fetch_or_acquire(int i, atomic_t *v) · 4 unmodified lines
753755 #endif
754756
755757 #ifndef atomic_fetch_or_release
756−static inline int
758+static __always_inline int
757759 atomic_fetch_or_release(int i, atomic_t *v)
758760 {
759761 __atomic_release_fence();
@@ -763,7 +765,7 @@ atomic_fetch_or_release(int i, atomic_t *v) · 3 unmodified lines
763765 #endif
764766
765767 #ifndef atomic_fetch_or
766−static inline int
768+static __always_inline int
767769 atomic_fetch_or(int i, atomic_t *v)
768770 {
769771 int ret;
@@ -784,7 +786,7 @@ atomic_fetch_or(int i, atomic_t *v) · 14 unmodified lines
784786 #else /* atomic_fetch_xor_relaxed */
785787
786788 #ifndef atomic_fetch_xor_acquire
787−static inline int
789+static __always_inline int
788790 atomic_fetch_xor_acquire(int i, atomic_t *v)
789791 {
790792 int ret = atomic_fetch_xor_relaxed(i, v);
@@ -795,7 +797,7 @@ atomic_fetch_xor_acquire(int i, atomic_t *v) · 4 unmodified lines
795797 #endif
796798
797799 #ifndef atomic_fetch_xor_release
798−static inline int
800+static __always_inline int
799801 atomic_fetch_xor_release(int i, atomic_t *v)
800802 {
801803 __atomic_release_fence();
@@ -805,7 +807,7 @@ atomic_fetch_xor_release(int i, atomic_t *v) · 3 unmodified lines
805807 #endif
806808
807809 #ifndef atomic_fetch_xor
808−static inline int
810+static __always_inline int
809811 atomic_fetch_xor(int i, atomic_t *v)
810812 {
811813 int ret;
@@ -826,7 +828,7 @@ atomic_fetch_xor(int i, atomic_t *v) · 14 unmodified lines
826828 #else /* atomic_xchg_relaxed */
827829
828830 #ifndef atomic_xchg_acquire
829−static inline int
831+static __always_inline int
830832 atomic_xchg_acquire(atomic_t *v, int i)
831833 {
832834 int ret = atomic_xchg_relaxed(v, i);
@@ -837,7 +839,7 @@ atomic_xchg_acquire(atomic_t *v, int i) · 4 unmodified lines
837839 #endif
838840
839841 #ifndef atomic_xchg_release
840−static inline int
842+static __always_inline int
841843 atomic_xchg_release(atomic_t *v, int i)
842844 {
843845 __atomic_release_fence();
@@ -847,7 +849,7 @@ atomic_xchg_release(atomic_t *v, int i) · 3 unmodified lines
847849 #endif
848850
849851 #ifndef atomic_xchg
850−static inline int
852+static __always_inline int
851853 atomic_xchg(atomic_t *v, int i)
852854 {
853855 int ret;
@@ -868,7 +870,7 @@ atomic_xchg(atomic_t *v, int i) · 14 unmodified lines
868870 #else /* atomic_cmpxchg_relaxed */
869871
870872 #ifndef atomic_cmpxchg_acquire
871−static inline int
873+static __always_inline int
872874 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
873875 {
874876 int ret = atomic_cmpxchg_relaxed(v, old, new);
@@ -879,7 +881,7 @@ atomic_cmpxchg_acquire(atomic_t *v, int old, int new) · 4 unmodified lines
879881 #endif
880882
881883 #ifndef atomic_cmpxchg_release
882−static inline int
884+static __always_inline int
883885 atomic_cmpxchg_release(atomic_t *v, int old, int new)
884886 {
885887 __atomic_release_fence();
@@ -889,7 +891,7 @@ atomic_cmpxchg_release(atomic_t *v, int old, int new) · 3 unmodified lines
889891 #endif
890892
891893 #ifndef atomic_cmpxchg
892−static inline int
894+static __always_inline int
893895 atomic_cmpxchg(atomic_t *v, int old, int new)
894896 {
895897 int ret;
@@ -911,7 +913,7 @@ atomic_cmpxchg(atomic_t *v, int old, int new) · 15 unmodified lines
911913 #endif /* atomic_try_cmpxchg */
912914
913915 #ifndef atomic_try_cmpxchg
914−static inline bool
916+static __always_inline bool
915917 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
916918 {
917919 int r, o = *old;
@@ -924,7 +926,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) · 6 unmodified lines
924926 #endif
925927
926928 #ifndef atomic_try_cmpxchg_acquire
927−static inline bool
929+static __always_inline bool
928930 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
929931 {
930932 int r, o = *old;
@@ -937,7 +939,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) · 6 unmodified lines
937939 #endif
938940
939941 #ifndef atomic_try_cmpxchg_release
940−static inline bool
942+static __always_inline bool
941943 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
942944 {
943945 int r, o = *old;
@@ -950,7 +952,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) · 6 unmodified lines
950952 #endif
951953
952954 #ifndef atomic_try_cmpxchg_relaxed
953−static inline bool
955+static __always_inline bool
954956 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
955957 {
956958 int r, o = *old;
@@ -965,7 +967,7 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) · 8 unmodified lines
965967 #else /* atomic_try_cmpxchg_relaxed */
966968
967969 #ifndef atomic_try_cmpxchg_acquire
968−static inline bool
970+static __always_inline bool
969971 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
970972 {
971973 bool ret = atomic_try_cmpxchg_relaxed(v, old, new);
@@ -976,7 +978,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) · 4 unmodified lines
976978 #endif
977979
978980 #ifndef atomic_try_cmpxchg_release
979−static inline bool
981+static __always_inline bool
980982 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
981983 {
982984 __atomic_release_fence();
@@ -986,7 +988,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) · 3 unmodified lines
986988 #endif
987989
988990 #ifndef atomic_try_cmpxchg
989−static inline bool
991+static __always_inline bool
990992 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
991993 {
992994 bool ret;
@@ -1010,7 +1012,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) · 17 unmodified lines
10101012 * true if the result is zero, or false for all
10111013 * other cases.
10121014 */
1013−static inline bool
1015+static __always_inline bool
10141016 atomic_sub_and_test(int i, atomic_t *v)
10151017 {
10161018 return atomic_sub_return(i, v) == 0;
@@ -1027,7 +1029,7 @@ atomic_sub_and_test(int i, atomic_t *v) · 10 unmodified lines
10271029 * returns true if the result is 0, or false for all other
10281030 * cases.
10291031 */
1030−static inline bool
1032+static __always_inline bool
10311033 atomic_dec_and_test(atomic_t *v)
10321034 {
10331035 return atomic_dec_return(v) == 0;
@@ -1044,7 +1046,7 @@ atomic_dec_and_test(atomic_t *v) · 10 unmodified lines
10441046 * and returns true if the result is zero, or false for all
10451047 * other cases.
10461048 */
1047−static inline bool
1049+static __always_inline bool
10481050 atomic_inc_and_test(atomic_t *v)
10491051 {
10501052 return atomic_inc_return(v) == 0;
@@ -1062,7 +1064,7 @@ atomic_inc_and_test(atomic_t *v) · 11 unmodified lines
10621064 * if the result is negative, or false when
10631065 * result is greater than or equal to zero.
10641066 */
1065−static inline bool
1067+static __always_inline bool
10661068 atomic_add_negative(int i, atomic_t *v)
10671069 {
10681070 return atomic_add_return(i, v) < 0;
@@ -1080,7 +1082,7 @@ atomic_add_negative(int i, atomic_t *v) · 11 unmodified lines
10801082 * Atomically adds @a to @v, so long as @v was not already @u.
10811083 * Returns original value of @v
10821084 */
1083−static inline int
1085+static __always_inline int
10841086 atomic_fetch_add_unless(atomic_t *v, int a, int u)
10851087 {
10861088 int c = atomic_read(v);
@@ -1105,7 +1107,7 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) · 18 unmodified lines
11051107 * Atomically adds @a to @v, if @v was not already @u.
11061108 * Returns true if the addition was done.
11071109 */
1108−static inline bool
1110+static __always_inline bool
11091111 atomic_add_unless(atomic_t *v, int a, int u)
11101112 {
11111113 return atomic_fetch_add_unless(v, a, u) != u;
@@ -1121,7 +1123,7 @@ atomic_add_unless(atomic_t *v, int a, int u) · 9 unmodified lines
11211123 * Atomically increments @v by 1, if @v is non-zero.
11221124 * Returns true if the increment was done.
11231125 */
1124−static inline bool
1126+static __always_inline bool
11251127 atomic_inc_not_zero(atomic_t *v)
11261128 {
11271129 return atomic_add_unless(v, 1, 0);
@@ -1130,7 +1132,7 @@ atomic_inc_not_zero(atomic_t *v) · 2 unmodified lines
11301132 #endif
11311133
11321134 #ifndef atomic_inc_unless_negative
1133−static inline bool
1135+static __always_inline bool
11341136 atomic_inc_unless_negative(atomic_t *v)
11351137 {
11361138 int c = atomic_read(v);
@@ -1146,7 +1148,7 @@ atomic_inc_unless_negative(atomic_t *v) · 9 unmodified lines
11461148 #endif
11471149
11481150 #ifndef atomic_dec_unless_positive
1149−static inline bool
1151+static __always_inline bool
11501152 atomic_dec_unless_positive(atomic_t *v)
11511153 {
11521154 int c = atomic_read(v);
@@ -1162,7 +1164,7 @@ atomic_dec_unless_positive(atomic_t *v) · 9 unmodified lines
11621164 #endif
11631165
11641166 #ifndef atomic_dec_if_positive
1165−static inline int
1167+static __always_inline int
11661168 atomic_dec_if_positive(atomic_t *v)
11671169 {
11681170 int dec, c = atomic_read(v);
@@ -1178,15 +1180,12 @@ atomic_dec_if_positive(atomic_t *v) · 9 unmodified lines
11781180 #define atomic_dec_if_positive atomic_dec_if_positive
11791181 #endif
11801182
1181−#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
1182−#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
1183−
11841183 #ifdef CONFIG_GENERIC_ATOMIC64
11851184 #include <asm-generic/atomic64.h>
11861185 #endif
11871186
11881187 #ifndef atomic64_read_acquire
1189−static inline s64
1188+static __always_inline s64
11901189 atomic64_read_acquire(const atomic64_t *v)
11911190 {
11921191 return smp_load_acquire(&(v)->counter);
@@ -1195,7 +1194,7 @@ atomic64_read_acquire(const atomic64_t *v) · 2 unmodified lines
11951194 #endif
11961195
11971196 #ifndef atomic64_set_release
1198−static inline void
1197+static __always_inline void
11991198 atomic64_set_release(atomic64_t *v, s64 i)
12001199 {
12011200 smp_store_release(&(v)->counter, i);
@@ -1210,7 +1209,7 @@ atomic64_set_release(atomic64_t *v, s64 i) · 8 unmodified lines
12101209 #else /* atomic64_add_return_relaxed */
12111210
12121211 #ifndef atomic64_add_return_acquire
1213−static inline s64
1212+static __always_inline s64
12141213 atomic64_add_return_acquire(s64 i, atomic64_t *v)
12151214 {
12161215 s64 ret = atomic64_add_return_relaxed(i, v);
@@ -1221,7 +1220,7 @@ atomic64_add_return_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
12211220 #endif
12221221
12231222 #ifndef atomic64_add_return_release
1224−static inline s64
1223+static __always_inline s64
12251224 atomic64_add_return_release(s64 i, atomic64_t *v)
12261225 {
12271226 __atomic_release_fence();
@@ -1231,7 +1230,7 @@ atomic64_add_return_release(s64 i, atomic64_t *v) · 3 unmodified lines
12311230 #endif
12321231
12331232 #ifndef atomic64_add_return
1234−static inline s64
1233+static __always_inline s64
12351234 atomic64_add_return(s64 i, atomic64_t *v)
12361235 {
12371236 s64 ret;
@@ -1252,7 +1251,7 @@ atomic64_add_return(s64 i, atomic64_t *v) · 14 unmodified lines
12521251 #else /* atomic64_fetch_add_relaxed */
12531252
12541253 #ifndef atomic64_fetch_add_acquire
1255−static inline s64
1254+static __always_inline s64
12561255 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
12571256 {
12581257 s64 ret = atomic64_fetch_add_relaxed(i, v);
@@ -1263,7 +1262,7 @@ atomic64_fetch_add_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
12631262 #endif
12641263
12651264 #ifndef atomic64_fetch_add_release
1266−static inline s64
1265+static __always_inline s64
12671266 atomic64_fetch_add_release(s64 i, atomic64_t *v)
12681267 {
12691268 __atomic_release_fence();
@@ -1273,7 +1272,7 @@ atomic64_fetch_add_release(s64 i, atomic64_t *v) · 3 unmodified lines
12731272 #endif
12741273
12751274 #ifndef atomic64_fetch_add
1276−static inline s64
1275+static __always_inline s64
12771276 atomic64_fetch_add(s64 i, atomic64_t *v)
12781277 {
12791278 s64 ret;
@@ -1294,7 +1293,7 @@ atomic64_fetch_add(s64 i, atomic64_t *v) · 14 unmodified lines
12941293 #else /* atomic64_sub_return_relaxed */
12951294
12961295 #ifndef atomic64_sub_return_acquire
1297−static inline s64
1296+static __always_inline s64
12981297 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
12991298 {
13001299 s64 ret = atomic64_sub_return_relaxed(i, v);
@@ -1305,7 +1304,7 @@ atomic64_sub_return_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
13051304 #endif
13061305
13071306 #ifndef atomic64_sub_return_release
1308−static inline s64
1307+static __always_inline s64
13091308 atomic64_sub_return_release(s64 i, atomic64_t *v)
13101309 {
13111310 __atomic_release_fence();
@@ -1315,7 +1314,7 @@ atomic64_sub_return_release(s64 i, atomic64_t *v) · 3 unmodified lines
13151314 #endif
13161315
13171316 #ifndef atomic64_sub_return
1318−static inline s64
1317+static __always_inline s64
13191318 atomic64_sub_return(s64 i, atomic64_t *v)
13201319 {
13211320 s64 ret;
@@ -1336,7 +1335,7 @@ atomic64_sub_return(s64 i, atomic64_t *v) · 14 unmodified lines
13361335 #else /* atomic64_fetch_sub_relaxed */
13371336
13381337 #ifndef atomic64_fetch_sub_acquire
1339−static inline s64
1338+static __always_inline s64
13401339 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
13411340 {
13421341 s64 ret = atomic64_fetch_sub_relaxed(i, v);
@@ -1347,7 +1346,7 @@ atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
13471346 #endif
13481347
13491348 #ifndef atomic64_fetch_sub_release
1350−static inline s64
1349+static __always_inline s64
13511350 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
13521351 {
13531352 __atomic_release_fence();
@@ -1357,7 +1356,7 @@ atomic64_fetch_sub_release(s64 i, atomic64_t *v) · 3 unmodified lines
13571356 #endif
13581357
13591358 #ifndef atomic64_fetch_sub
1360−static inline s64
1359+static __always_inline s64
13611360 atomic64_fetch_sub(s64 i, atomic64_t *v)
13621361 {
13631362 s64 ret;
@@ -1372,7 +1371,7 @@ atomic64_fetch_sub(s64 i, atomic64_t *v) · 8 unmodified lines
13721371 #endif /* atomic64_fetch_sub_relaxed */
13731372
13741373 #ifndef atomic64_inc
1375−static inline void
1374+static __always_inline void
13761375 atomic64_inc(atomic64_t *v)
13771376 {
13781377 atomic64_add(1, v);
@@ -1388,7 +1387,7 @@ atomic64_inc(atomic64_t *v) · 9 unmodified lines
13881387 #endif /* atomic64_inc_return */
13891388
13901389 #ifndef atomic64_inc_return
1391−static inline s64
1390+static __always_inline s64
13921391 atomic64_inc_return(atomic64_t *v)
13931392 {
13941393 return atomic64_add_return(1, v);
@@ -1397,7 +1396,7 @@ atomic64_inc_return(atomic64_t *v) · 2 unmodified lines
13971396 #endif
13981397
13991398 #ifndef atomic64_inc_return_acquire
1400−static inline s64
1399+static __always_inline s64
14011400 atomic64_inc_return_acquire(atomic64_t *v)
14021401 {
14031402 return atomic64_add_return_acquire(1, v);
@@ -1406,7 +1405,7 @@ atomic64_inc_return_acquire(atomic64_t *v) · 2 unmodified lines
14061405 #endif
14071406
14081407 #ifndef atomic64_inc_return_release
1409−static inline s64
1408+static __always_inline s64
14101409 atomic64_inc_return_release(atomic64_t *v)
14111410 {
14121411 return atomic64_add_return_release(1, v);
@@ -1415,7 +1414,7 @@ atomic64_inc_return_release(atomic64_t *v) · 2 unmodified lines
14151414 #endif
14161415
14171416 #ifndef atomic64_inc_return_relaxed
1418−static inline s64
1417+static __always_inline s64
14191418 atomic64_inc_return_relaxed(atomic64_t *v)
14201419 {
14211420 return atomic64_add_return_relaxed(1, v);
@@ -1426,7 +1425,7 @@ atomic64_inc_return_relaxed(atomic64_t *v) · 4 unmodified lines
14261425 #else /* atomic64_inc_return_relaxed */
14271426
14281427 #ifndef atomic64_inc_return_acquire
1429−static inline s64
1428+static __always_inline s64
14301429 atomic64_inc_return_acquire(atomic64_t *v)
14311430 {
14321431 s64 ret = atomic64_inc_return_relaxed(v);
@@ -1437,7 +1436,7 @@ atomic64_inc_return_acquire(atomic64_t *v) · 4 unmodified lines
14371436 #endif
14381437
14391438 #ifndef atomic64_inc_return_release
1440−static inline s64
1439+static __always_inline s64
14411440 atomic64_inc_return_release(atomic64_t *v)
14421441 {
14431442 __atomic_release_fence();
@@ -1447,7 +1446,7 @@ atomic64_inc_return_release(atomic64_t *v) · 3 unmodified lines
14471446 #endif
14481447
14491448 #ifndef atomic64_inc_return
1450−static inline s64
1449+static __always_inline s64
14511450 atomic64_inc_return(atomic64_t *v)
14521451 {
14531452 s64 ret;
@@ -1469,7 +1468,7 @@ atomic64_inc_return(atomic64_t *v) · 15 unmodified lines
14691468 #endif /* atomic64_fetch_inc */
14701469
14711470 #ifndef atomic64_fetch_inc
1472−static inline s64
1471+static __always_inline s64
14731472 atomic64_fetch_inc(atomic64_t *v)
14741473 {
14751474 return atomic64_fetch_add(1, v);
@@ -1478,7 +1477,7 @@ atomic64_fetch_inc(atomic64_t *v) · 2 unmodified lines
14781477 #endif
14791478
14801479 #ifndef atomic64_fetch_inc_acquire
1481−static inline s64
1480+static __always_inline s64
14821481 atomic64_fetch_inc_acquire(atomic64_t *v)
14831482 {
14841483 return atomic64_fetch_add_acquire(1, v);
@@ -1487,7 +1486,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) · 2 unmodified lines
14871486 #endif
14881487
14891488 #ifndef atomic64_fetch_inc_release
1490−static inline s64
1489+static __always_inline s64
14911490 atomic64_fetch_inc_release(atomic64_t *v)
14921491 {
14931492 return atomic64_fetch_add_release(1, v);
@@ -1496,7 +1495,7 @@ atomic64_fetch_inc_release(atomic64_t *v) · 2 unmodified lines
14961495 #endif
14971496
14981497 #ifndef atomic64_fetch_inc_relaxed
1499−static inline s64
1498+static __always_inline s64
15001499 atomic64_fetch_inc_relaxed(atomic64_t *v)
15011500 {
15021501 return atomic64_fetch_add_relaxed(1, v);
@@ -1507,7 +1506,7 @@ atomic64_fetch_inc_relaxed(atomic64_t *v) · 4 unmodified lines
15071506 #else /* atomic64_fetch_inc_relaxed */
15081507
15091508 #ifndef atomic64_fetch_inc_acquire
1510−static inline s64
1509+static __always_inline s64
15111510 atomic64_fetch_inc_acquire(atomic64_t *v)
15121511 {
15131512 s64 ret = atomic64_fetch_inc_relaxed(v);
@@ -1518,7 +1517,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) · 4 unmodified lines
15181517 #endif
15191518
15201519 #ifndef atomic64_fetch_inc_release
1521−static inline s64
1520+static __always_inline s64
15221521 atomic64_fetch_inc_release(atomic64_t *v)
15231522 {
15241523 __atomic_release_fence();
@@ -1528,7 +1527,7 @@ atomic64_fetch_inc_release(atomic64_t *v) · 3 unmodified lines
15281527 #endif
15291528
15301529 #ifndef atomic64_fetch_inc
1531−static inline s64
1530+static __always_inline s64
15321531 atomic64_fetch_inc(atomic64_t *v)
15331532 {
15341533 s64 ret;
@@ -1543,7 +1542,7 @@ atomic64_fetch_inc(atomic64_t *v) · 8 unmodified lines
15431542 #endif /* atomic64_fetch_inc_relaxed */
15441543
15451544 #ifndef atomic64_dec
1546−static inline void
1545+static __always_inline void
15471546 atomic64_dec(atomic64_t *v)
15481547 {
15491548 atomic64_sub(1, v);
@@ -1559,7 +1558,7 @@ atomic64_dec(atomic64_t *v) · 9 unmodified lines
15591558 #endif /* atomic64_dec_return */
15601559
15611560 #ifndef atomic64_dec_return
1562−static inline s64
1561+static __always_inline s64
15631562 atomic64_dec_return(atomic64_t *v)
15641563 {
15651564 return atomic64_sub_return(1, v);
@@ -1568,7 +1567,7 @@ atomic64_dec_return(atomic64_t *v) · 2 unmodified lines
15681567 #endif
15691568
15701569 #ifndef atomic64_dec_return_acquire
1571−static inline s64
1570+static __always_inline s64
15721571 atomic64_dec_return_acquire(atomic64_t *v)
15731572 {
15741573 return atomic64_sub_return_acquire(1, v);
@@ -1577,7 +1576,7 @@ atomic64_dec_return_acquire(atomic64_t *v) · 2 unmodified lines
15771576 #endif
15781577
15791578 #ifndef atomic64_dec_return_release
1580−static inline s64
1579+static __always_inline s64
15811580 atomic64_dec_return_release(atomic64_t *v)
15821581 {
15831582 return atomic64_sub_return_release(1, v);
@@ -1586,7 +1585,7 @@ atomic64_dec_return_release(atomic64_t *v) · 2 unmodified lines
15861585 #endif
15871586
15881587 #ifndef atomic64_dec_return_relaxed
1589−static inline s64
1588+static __always_inline s64
15901589 atomic64_dec_return_relaxed(atomic64_t *v)
15911590 {
15921591 return atomic64_sub_return_relaxed(1, v);
@@ -1597,7 +1596,7 @@ atomic64_dec_return_relaxed(atomic64_t *v) · 4 unmodified lines
15971596 #else /* atomic64_dec_return_relaxed */
15981597
15991598 #ifndef atomic64_dec_return_acquire
1600−static inline s64
1599+static __always_inline s64
16011600 atomic64_dec_return_acquire(atomic64_t *v)
16021601 {
16031602 s64 ret = atomic64_dec_return_relaxed(v);
@@ -1608,7 +1607,7 @@ atomic64_dec_return_acquire(atomic64_t *v) · 4 unmodified lines
16081607 #endif
16091608
16101609 #ifndef atomic64_dec_return_release
1611−static inline s64
1610+static __always_inline s64
16121611 atomic64_dec_return_release(atomic64_t *v)
16131612 {
16141613 __atomic_release_fence();
@@ -1618,7 +1617,7 @@ atomic64_dec_return_release(atomic64_t *v) · 3 unmodified lines
16181617 #endif
16191618
16201619 #ifndef atomic64_dec_return
1621−static inline s64
1620+static __always_inline s64
16221621 atomic64_dec_return(atomic64_t *v)
16231622 {
16241623 s64 ret;
@@ -1640,7 +1639,7 @@ atomic64_dec_return(atomic64_t *v) · 15 unmodified lines
16401639 #endif /* atomic64_fetch_dec */
16411640
16421641 #ifndef atomic64_fetch_dec
1643−static inline s64
1642+static __always_inline s64
16441643 atomic64_fetch_dec(atomic64_t *v)
16451644 {
16461645 return atomic64_fetch_sub(1, v);
@@ -1649,7 +1648,7 @@ atomic64_fetch_dec(atomic64_t *v) · 2 unmodified lines
16491648 #endif
16501649
16511650 #ifndef atomic64_fetch_dec_acquire
1652−static inline s64
1651+static __always_inline s64
16531652 atomic64_fetch_dec_acquire(atomic64_t *v)
16541653 {
16551654 return atomic64_fetch_sub_acquire(1, v);
@@ -1658,7 +1657,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) · 2 unmodified lines
16581657 #endif
16591658
16601659 #ifndef atomic64_fetch_dec_release
1661−static inline s64
1660+static __always_inline s64
16621661 atomic64_fetch_dec_release(atomic64_t *v)
16631662 {
16641663 return atomic64_fetch_sub_release(1, v);
@@ -1667,7 +1666,7 @@ atomic64_fetch_dec_release(atomic64_t *v) · 2 unmodified lines
16671666 #endif
16681667
16691668 #ifndef atomic64_fetch_dec_relaxed
1670−static inline s64
1669+static __always_inline s64
16711670 atomic64_fetch_dec_relaxed(atomic64_t *v)
16721671 {
16731672 return atomic64_fetch_sub_relaxed(1, v);
@@ -1678,7 +1677,7 @@ atomic64_fetch_dec_relaxed(atomic64_t *v) · 4 unmodified lines
16781677 #else /* atomic64_fetch_dec_relaxed */
16791678
16801679 #ifndef atomic64_fetch_dec_acquire
1681−static inline s64
1680+static __always_inline s64
16821681 atomic64_fetch_dec_acquire(atomic64_t *v)
16831682 {
16841683 s64 ret = atomic64_fetch_dec_relaxed(v);
@@ -1689,7 +1688,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) · 4 unmodified lines
16891688 #endif
16901689
16911690 #ifndef atomic64_fetch_dec_release
1692−static inline s64
1691+static __always_inline s64
16931692 atomic64_fetch_dec_release(atomic64_t *v)
16941693 {
16951694 __atomic_release_fence();
@@ -1699,7 +1698,7 @@ atomic64_fetch_dec_release(atomic64_t *v) · 3 unmodified lines
16991698 #endif
17001699
17011700 #ifndef atomic64_fetch_dec
1702−static inline s64
1701+static __always_inline s64
17031702 atomic64_fetch_dec(atomic64_t *v)
17041703 {
17051704 s64 ret;
@@ -1720,7 +1719,7 @@ atomic64_fetch_dec(atomic64_t *v) · 14 unmodified lines
17201719 #else /* atomic64_fetch_and_relaxed */
17211720
17221721 #ifndef atomic64_fetch_and_acquire
1723−static inline s64
1722+static __always_inline s64
17241723 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
17251724 {
17261725 s64 ret = atomic64_fetch_and_relaxed(i, v);
@@ -1731,7 +1730,7 @@ atomic64_fetch_and_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
17311730 #endif
17321731
17331732 #ifndef atomic64_fetch_and_release
1734−static inline s64
1733+static __always_inline s64
17351734 atomic64_fetch_and_release(s64 i, atomic64_t *v)
17361735 {
17371736 __atomic_release_fence();
@@ -1741,7 +1740,7 @@ atomic64_fetch_and_release(s64 i, atomic64_t *v) · 3 unmodified lines
17411740 #endif
17421741
17431742 #ifndef atomic64_fetch_and
1744−static inline s64
1743+static __always_inline s64
17451744 atomic64_fetch_and(s64 i, atomic64_t *v)
17461745 {
17471746 s64 ret;
@@ -1756,7 +1755,7 @@ atomic64_fetch_and(s64 i, atomic64_t *v) · 8 unmodified lines
17561755 #endif /* atomic64_fetch_and_relaxed */
17571756
17581757 #ifndef atomic64_andnot
1759−static inline void
1758+static __always_inline void
17601759 atomic64_andnot(s64 i, atomic64_t *v)
17611760 {
17621761 atomic64_and(~i, v);
@@ -1772,7 +1771,7 @@ atomic64_andnot(s64 i, atomic64_t *v) · 9 unmodified lines
17721771 #endif /* atomic64_fetch_andnot */
17731772
17741773 #ifndef atomic64_fetch_andnot
1775−static inline s64
1774+static __always_inline s64
17761775 atomic64_fetch_andnot(s64 i, atomic64_t *v)
17771776 {
17781777 return atomic64_fetch_and(~i, v);
@@ -1781,7 +1780,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) · 2 unmodified lines
17811780 #endif
17821781
17831782 #ifndef atomic64_fetch_andnot_acquire
1784−static inline s64
1783+static __always_inline s64
17851784 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
17861785 {
17871786 return atomic64_fetch_and_acquire(~i, v);
@@ -1790,7 +1789,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) · 2 unmodified lines
17901789 #endif
17911790
17921791 #ifndef atomic64_fetch_andnot_release
1793−static inline s64
1792+static __always_inline s64
17941793 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
17951794 {
17961795 return atomic64_fetch_and_release(~i, v);
@@ -1799,7 +1798,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) · 2 unmodified lines
17991798 #endif
18001799
18011800 #ifndef atomic64_fetch_andnot_relaxed
1802−static inline s64
1801+static __always_inline s64
18031802 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
18041803 {
18051804 return atomic64_fetch_and_relaxed(~i, v);
@@ -1810,7 +1809,7 @@ atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) · 4 unmodified lines
18101809 #else /* atomic64_fetch_andnot_relaxed */
18111810
18121811 #ifndef atomic64_fetch_andnot_acquire
1813−static inline s64
1812+static __always_inline s64
18141813 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
18151814 {
18161815 s64 ret = atomic64_fetch_andnot_relaxed(i, v);
@@ -1821,7 +1820,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
18211820 #endif
18221821
18231822 #ifndef atomic64_fetch_andnot_release
1824−static inline s64
1823+static __always_inline s64
18251824 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
18261825 {
18271826 __atomic_release_fence();
@@ -1831,7 +1830,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) · 3 unmodified lines
18311830 #endif
18321831
18331832 #ifndef atomic64_fetch_andnot
1834−static inline s64
1833+static __always_inline s64
18351834 atomic64_fetch_andnot(s64 i, atomic64_t *v)
18361835 {
18371836 s64 ret;
@@ -1852,7 +1851,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) · 14 unmodified lines
18521851 #else /* atomic64_fetch_or_relaxed */
18531852
18541853 #ifndef atomic64_fetch_or_acquire
1855−static inline s64
1854+static __always_inline s64
18561855 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
18571856 {
18581857 s64 ret = atomic64_fetch_or_relaxed(i, v);
@@ -1863,7 +1862,7 @@ atomic64_fetch_or_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
18631862 #endif
18641863
18651864 #ifndef atomic64_fetch_or_release
1866−static inline s64
1865+static __always_inline s64
18671866 atomic64_fetch_or_release(s64 i, atomic64_t *v)
18681867 {
18691868 __atomic_release_fence();
@@ -1873,7 +1872,7 @@ atomic64_fetch_or_release(s64 i, atomic64_t *v) · 3 unmodified lines
18731872 #endif
18741873
18751874 #ifndef atomic64_fetch_or
1876−static inline s64
1875+static __always_inline s64
18771876 atomic64_fetch_or(s64 i, atomic64_t *v)
18781877 {
18791878 s64 ret;
@@ -1894,7 +1893,7 @@ atomic64_fetch_or(s64 i, atomic64_t *v) · 14 unmodified lines
18941893 #else /* atomic64_fetch_xor_relaxed */
18951894
18961895 #ifndef atomic64_fetch_xor_acquire
1897−static inline s64
1896+static __always_inline s64
18981897 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
18991898 {
19001899 s64 ret = atomic64_fetch_xor_relaxed(i, v);
@@ -1905,7 +1904,7 @@ atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
19051904 #endif
19061905
19071906 #ifndef atomic64_fetch_xor_release
1908−static inline s64
1907+static __always_inline s64
19091908 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
19101909 {
19111910 __atomic_release_fence();
@@ -1915,7 +1914,7 @@ atomic64_fetch_xor_release(s64 i, atomic64_t *v) · 3 unmodified lines
19151914 #endif
19161915
19171916 #ifndef atomic64_fetch_xor
1918−static inline s64
1917+static __always_inline s64
19191918 atomic64_fetch_xor(s64 i, atomic64_t *v)
19201919 {
19211920 s64 ret;
@@ -1936,7 +1935,7 @@ atomic64_fetch_xor(s64 i, atomic64_t *v) · 14 unmodified lines
19361935 #else /* atomic64_xchg_relaxed */
19371936
19381937 #ifndef atomic64_xchg_acquire
1939−static inline s64
1938+static __always_inline s64
19401939 atomic64_xchg_acquire(atomic64_t *v, s64 i)
19411940 {
19421941 s64 ret = atomic64_xchg_relaxed(v, i);
@@ -1947,7 +1946,7 @@ atomic64_xchg_acquire(atomic64_t *v, s64 i) · 4 unmodified lines
19471946 #endif
19481947
19491948 #ifndef atomic64_xchg_release
1950−static inline s64
1949+static __always_inline s64
19511950 atomic64_xchg_release(atomic64_t *v, s64 i)
19521951 {
19531952 __atomic_release_fence();
@@ -1957,7 +1956,7 @@ atomic64_xchg_release(atomic64_t *v, s64 i) · 3 unmodified lines
19571956 #endif
19581957
19591958 #ifndef atomic64_xchg
1960−static inline s64
1959+static __always_inline s64
19611960 atomic64_xchg(atomic64_t *v, s64 i)
19621961 {
19631962 s64 ret;
@@ -1978,7 +1977,7 @@ atomic64_xchg(atomic64_t *v, s64 i) · 14 unmodified lines
19781977 #else /* atomic64_cmpxchg_relaxed */
19791978
19801979 #ifndef atomic64_cmpxchg_acquire
1981−static inline s64
1980+static __always_inline s64
19821981 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
19831982 {
19841983 s64 ret = atomic64_cmpxchg_relaxed(v, old, new);
@@ -1989,7 +1988,7 @@ atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) · 4 unmodified lines
19891988 #endif
19901989
19911990 #ifndef atomic64_cmpxchg_release
1992−static inline s64
1991+static __always_inline s64
19931992 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
19941993 {
19951994 __atomic_release_fence();
@@ -1999,7 +1998,7 @@ atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) · 3 unmodified lines
19991998 #endif
20001999
20012000 #ifndef atomic64_cmpxchg
2002−static inline s64
2001+static __always_inline s64
20032002 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
20042003 {
20052004 s64 ret;
@@ -2021,7 +2020,7 @@ atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) · 15 unmodified lines
20212020 #endif /* atomic64_try_cmpxchg */
20222021
20232022 #ifndef atomic64_try_cmpxchg
2024−static inline bool
2023+static __always_inline bool
20252024 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
20262025 {
20272026 s64 r, o = *old;
@@ -2034,7 +2033,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
20342033 #endif
20352034
20362035 #ifndef atomic64_try_cmpxchg_acquire
2037−static inline bool
2036+static __always_inline bool
20382037 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
20392038 {
20402039 s64 r, o = *old;
@@ -2047,7 +2046,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
20472046 #endif
20482047
20492048 #ifndef atomic64_try_cmpxchg_release
2050−static inline bool
2049+static __always_inline bool
20512050 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
20522051 {
20532052 s64 r, o = *old;
@@ -2060,7 +2059,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
20602059 #endif
20612060
20622061 #ifndef atomic64_try_cmpxchg_relaxed
2063−static inline bool
2062+static __always_inline bool
20642063 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
20652064 {
20662065 s64 r, o = *old;
@@ -2075,7 +2074,7 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) · 8 unmodified lines
20752074 #else /* atomic64_try_cmpxchg_relaxed */
20762075
20772076 #ifndef atomic64_try_cmpxchg_acquire
2078−static inline bool
2077+static __always_inline bool
20792078 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
20802079 {
20812080 bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);
@@ -2086,7 +2085,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) · 4 unmodified lines
20862085 #endif
20872086
20882087 #ifndef atomic64_try_cmpxchg_release
2089−static inline bool
2088+static __always_inline bool
20902089 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
20912090 {
20922091 __atomic_release_fence();
@@ -2096,7 +2095,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) · 3 unmodified lines
20962095 #endif
20972096
20982097 #ifndef atomic64_try_cmpxchg
2099−static inline bool
2098+static __always_inline bool
21002099 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
21012100 {
21022101 bool ret;
@@ -2120,7 +2119,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) · 17 unmodified lines
21202119 * true if the result is zero, or false for all
21212120 * other cases.
21222121 */
2123−static inline bool
2122+static __always_inline bool
21242123 atomic64_sub_and_test(s64 i, atomic64_t *v)
21252124 {
21262125 return atomic64_sub_return(i, v) == 0;
@@ -2137,7 +2136,7 @@ atomic64_sub_and_test(s64 i, atomic64_t *v) · 10 unmodified lines
21372136 * returns true if the result is 0, or false for all other
21382137 * cases.
21392138 */
2140−static inline bool
2139+static __always_inline bool
21412140 atomic64_dec_and_test(atomic64_t *v)
21422141 {
21432142 return atomic64_dec_return(v) == 0;
@@ -2154,7 +2153,7 @@ atomic64_dec_and_test(atomic64_t *v) · 10 unmodified lines
21542153 * and returns true if the result is zero, or false for all
21552154 * other cases.
21562155 */
2157−static inline bool
2156+static __always_inline bool
21582157 atomic64_inc_and_test(atomic64_t *v)
21592158 {
21602159 return atomic64_inc_return(v) == 0;
@@ -2172,7 +2171,7 @@ atomic64_inc_and_test(atomic64_t *v) · 11 unmodified lines
21722171 * if the result is negative, or false when
21732172 * result is greater than or equal to zero.
21742173 */
2175−static inline bool
2174+static __always_inline bool
21762175 atomic64_add_negative(s64 i, atomic64_t *v)
21772176 {
21782177 return atomic64_add_return(i, v) < 0;
@@ -2190,7 +2189,7 @@ atomic64_add_negative(s64 i, atomic64_t *v) · 11 unmodified lines
21902189 * Atomically adds @a to @v, so long as @v was not already @u.
21912190 * Returns original value of @v
21922191 */
2193−static inline s64
2192+static __always_inline s64
21942193 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
21952194 {
21962195 s64 c = atomic64_read(v);
@@ -2215,7 +2214,7 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) · 18 unmodified lines
22152214 * Atomically adds @a to @v, if @v was not already @u.
22162215 * Returns true if the addition was done.
22172216 */
2218−static inline bool
2217+static __always_inline bool
22192218 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
22202219 {
22212220 return atomic64_fetch_add_unless(v, a, u) != u;
@@ -2231,7 +2230,7 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) · 9 unmodified lines
22312230 * Atomically increments @v by 1, if @v is non-zero.
22322231 * Returns true if the increment was done.
22332232 */
2234−static inline bool
2233+static __always_inline bool
22352234 atomic64_inc_not_zero(atomic64_t *v)
22362235 {
22372236 return atomic64_add_unless(v, 1, 0);
@@ -2240,7 +2239,7 @@ atomic64_inc_not_zero(atomic64_t *v) · 2 unmodified lines
22402239 #endif
22412240
22422241 #ifndef atomic64_inc_unless_negative
2243−static inline bool
2242+static __always_inline bool
22442243 atomic64_inc_unless_negative(atomic64_t *v)
22452244 {
22462245 s64 c = atomic64_read(v);
@@ -2256,7 +2255,7 @@ atomic64_inc_unless_negative(atomic64_t *v) · 9 unmodified lines
22562255 #endif
22572256
22582257 #ifndef atomic64_dec_unless_positive
2259−static inline bool
2258+static __always_inline bool
22602259 atomic64_dec_unless_positive(atomic64_t *v)
22612260 {
22622261 s64 c = atomic64_read(v);
@@ -2272,7 +2271,7 @@ atomic64_dec_unless_positive(atomic64_t *v) · 9 unmodified lines
22722271 #endif
22732272
22742273 #ifndef atomic64_dec_if_positive
2275−static inline s64
2274+static __always_inline s64
22762275 atomic64_dec_if_positive(atomic64_t *v)
22772276 {
22782277 s64 dec, c = atomic64_read(v);
@@ -2288,8 +2287,5 @@ atomic64_dec_if_positive(atomic64_t *v) · 9 unmodified lines
22882287 #define atomic64_dec_if_positive atomic64_dec_if_positive
22892288 #endif
22902289
2291−#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
2292−#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
2293−
22942290 #endif /* _LINUX_ATOMIC_FALLBACK_H */
2295−// 25de4a2804d70f57e994fe3b419148658bb5378a
2291+// 1fac0941c79bf0ae100723cc2ac9b94061f0b67a
include/linux/atomic.h+11−0
@@ -25,6 +25,12 @@ · 24 unmodified lines
2525 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
2626 */
2727
28+#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
29+#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
30+
31+#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
32+#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
33+
2834 /*
2935 * The idea here is to build acquire/release variants by adding explicit
3036 * barriers on top of the relaxed variant. In the case where the relaxed
@@ -71,7 +77,12 @@ · 40 unmodified lines
7177 __ret; \
7278 })
7379
80+#ifdef ARCH_ATOMIC
81+#include <linux/atomic-arch-fallback.h>
82+#include <asm-generic/atomic-instrumented.h>
83+#else
7484 #include <linux/atomic-fallback.h>
85+#endif
7586
7687 #include <asm-generic/atomic-long.h>
7788
scripts/atomic/fallbacks/acquire+3−3
@@ -1,8 +1,8 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}${name}${sfx}_acquire(${params})
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}${name}${sfx}_acquire(${params})
44 {
5− ${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});
5+ ${ret} ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
66 __atomic_acquire_fence();
77 return ret;
88 }
scripts/atomic/fallbacks/add_negative+4−4
@@ -1,6 +1,6 @@
11 cat <<EOF
22 /**
3− * ${atomic}_add_negative - add and test if negative
3+ * ${arch}${atomic}_add_negative - add and test if negative
44 * @i: integer value to add
55 * @v: pointer of type ${atomic}_t
66 *
@@ -8,9 +8,9 @@ cat <<EOF · 1 unmodified line
88 * if the result is negative, or false when
99 * result is greater than or equal to zero.
1010 */
11−static inline bool
12−${atomic}_add_negative(${int} i, ${atomic}_t *v)
11+static __always_inline bool
12+${arch}${atomic}_add_negative(${int} i, ${atomic}_t *v)
1313 {
14− return ${atomic}_add_return(i, v) < 0;
14+ return ${arch}${atomic}_add_return(i, v) < 0;
1515 }
1616 EOF
scripts/atomic/fallbacks/add_unless+4−4
@@ -1,6 +1,6 @@
11 cat << EOF
22 /**
3− * ${atomic}_add_unless - add unless the number is already a given value
3+ * ${arch}${atomic}_add_unless - add unless the number is already a given value
44 * @v: pointer of type ${atomic}_t
55 * @a: the amount to add to v...
66 * @u: ...unless v is equal to u.
@@ -8,9 +8,9 @@ cat << EOF · 1 unmodified line
88 * Atomically adds @a to @v, if @v was not already @u.
99 * Returns true if the addition was done.
1010 */
11−static inline bool
12−${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
11+static __always_inline bool
12+${arch}${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
1313 {
14− return ${atomic}_fetch_add_unless(v, a, u) != u;
14+ return ${arch}${atomic}_fetch_add_unless(v, a, u) != u;
1515 }
1616 EOF
scripts/atomic/fallbacks/andnot+3−3
@@ -1,7 +1,7 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
44 {
5− ${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v);
5+ ${retstmt}${arch}${atomic}_${pfx}and${sfx}${order}(~i, v);
66 }
77 EOF
scripts/atomic/fallbacks/dec+3−3
@@ -1,7 +1,7 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
44 {
5− ${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v);
5+ ${retstmt}${arch}${atomic}_${pfx}sub${sfx}${order}(1, v);
66 }
77 EOF
scripts/atomic/fallbacks/dec_and_test+4−4
@@ -1,15 +1,15 @@
11 cat <<EOF
22 /**
3− * ${atomic}_dec_and_test - decrement and test
3+ * ${arch}${atomic}_dec_and_test - decrement and test
44 * @v: pointer of type ${atomic}_t
55 *
66 * Atomically decrements @v by 1 and
77 * returns true if the result is 0, or false for all other
88 * cases.
99 */
10−static inline bool
11−${atomic}_dec_and_test(${atomic}_t *v)
10+static __always_inline bool
11+${arch}${atomic}_dec_and_test(${atomic}_t *v)
1212 {
13− return ${atomic}_dec_return(v) == 0;
13+ return ${arch}${atomic}_dec_return(v) == 0;
1414 }
1515 EOF
scripts/atomic/fallbacks/dec_if_positive+4−4
@@ -1,14 +1,14 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_dec_if_positive(${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_dec_if_positive(${atomic}_t *v)
44 {
5− ${int} dec, c = ${atomic}_read(v);
5+ ${int} dec, c = ${arch}${atomic}_read(v);
66
77 do {
88 dec = c - 1;
99 if (unlikely(dec < 0))
1010 break;
11− } while (!${atomic}_try_cmpxchg(v, &c, dec));
11+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, dec));
1212
1313 return dec;
1414 }
scripts/atomic/fallbacks/dec_unless_positive+4−4
@@ -1,13 +1,13 @@
11 cat <<EOF
2−static inline bool
3−${atomic}_dec_unless_positive(${atomic}_t *v)
2+static __always_inline bool
3+${arch}${atomic}_dec_unless_positive(${atomic}_t *v)
44 {
5− ${int} c = ${atomic}_read(v);
5+ ${int} c = ${arch}${atomic}_read(v);
66
77 do {
88 if (unlikely(c > 0))
99 return false;
10− } while (!${atomic}_try_cmpxchg(v, &c, c - 1));
10+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, c - 1));
1111
1212 return true;
1313 }
scripts/atomic/fallbacks/fence+3−3
@@ -1,10 +1,10 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}${name}${sfx}(${params})
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}${name}${sfx}(${params})
44 {
55 ${ret} ret;
66 __atomic_pre_full_fence();
7− ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});
7+ ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
88 __atomic_post_full_fence();
99 return ret;
1010 }
scripts/atomic/fallbacks/fetch_add_unless+5−5
@@ -1,6 +1,6 @@
11 cat << EOF
22 /**
3− * ${atomic}_fetch_add_unless - add unless the number is already a given value
3+ * ${arch}${atomic}_fetch_add_unless - add unless the number is already a given value
44 * @v: pointer of type ${atomic}_t
55 * @a: the amount to add to v...
66 * @u: ...unless v is equal to u.
@@ -8,15 +8,15 @@ cat << EOF · 1 unmodified line
88 * Atomically adds @a to @v, so long as @v was not already @u.
99 * Returns original value of @v
1010 */
11−static inline ${int}
12−${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
11+static __always_inline ${int}
12+${arch}${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
1313 {
14− ${int} c = ${atomic}_read(v);
14+ ${int} c = ${arch}${atomic}_read(v);
1515
1616 do {
1717 if (unlikely(c == u))
1818 break;
19− } while (!${atomic}_try_cmpxchg(v, &c, c + a));
19+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + a));
2020
2121 return c;
2222 }
scripts/atomic/fallbacks/inc+3−3
@@ -1,7 +1,7 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
44 {
5− ${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v);
5+ ${retstmt}${arch}${atomic}_${pfx}add${sfx}${order}(1, v);
66 }
77 EOF
scripts/atomic/fallbacks/inc_and_test+4−4
@@ -1,15 +1,15 @@
11 cat <<EOF
22 /**
3− * ${atomic}_inc_and_test - increment and test
3+ * ${arch}${atomic}_inc_and_test - increment and test
44 * @v: pointer of type ${atomic}_t
55 *
66 * Atomically increments @v by 1
77 * and returns true if the result is zero, or false for all
88 * other cases.
99 */
10−static inline bool
11−${atomic}_inc_and_test(${atomic}_t *v)
10+static __always_inline bool
11+${arch}${atomic}_inc_and_test(${atomic}_t *v)
1212 {
13− return ${atomic}_inc_return(v) == 0;
13+ return ${arch}${atomic}_inc_return(v) == 0;
1414 }
1515 EOF
scripts/atomic/fallbacks/inc_not_zero+4−4
@@ -1,14 +1,14 @@
11 cat <<EOF
22 /**
3− * ${atomic}_inc_not_zero - increment unless the number is zero
3+ * ${arch}${atomic}_inc_not_zero - increment unless the number is zero
44 * @v: pointer of type ${atomic}_t
55 *
66 * Atomically increments @v by 1, if @v is non-zero.
77 * Returns true if the increment was done.
88 */
9−static inline bool
10−${atomic}_inc_not_zero(${atomic}_t *v)
9+static __always_inline bool
10+${arch}${atomic}_inc_not_zero(${atomic}_t *v)
1111 {
12− return ${atomic}_add_unless(v, 1, 0);
12+ return ${arch}${atomic}_add_unless(v, 1, 0);
1313 }
1414 EOF
scripts/atomic/fallbacks/inc_unless_negative+4−4
@@ -1,13 +1,13 @@
11 cat <<EOF
2−static inline bool
3−${atomic}_inc_unless_negative(${atomic}_t *v)
2+static __always_inline bool
3+${arch}${atomic}_inc_unless_negative(${atomic}_t *v)
44 {
5− ${int} c = ${atomic}_read(v);
5+ ${int} c = ${arch}${atomic}_read(v);
66
77 do {
88 if (unlikely(c < 0))
99 return false;
10− } while (!${atomic}_try_cmpxchg(v, &c, c + 1));
10+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + 1));
1111
1212 return true;
1313 }
scripts/atomic/fallbacks/read_acquire+2−2
@@ -1,6 +1,6 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_read_acquire(const ${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_read_acquire(const ${atomic}_t *v)
44 {
55 return smp_load_acquire(&(v)->counter);
66 }
scripts/atomic/fallbacks/release+3−3
@@ -1,8 +1,8 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}${name}${sfx}_release(${params})
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}${name}${sfx}_release(${params})
44 {
55 __atomic_release_fence();
6− ${retstmt}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
6+ ${retstmt}${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
77 }
88 EOF
scripts/atomic/fallbacks/set_release+2−2
@@ -1,6 +1,6 @@
11 cat <<EOF
2−static inline void
3−${atomic}_set_release(${atomic}_t *v, ${int} i)
2+static __always_inline void
3+${arch}${atomic}_set_release(${atomic}_t *v, ${int} i)
44 {
55 smp_store_release(&(v)->counter, i);
66 }
scripts/atomic/fallbacks/sub_and_test+4−4
@@ -1,6 +1,6 @@
11 cat <<EOF
22 /**
3− * ${atomic}_sub_and_test - subtract value from variable and test result
3+ * ${arch}${atomic}_sub_and_test - subtract value from variable and test result
44 * @i: integer value to subtract
55 * @v: pointer of type ${atomic}_t
66 *
@@ -8,9 +8,9 @@ cat <<EOF · 1 unmodified line
88 * true if the result is zero, or false for all
99 * other cases.
1010 */
11−static inline bool
12−${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
11+static __always_inline bool
12+${arch}${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
1313 {
14− return ${atomic}_sub_return(i, v) == 0;
14+ return ${arch}${atomic}_sub_return(i, v) == 0;
1515 }
1616 EOF
scripts/atomic/fallbacks/try_cmpxchg+3−3
@@ -1,9 +1,9 @@
11 cat <<EOF
2−static inline bool
3−${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
2+static __always_inline bool
3+${arch}${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
44 {
55 ${int} r, o = *old;
6− r = ${atomic}_cmpxchg${order}(v, o, new);
6+ r = ${arch}${atomic}_cmpxchg${order}(v, o, new);
77 if (unlikely(r != o))
88 *old = r;
99 return likely(r == o);
scripts/atomic/gen-atomic-fallback.sh+15−16
@@ -2,10 +2,11 @@ · 1 unmodified line
22 # SPDX-License-Identifier: GPL-2.0
33
44 ATOMICDIR=$(dirname $0)
5+ARCH=$2
56
67 . ${ATOMICDIR}/atomic-tbl.sh
78
8−#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...)
9+#gen_template_fallback(template, meta, pfx, name, sfx, order, arch, atomic, int, args...)
910 gen_template_fallback()
1011 {
1112 local template="$1"; shift
@@ -14,10 +15,11 @@ gen_template_fallback() · 2 unmodified lines
1415 local name="$1"; shift
1516 local sfx="$1"; shift
1617 local order="$1"; shift
18+ local arch="$1"; shift
1719 local atomic="$1"; shift
1820 local int="$1"; shift
1921
20− local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
22+ local atomicname="${arch}${atomic}_${pfx}${name}${sfx}${order}"
2123
2224 local ret="$(gen_ret_type "${meta}" "${int}")"
2325 local retstmt="$(gen_ret_stmt "${meta}")"
@@ -32,7 +34,7 @@ gen_template_fallback() · 8 unmodified lines
3234 fi
3335 }
3436
35−#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
37+#gen_proto_fallback(meta, pfx, name, sfx, order, arch, atomic, int, args...)
3638 gen_proto_fallback()
3739 {
3840 local meta="$1"; shift
@@ -56,16 +58,17 @@ cat << EOF · 17 unmodified lines
5658 EOF
5759 }
5860
59−#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
61+#gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...)
6062 gen_proto_order_variants()
6163 {
6264 local meta="$1"; shift
6365 local pfx="$1"; shift
6466 local name="$1"; shift
6567 local sfx="$1"; shift
66− local atomic="$1"
68+ local arch="$1"
69+ local atomic="$2"
6770
68− local basename="${atomic}_${pfx}${name}${sfx}"
71+ local basename="${arch}${atomic}_${pfx}${name}${sfx}"
6972
7073 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
7174
@@ -94,7 +97,7 @@ gen_proto_order_variants() · 22 unmodified lines
9497 gen_basic_fallbacks "${basename}"
9598
9699 if [ ! -z "${template}" ]; then
97− printf "#endif /* ${atomic}_${pfx}${name}${sfx} */\n\n"
100+ printf "#endif /* ${arch}${atomic}_${pfx}${name}${sfx} */\n\n"
98101 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
99102 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
100103 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
@@ -149,20 +152,19 @@ cat << EOF · 48 unmodified lines
149152 #ifndef _LINUX_ATOMIC_FALLBACK_H
150153 #define _LINUX_ATOMIC_FALLBACK_H
151154
155+#include <linux/compiler.h>
156+
152157 EOF
153158
154−for xchg in "xchg" "cmpxchg" "cmpxchg64"; do
159+for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do
155160 gen_xchg_fallbacks "${xchg}"
156161 done
157162
158163 grep '^[a-z]' "$1" | while read name meta args; do
159− gen_proto "${meta}" "${name}" "atomic" "int" ${args}
164+ gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args}
160165 done
161166
162167 cat <<EOF
163−#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
164−#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
165−
166168 #ifdef CONFIG_GENERIC_ATOMIC64
167169 #include <asm-generic/atomic64.h>
168170 #endif
@@ -170,12 +172,9 @@ cat <<EOF · 1 unmodified line
170172 EOF
171173
172174 grep '^[a-z]' "$1" | while read name meta args; do
173− gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
175+ gen_proto "${meta}" "${name}" "${ARCH}" "atomic64" "s64" ${args}
174176 done
175177
176178 cat <<EOF
177−#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
178−#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
179−
180179 #endif /* _LINUX_ATOMIC_FALLBACK_H */
181180 EOF
scripts/atomic/gen-atomics.sh+3−2
@@ -10,10 +10,11 @@ LINUXDIR=${ATOMICDIR}/../.. · 9 unmodified lines
1010 cat <<EOF |
1111 gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h
1212 gen-atomic-long.sh asm-generic/atomic-long.h
13+gen-atomic-fallback.sh linux/atomic-arch-fallback.h arch_
1314 gen-atomic-fallback.sh linux/atomic-fallback.h
1415 EOF
15−while read script header; do
16− /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} > ${LINUXDIR}/include/${header}
16+while read script header args; do
17+ /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header}
1718 HASH="$(sha1sum ${LINUXDIR}/include/${header})"
1819 HASH="${HASH%% *}"
1920 printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header}