History

Merge tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull atomics rework from Thomas Gleixner: "Peter Zijlstras rework of atomics and fallbacks. This solves two problems: 1) Compilers uninline small atomic_* static inline functions which can expose them to instrumentation. 2) The instrumentation of atomic primitives was done at the architecture level while composites or fallbacks were provided at the generic level. As a result there are no uninstrumented variants of the fallbacks. Both issues were in the way of fully isolating fragile entry code pathes and especially the text poke int3 handler which is prone to an endless recursion problem when anything in that code path is about to be instrumented. This was always a problem, but got elevated due to the new batch mode updates of tracing. The solution is to mark the functions __always_inline and to flip the fallback and instrumentation so the non-instrumented variants are at the architecture level and the instrumentation is done in generic code. The latter introduces another fallback variant which will go away once all architectures have been moved over to arch_atomic_*" * tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/atomics: Flip fallbacks and instrumentation asm-generic/atomic: Use __always_inline for fallback wrappers

Linus Torvalds committed 6 years ago

Commit 9716e572 parents b1a6274 37f817328 files · +2594 −269Browse files

Unified · Split

28 files
  1. arch/arm64/include/asm/atomic.h +3 −3
  2. arch/x86/include/asm/atomic.h +13 −4
  3. arch/x86/include/asm/atomic64_32.h +9 −0
  4. arch/x86/include/asm/atomic64_64.h +12 −3
  5. include/linux/atomic-arch-fallback.h +2291 −0
  6. include/linux/atomic-fallback.h +171 −175
  7. include/linux/atomic.h +11 −0
  8. scripts/atomic/fallbacks/acquire +3 −3
  9. scripts/atomic/fallbacks/add_negative +4 −4
  10. scripts/atomic/fallbacks/add_unless +4 −4
  11. scripts/atomic/fallbacks/andnot +3 −3
  12. scripts/atomic/fallbacks/dec +3 −3
  13. scripts/atomic/fallbacks/dec_and_test +4 −4
  14. scripts/atomic/fallbacks/dec_if_positive +4 −4
  15. scripts/atomic/fallbacks/dec_unless_positive +4 −4
  16. scripts/atomic/fallbacks/fence +3 −3
  17. scripts/atomic/fallbacks/fetch_add_unless +5 −5
  18. scripts/atomic/fallbacks/inc +3 −3
  19. scripts/atomic/fallbacks/inc_and_test +4 −4
  20. scripts/atomic/fallbacks/inc_not_zero +4 −4
  21. scripts/atomic/fallbacks/inc_unless_negative +4 −4
  22. scripts/atomic/fallbacks/read_acquire +2 −2
  23. scripts/atomic/fallbacks/release +3 −3
  24. scripts/atomic/fallbacks/set_release +2 −2
  25. scripts/atomic/fallbacks/sub_and_test +4 −4
  26. scripts/atomic/fallbacks/try_cmpxchg +3 −3
  27. scripts/atomic/gen-atomic-fallback.sh +15 −16
  28. scripts/atomic/gen-atomics.sh +3 −2
arch/arm64/include/asm/atomic.h+3−3
@@ -101,8 +101,8 @@ static inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 100 unmodified lines
101 101
102#define ATOMIC_INIT(i) { (i) }102#define ATOMIC_INIT(i) { (i) }
103 103
104#define arch_atomic_read(v) READ_ONCE((v)->counter)104#define arch_atomic_read(v) __READ_ONCE((v)->counter)
105#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))105#define arch_atomic_set(v, i) __WRITE_ONCE(((v)->counter), (i))
106 106
107#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed107#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
108#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire108#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
@@ -225,6 +225,6 @@ static inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 116 unmodified lines
225 225
226#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive226#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
227 227
228#include <asm-generic/atomic-instrumented.h>228#define ARCH_ATOMIC
229 229
230#endif /* __ASM_ATOMIC_H */230#endif /* __ASM_ATOMIC_H */
arch/x86/include/asm/atomic.h+13−4
@@ -28,7 +28,7 @@ static __always_inline int arch_atomic_read(const atomic_t *v) · 27 unmodified lines
28 * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here,28 * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here,
29 * it's non-inlined function that increases binary size and stack usage.29 * it's non-inlined function that increases binary size and stack usage.
30 */30 */
31 return READ_ONCE((v)->counter);31 return __READ_ONCE((v)->counter);
32}32}
33 33
34/**34/**
@@ -40,7 +40,7 @@ static __always_inline int arch_atomic_read(const atomic_t *v) · 5 unmodified lines
40 */40 */
41static __always_inline void arch_atomic_set(atomic_t *v, int i)41static __always_inline void arch_atomic_set(atomic_t *v, int i)
42{42{
43 WRITE_ONCE(v->counter, i);43 __WRITE_ONCE(v->counter, i);
44}44}
45 45
46/**46/**
@@ -166,6 +166,7 @@ static __always_inline int arch_atomic_add_return(int i, atomic_t *v) · 119 unmodified lines
166{166{
167 return i + xadd(&v->counter, i);167 return i + xadd(&v->counter, i);
168}168}
169#define arch_atomic_add_return arch_atomic_add_return
169 170
170/**171/**
171 * arch_atomic_sub_return - subtract integer and return172 * arch_atomic_sub_return - subtract integer and return
@@ -178,32 +179,37 @@ static __always_inline int arch_atomic_sub_return(int i, atomic_t *v) · 6 unmodified lines
178{179{
179 return arch_atomic_add_return(-i, v);180 return arch_atomic_add_return(-i, v);
180}181}
182#define arch_atomic_sub_return arch_atomic_sub_return
181 183
182static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)184static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)
183{185{
184 return xadd(&v->counter, i);186 return xadd(&v->counter, i);
185}187}
188#define arch_atomic_fetch_add arch_atomic_fetch_add
186 189
187static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v)190static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v)
188{191{
189 return xadd(&v->counter, -i);192 return xadd(&v->counter, -i);
190}193}
194#define arch_atomic_fetch_sub arch_atomic_fetch_sub
191 195
192static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)196static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
193{197{
194 return arch_cmpxchg(&v->counter, old, new);198 return arch_cmpxchg(&v->counter, old, new);
195}199}
200#define arch_atomic_cmpxchg arch_atomic_cmpxchg
196 201
197#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
198static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)202static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
199{203{
200 return try_cmpxchg(&v->counter, old, new);204 return try_cmpxchg(&v->counter, old, new);
201}205}
206#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
202 207
203static inline int arch_atomic_xchg(atomic_t *v, int new)208static inline int arch_atomic_xchg(atomic_t *v, int new)
204{209{
205 return arch_xchg(&v->counter, new);210 return arch_xchg(&v->counter, new);
206}211}
212#define arch_atomic_xchg arch_atomic_xchg
207 213
208static inline void arch_atomic_and(int i, atomic_t *v)214static inline void arch_atomic_and(int i, atomic_t *v)
209{215{
@@ -221,6 +227,7 @@ static inline int arch_atomic_fetch_and(int i, atomic_t *v) · 11 unmodified lines
221 227
222 return val;228 return val;
223}229}
230#define arch_atomic_fetch_and arch_atomic_fetch_and
224 231
225static inline void arch_atomic_or(int i, atomic_t *v)232static inline void arch_atomic_or(int i, atomic_t *v)
226{233{
@@ -238,6 +245,7 @@ static inline int arch_atomic_fetch_or(int i, atomic_t *v) · 11 unmodified lines
238 245
239 return val;246 return val;
240}247}
248#define arch_atomic_fetch_or arch_atomic_fetch_or
241 249
242static inline void arch_atomic_xor(int i, atomic_t *v)250static inline void arch_atomic_xor(int i, atomic_t *v)
243{251{
@@ -255,6 +263,7 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) · 11 unmodified lines
255 263
256 return val;264 return val;
257}265}
266#define arch_atomic_fetch_xor arch_atomic_fetch_xor
258 267
259#ifdef CONFIG_X86_32268#ifdef CONFIG_X86_32
260# include <asm/atomic64_32.h>269# include <asm/atomic64_32.h>
@@ -262,6 +271,6 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) · 1 unmodified line
262# include <asm/atomic64_64.h>271# include <asm/atomic64_64.h>
263#endif272#endif
264 273
265#include <asm-generic/atomic-instrumented.h>274#define ARCH_ATOMIC
266 275
267#endif /* _ASM_X86_ATOMIC_H */276#endif /* _ASM_X86_ATOMIC_H */
arch/x86/include/asm/atomic64_32.h+9−0
@@ -75,6 +75,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) · 74 unmodified lines
75{75{
76 return arch_cmpxchg64(&v->counter, o, n);76 return arch_cmpxchg64(&v->counter, o, n);
77}77}
78#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
78 79
79/**80/**
80 * arch_atomic64_xchg - xchg atomic64 variable81 * arch_atomic64_xchg - xchg atomic64 variable
@@ -94,6 +95,7 @@ static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) · 13 unmodified lines
94 : "memory");95 : "memory");
95 return o;96 return o;
96}97}
98#define arch_atomic64_xchg arch_atomic64_xchg
97 99
98/**100/**
99 * arch_atomic64_set - set atomic64 variable101 * arch_atomic64_set - set atomic64 variable
@@ -138,6 +140,7 @@ static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) · 38 unmodified lines
138 ASM_NO_INPUT_CLOBBER("memory"));140 ASM_NO_INPUT_CLOBBER("memory"));
139 return i;141 return i;
140}142}
143#define arch_atomic64_add_return arch_atomic64_add_return
141 144
142/*145/*
143 * Other variants with different arithmetic operators:146 * Other variants with different arithmetic operators:
@@ -149,6 +152,7 @@ static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) · 5 unmodified lines
149 ASM_NO_INPUT_CLOBBER("memory"));152 ASM_NO_INPUT_CLOBBER("memory"));
150 return i;153 return i;
151}154}
155#define arch_atomic64_sub_return arch_atomic64_sub_return
152 156
153static inline s64 arch_atomic64_inc_return(atomic64_t *v)157static inline s64 arch_atomic64_inc_return(atomic64_t *v)
154{158{
@@ -242,6 +246,7 @@ static inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) · 87 unmodified lines
242 "S" (v) : "memory");246 "S" (v) : "memory");
243 return (int)a;247 return (int)a;
244}248}
249#define arch_atomic64_add_unless arch_atomic64_add_unless
245 250
246static inline int arch_atomic64_inc_not_zero(atomic64_t *v)251static inline int arch_atomic64_inc_not_zero(atomic64_t *v)
247{252{
@@ -281,6 +286,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) · 33 unmodified lines
281 286
282 return old;287 return old;
283}288}
289#define arch_atomic64_fetch_and arch_atomic64_fetch_and
284 290
285static inline void arch_atomic64_or(s64 i, atomic64_t *v)291static inline void arch_atomic64_or(s64 i, atomic64_t *v)
286{292{
@@ -299,6 +305,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) · 12 unmodified lines
299 305
300 return old;306 return old;
301}307}
308#define arch_atomic64_fetch_or arch_atomic64_fetch_or
302 309
303static inline void arch_atomic64_xor(s64 i, atomic64_t *v)310static inline void arch_atomic64_xor(s64 i, atomic64_t *v)
304{311{
@@ -317,6 +324,7 @@ static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) · 12 unmodified lines
317 324
318 return old;325 return old;
319}326}
327#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
320 328
321static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)329static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
322{330{
@@ -327,6 +335,7 @@ static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) · 4 unmodified lines
327 335
328 return old;336 return old;
329}337}
338#define arch_atomic64_fetch_add arch_atomic64_fetch_add
330 339
331#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v))340#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v))
332 341
arch/x86/include/asm/atomic64_64.h+12−3
@@ -19,7 +19,7 @@ · 18 unmodified lines
19 */19 */
20static inline s64 arch_atomic64_read(const atomic64_t *v)20static inline s64 arch_atomic64_read(const atomic64_t *v)
21{21{
22 return READ_ONCE((v)->counter);22 return __READ_ONCE((v)->counter);
23}23}
24 24
25/**25/**
@@ -31,7 +31,7 @@ static inline s64 arch_atomic64_read(const atomic64_t *v) · 5 unmodified lines
31 */31 */
32static inline void arch_atomic64_set(atomic64_t *v, s64 i)32static inline void arch_atomic64_set(atomic64_t *v, s64 i)
33{33{
34 WRITE_ONCE(v->counter, i);34 __WRITE_ONCE(v->counter, i);
35}35}
36 36
37/**37/**
@@ -159,37 +159,43 @@ static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) · 121 unmodified lines
159{159{
160 return i + xadd(&v->counter, i);160 return i + xadd(&v->counter, i);
161}161}
162#define arch_atomic64_add_return arch_atomic64_add_return
162 163
163static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)164static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
164{165{
165 return arch_atomic64_add_return(-i, v);166 return arch_atomic64_add_return(-i, v);
166}167}
168#define arch_atomic64_sub_return arch_atomic64_sub_return
167 169
168static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)170static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
169{171{
170 return xadd(&v->counter, i);172 return xadd(&v->counter, i);
171}173}
174#define arch_atomic64_fetch_add arch_atomic64_fetch_add
172 175
173static inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)176static inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
174{177{
175 return xadd(&v->counter, -i);178 return xadd(&v->counter, -i);
176}179}
180#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
177 181
178static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)182static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
179{183{
180 return arch_cmpxchg(&v->counter, old, new);184 return arch_cmpxchg(&v->counter, old, new);
181}185}
186#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
182 187
183#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
184static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)188static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
185{189{
186 return try_cmpxchg(&v->counter, old, new);190 return try_cmpxchg(&v->counter, old, new);
187}191}
192#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
188 193
189static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)194static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)
190{195{
191 return arch_xchg(&v->counter, new);196 return arch_xchg(&v->counter, new);
192}197}
198#define arch_atomic64_xchg arch_atomic64_xchg
193 199
194static inline void arch_atomic64_and(s64 i, atomic64_t *v)200static inline void arch_atomic64_and(s64 i, atomic64_t *v)
195{201{
@@ -207,6 +213,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) · 11 unmodified lines
207 } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));213 } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
208 return val;214 return val;
209}215}
216#define arch_atomic64_fetch_and arch_atomic64_fetch_and
210 217
211static inline void arch_atomic64_or(s64 i, atomic64_t *v)218static inline void arch_atomic64_or(s64 i, atomic64_t *v)
212{219{
@@ -224,6 +231,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) · 11 unmodified lines
224 } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));231 } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
225 return val;232 return val;
226}233}
234#define arch_atomic64_fetch_or arch_atomic64_fetch_or
227 235
228static inline void arch_atomic64_xor(s64 i, atomic64_t *v)236static inline void arch_atomic64_xor(s64 i, atomic64_t *v)
229{237{
@@ -241,5 +249,6 @@ static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) · 11 unmodified lines
241 } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));249 } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
242 return val;250 return val;
243}251}
252#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
244 253
245#endif /* _ASM_X86_ATOMIC64_64_H */254#endif /* _ASM_X86_ATOMIC64_64_H */
include/linux/atomic-arch-fallback.h+2291−0
@@ -0,0 +1,2291 @@
1// SPDX-License-Identifier: GPL-2.0
2
3// Generated by scripts/atomic/gen-atomic-fallback.sh
4// DO NOT MODIFY THIS FILE DIRECTLY
5
6#ifndef _LINUX_ATOMIC_FALLBACK_H
7#define _LINUX_ATOMIC_FALLBACK_H
8
9#include <linux/compiler.h>
10
11#ifndef arch_xchg_relaxed
12#define arch_xchg_relaxed arch_xchg
13#define arch_xchg_acquire arch_xchg
14#define arch_xchg_release arch_xchg
15#else /* arch_xchg_relaxed */
16
17#ifndef arch_xchg_acquire
18#define arch_xchg_acquire(...) \
19 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20#endif
21
22#ifndef arch_xchg_release
23#define arch_xchg_release(...) \
24 __atomic_op_release(arch_xchg, __VA_ARGS__)
25#endif
26
27#ifndef arch_xchg
28#define arch_xchg(...) \
29 __atomic_op_fence(arch_xchg, __VA_ARGS__)
30#endif
31
32#endif /* arch_xchg_relaxed */
33
34#ifndef arch_cmpxchg_relaxed
35#define arch_cmpxchg_relaxed arch_cmpxchg
36#define arch_cmpxchg_acquire arch_cmpxchg
37#define arch_cmpxchg_release arch_cmpxchg
38#else /* arch_cmpxchg_relaxed */
39
40#ifndef arch_cmpxchg_acquire
41#define arch_cmpxchg_acquire(...) \
42 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
43#endif
44
45#ifndef arch_cmpxchg_release
46#define arch_cmpxchg_release(...) \
47 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
48#endif
49
50#ifndef arch_cmpxchg
51#define arch_cmpxchg(...) \
52 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53#endif
54
55#endif /* arch_cmpxchg_relaxed */
56
57#ifndef arch_cmpxchg64_relaxed
58#define arch_cmpxchg64_relaxed arch_cmpxchg64
59#define arch_cmpxchg64_acquire arch_cmpxchg64
60#define arch_cmpxchg64_release arch_cmpxchg64
61#else /* arch_cmpxchg64_relaxed */
62
63#ifndef arch_cmpxchg64_acquire
64#define arch_cmpxchg64_acquire(...) \
65 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
66#endif
67
68#ifndef arch_cmpxchg64_release
69#define arch_cmpxchg64_release(...) \
70 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
71#endif
72
73#ifndef arch_cmpxchg64
74#define arch_cmpxchg64(...) \
75 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
76#endif
77
78#endif /* arch_cmpxchg64_relaxed */
79
80#ifndef arch_atomic_read_acquire
81static __always_inline int
82arch_atomic_read_acquire(const atomic_t *v)
83{
84 return smp_load_acquire(&(v)->counter);
85}
86#define arch_atomic_read_acquire arch_atomic_read_acquire
87#endif
88
89#ifndef arch_atomic_set_release
90static __always_inline void
91arch_atomic_set_release(atomic_t *v, int i)
92{
93 smp_store_release(&(v)->counter, i);
94}
95#define arch_atomic_set_release arch_atomic_set_release
96#endif
97
98#ifndef arch_atomic_add_return_relaxed
99#define arch_atomic_add_return_acquire arch_atomic_add_return
100#define arch_atomic_add_return_release arch_atomic_add_return
101#define arch_atomic_add_return_relaxed arch_atomic_add_return
102#else /* arch_atomic_add_return_relaxed */
103
104#ifndef arch_atomic_add_return_acquire
105static __always_inline int
106arch_atomic_add_return_acquire(int i, atomic_t *v)
107{
108 int ret = arch_atomic_add_return_relaxed(i, v);
109 __atomic_acquire_fence();
110 return ret;
111}
112#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
113#endif
114
115#ifndef arch_atomic_add_return_release
116static __always_inline int
117arch_atomic_add_return_release(int i, atomic_t *v)
118{
119 __atomic_release_fence();
120 return arch_atomic_add_return_relaxed(i, v);
121}
122#define arch_atomic_add_return_release arch_atomic_add_return_release
123#endif
124
125#ifndef arch_atomic_add_return
126static __always_inline int
127arch_atomic_add_return(int i, atomic_t *v)
128{
129 int ret;
130 __atomic_pre_full_fence();
131 ret = arch_atomic_add_return_relaxed(i, v);
132 __atomic_post_full_fence();
133 return ret;
134}
135#define arch_atomic_add_return arch_atomic_add_return
136#endif
137
138#endif /* arch_atomic_add_return_relaxed */
139
140#ifndef arch_atomic_fetch_add_relaxed
141#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
142#define arch_atomic_fetch_add_release arch_atomic_fetch_add
143#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
144#else /* arch_atomic_fetch_add_relaxed */
145
146#ifndef arch_atomic_fetch_add_acquire
147static __always_inline int
148arch_atomic_fetch_add_acquire(int i, atomic_t *v)
149{
150 int ret = arch_atomic_fetch_add_relaxed(i, v);
151 __atomic_acquire_fence();
152 return ret;
153}
154#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
155#endif
156
157#ifndef arch_atomic_fetch_add_release
158static __always_inline int
159arch_atomic_fetch_add_release(int i, atomic_t *v)
160{
161 __atomic_release_fence();
162 return arch_atomic_fetch_add_relaxed(i, v);
163}
164#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
165#endif
166
167#ifndef arch_atomic_fetch_add
168static __always_inline int
169arch_atomic_fetch_add(int i, atomic_t *v)
170{
171 int ret;
172 __atomic_pre_full_fence();
173 ret = arch_atomic_fetch_add_relaxed(i, v);
174 __atomic_post_full_fence();
175 return ret;
176}
177#define arch_atomic_fetch_add arch_atomic_fetch_add
178#endif
179
180#endif /* arch_atomic_fetch_add_relaxed */
181
182#ifndef arch_atomic_sub_return_relaxed
183#define arch_atomic_sub_return_acquire arch_atomic_sub_return
184#define arch_atomic_sub_return_release arch_atomic_sub_return
185#define arch_atomic_sub_return_relaxed arch_atomic_sub_return
186#else /* arch_atomic_sub_return_relaxed */
187
188#ifndef arch_atomic_sub_return_acquire
189static __always_inline int
190arch_atomic_sub_return_acquire(int i, atomic_t *v)
191{
192 int ret = arch_atomic_sub_return_relaxed(i, v);
193 __atomic_acquire_fence();
194 return ret;
195}
196#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
197#endif
198
199#ifndef arch_atomic_sub_return_release
200static __always_inline int
201arch_atomic_sub_return_release(int i, atomic_t *v)
202{
203 __atomic_release_fence();
204 return arch_atomic_sub_return_relaxed(i, v);
205}
206#define arch_atomic_sub_return_release arch_atomic_sub_return_release
207#endif
208
209#ifndef arch_atomic_sub_return
210static __always_inline int
211arch_atomic_sub_return(int i, atomic_t *v)
212{
213 int ret;
214 __atomic_pre_full_fence();
215 ret = arch_atomic_sub_return_relaxed(i, v);
216 __atomic_post_full_fence();
217 return ret;
218}
219#define arch_atomic_sub_return arch_atomic_sub_return
220#endif
221
222#endif /* arch_atomic_sub_return_relaxed */
223
224#ifndef arch_atomic_fetch_sub_relaxed
225#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
226#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
227#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
228#else /* arch_atomic_fetch_sub_relaxed */
229
230#ifndef arch_atomic_fetch_sub_acquire
231static __always_inline int
232arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
233{
234 int ret = arch_atomic_fetch_sub_relaxed(i, v);
235 __atomic_acquire_fence();
236 return ret;
237}
238#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
239#endif
240
241#ifndef arch_atomic_fetch_sub_release
242static __always_inline int
243arch_atomic_fetch_sub_release(int i, atomic_t *v)
244{
245 __atomic_release_fence();
246 return arch_atomic_fetch_sub_relaxed(i, v);
247}
248#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
249#endif
250
251#ifndef arch_atomic_fetch_sub
252static __always_inline int
253arch_atomic_fetch_sub(int i, atomic_t *v)
254{
255 int ret;
256 __atomic_pre_full_fence();
257 ret = arch_atomic_fetch_sub_relaxed(i, v);
258 __atomic_post_full_fence();
259 return ret;
260}
261#define arch_atomic_fetch_sub arch_atomic_fetch_sub
262#endif
263
264#endif /* arch_atomic_fetch_sub_relaxed */
265
266#ifndef arch_atomic_inc
267static __always_inline void
268arch_atomic_inc(atomic_t *v)
269{
270 arch_atomic_add(1, v);
271}
272#define arch_atomic_inc arch_atomic_inc
273#endif
274
275#ifndef arch_atomic_inc_return_relaxed
276#ifdef arch_atomic_inc_return
277#define arch_atomic_inc_return_acquire arch_atomic_inc_return
278#define arch_atomic_inc_return_release arch_atomic_inc_return
279#define arch_atomic_inc_return_relaxed arch_atomic_inc_return
280#endif /* arch_atomic_inc_return */
281
282#ifndef arch_atomic_inc_return
283static __always_inline int
284arch_atomic_inc_return(atomic_t *v)
285{
286 return arch_atomic_add_return(1, v);
287}
288#define arch_atomic_inc_return arch_atomic_inc_return
289#endif
290
291#ifndef arch_atomic_inc_return_acquire
292static __always_inline int
293arch_atomic_inc_return_acquire(atomic_t *v)
294{
295 return arch_atomic_add_return_acquire(1, v);
296}
297#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
298#endif
299
300#ifndef arch_atomic_inc_return_release
301static __always_inline int
302arch_atomic_inc_return_release(atomic_t *v)
303{
304 return arch_atomic_add_return_release(1, v);
305}
306#define arch_atomic_inc_return_release arch_atomic_inc_return_release
307#endif
308
309#ifndef arch_atomic_inc_return_relaxed
310static __always_inline int
311arch_atomic_inc_return_relaxed(atomic_t *v)
312{
313 return arch_atomic_add_return_relaxed(1, v);
314}
315#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
316#endif
317
318#else /* arch_atomic_inc_return_relaxed */
319
320#ifndef arch_atomic_inc_return_acquire
321static __always_inline int
322arch_atomic_inc_return_acquire(atomic_t *v)
323{
324 int ret = arch_atomic_inc_return_relaxed(v);
325 __atomic_acquire_fence();
326 return ret;
327}
328#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
329#endif
330
331#ifndef arch_atomic_inc_return_release
332static __always_inline int
333arch_atomic_inc_return_release(atomic_t *v)
334{
335 __atomic_release_fence();
336 return arch_atomic_inc_return_relaxed(v);
337}
338#define arch_atomic_inc_return_release arch_atomic_inc_return_release
339#endif
340
341#ifndef arch_atomic_inc_return
342static __always_inline int
343arch_atomic_inc_return(atomic_t *v)
344{
345 int ret;
346 __atomic_pre_full_fence();
347 ret = arch_atomic_inc_return_relaxed(v);
348 __atomic_post_full_fence();
349 return ret;
350}
351#define arch_atomic_inc_return arch_atomic_inc_return
352#endif
353
354#endif /* arch_atomic_inc_return_relaxed */
355
356#ifndef arch_atomic_fetch_inc_relaxed
357#ifdef arch_atomic_fetch_inc
358#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
359#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
360#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
361#endif /* arch_atomic_fetch_inc */
362
363#ifndef arch_atomic_fetch_inc
364static __always_inline int
365arch_atomic_fetch_inc(atomic_t *v)
366{
367 return arch_atomic_fetch_add(1, v);
368}
369#define arch_atomic_fetch_inc arch_atomic_fetch_inc
370#endif
371
372#ifndef arch_atomic_fetch_inc_acquire
373static __always_inline int
374arch_atomic_fetch_inc_acquire(atomic_t *v)
375{
376 return arch_atomic_fetch_add_acquire(1, v);
377}
378#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
379#endif
380
381#ifndef arch_atomic_fetch_inc_release
382static __always_inline int
383arch_atomic_fetch_inc_release(atomic_t *v)
384{
385 return arch_atomic_fetch_add_release(1, v);
386}
387#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
388#endif
389
390#ifndef arch_atomic_fetch_inc_relaxed
391static __always_inline int
392arch_atomic_fetch_inc_relaxed(atomic_t *v)
393{
394 return arch_atomic_fetch_add_relaxed(1, v);
395}
396#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
397#endif
398
399#else /* arch_atomic_fetch_inc_relaxed */
400
401#ifndef arch_atomic_fetch_inc_acquire
402static __always_inline int
403arch_atomic_fetch_inc_acquire(atomic_t *v)
404{
405 int ret = arch_atomic_fetch_inc_relaxed(v);
406 __atomic_acquire_fence();
407 return ret;
408}
409#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
410#endif
411
412#ifndef arch_atomic_fetch_inc_release
413static __always_inline int
414arch_atomic_fetch_inc_release(atomic_t *v)
415{
416 __atomic_release_fence();
417 return arch_atomic_fetch_inc_relaxed(v);
418}
419#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
420#endif
421
422#ifndef arch_atomic_fetch_inc
423static __always_inline int
424arch_atomic_fetch_inc(atomic_t *v)
425{
426 int ret;
427 __atomic_pre_full_fence();
428 ret = arch_atomic_fetch_inc_relaxed(v);
429 __atomic_post_full_fence();
430 return ret;
431}
432#define arch_atomic_fetch_inc arch_atomic_fetch_inc
433#endif
434
435#endif /* arch_atomic_fetch_inc_relaxed */
436
437#ifndef arch_atomic_dec
438static __always_inline void
439arch_atomic_dec(atomic_t *v)
440{
441 arch_atomic_sub(1, v);
442}
443#define arch_atomic_dec arch_atomic_dec
444#endif
445
446#ifndef arch_atomic_dec_return_relaxed
447#ifdef arch_atomic_dec_return
448#define arch_atomic_dec_return_acquire arch_atomic_dec_return
449#define arch_atomic_dec_return_release arch_atomic_dec_return
450#define arch_atomic_dec_return_relaxed arch_atomic_dec_return
451#endif /* arch_atomic_dec_return */
452
453#ifndef arch_atomic_dec_return
454static __always_inline int
455arch_atomic_dec_return(atomic_t *v)
456{
457 return arch_atomic_sub_return(1, v);
458}
459#define arch_atomic_dec_return arch_atomic_dec_return
460#endif
461
462#ifndef arch_atomic_dec_return_acquire
463static __always_inline int
464arch_atomic_dec_return_acquire(atomic_t *v)
465{
466 return arch_atomic_sub_return_acquire(1, v);
467}
468#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
469#endif
470
471#ifndef arch_atomic_dec_return_release
472static __always_inline int
473arch_atomic_dec_return_release(atomic_t *v)
474{
475 return arch_atomic_sub_return_release(1, v);
476}
477#define arch_atomic_dec_return_release arch_atomic_dec_return_release
478#endif
479
480#ifndef arch_atomic_dec_return_relaxed
481static __always_inline int
482arch_atomic_dec_return_relaxed(atomic_t *v)
483{
484 return arch_atomic_sub_return_relaxed(1, v);
485}
486#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
487#endif
488
489#else /* arch_atomic_dec_return_relaxed */
490
491#ifndef arch_atomic_dec_return_acquire
492static __always_inline int
493arch_atomic_dec_return_acquire(atomic_t *v)
494{
495 int ret = arch_atomic_dec_return_relaxed(v);
496 __atomic_acquire_fence();
497 return ret;
498}
499#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
500#endif
501
502#ifndef arch_atomic_dec_return_release
503static __always_inline int
504arch_atomic_dec_return_release(atomic_t *v)
505{
506 __atomic_release_fence();
507 return arch_atomic_dec_return_relaxed(v);
508}
509#define arch_atomic_dec_return_release arch_atomic_dec_return_release
510#endif
511
512#ifndef arch_atomic_dec_return
513static __always_inline int
514arch_atomic_dec_return(atomic_t *v)
515{
516 int ret;
517 __atomic_pre_full_fence();
518 ret = arch_atomic_dec_return_relaxed(v);
519 __atomic_post_full_fence();
520 return ret;
521}
522#define arch_atomic_dec_return arch_atomic_dec_return
523#endif
524
525#endif /* arch_atomic_dec_return_relaxed */
526
527#ifndef arch_atomic_fetch_dec_relaxed
528#ifdef arch_atomic_fetch_dec
529#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
530#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
531#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
532#endif /* arch_atomic_fetch_dec */
533
534#ifndef arch_atomic_fetch_dec
535static __always_inline int
536arch_atomic_fetch_dec(atomic_t *v)
537{
538 return arch_atomic_fetch_sub(1, v);
539}
540#define arch_atomic_fetch_dec arch_atomic_fetch_dec
541#endif
542
543#ifndef arch_atomic_fetch_dec_acquire
544static __always_inline int
545arch_atomic_fetch_dec_acquire(atomic_t *v)
546{
547 return arch_atomic_fetch_sub_acquire(1, v);
548}
549#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
550#endif
551
552#ifndef arch_atomic_fetch_dec_release
553static __always_inline int
554arch_atomic_fetch_dec_release(atomic_t *v)
555{
556 return arch_atomic_fetch_sub_release(1, v);
557}
558#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
559#endif
560
561#ifndef arch_atomic_fetch_dec_relaxed
562static __always_inline int
563arch_atomic_fetch_dec_relaxed(atomic_t *v)
564{
565 return arch_atomic_fetch_sub_relaxed(1, v);
566}
567#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
568#endif
569
570#else /* arch_atomic_fetch_dec_relaxed */
571
572#ifndef arch_atomic_fetch_dec_acquire
573static __always_inline int
574arch_atomic_fetch_dec_acquire(atomic_t *v)
575{
576 int ret = arch_atomic_fetch_dec_relaxed(v);
577 __atomic_acquire_fence();
578 return ret;
579}
580#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
581#endif
582
583#ifndef arch_atomic_fetch_dec_release
584static __always_inline int
585arch_atomic_fetch_dec_release(atomic_t *v)
586{
587 __atomic_release_fence();
588 return arch_atomic_fetch_dec_relaxed(v);
589}
590#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
591#endif
592
593#ifndef arch_atomic_fetch_dec
594static __always_inline int
595arch_atomic_fetch_dec(atomic_t *v)
596{
597 int ret;
598 __atomic_pre_full_fence();
599 ret = arch_atomic_fetch_dec_relaxed(v);
600 __atomic_post_full_fence();
601 return ret;
602}
603#define arch_atomic_fetch_dec arch_atomic_fetch_dec
604#endif
605
606#endif /* arch_atomic_fetch_dec_relaxed */
607
608#ifndef arch_atomic_fetch_and_relaxed
609#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
610#define arch_atomic_fetch_and_release arch_atomic_fetch_and
611#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
612#else /* arch_atomic_fetch_and_relaxed */
613
614#ifndef arch_atomic_fetch_and_acquire
615static __always_inline int
616arch_atomic_fetch_and_acquire(int i, atomic_t *v)
617{
618 int ret = arch_atomic_fetch_and_relaxed(i, v);
619 __atomic_acquire_fence();
620 return ret;
621}
622#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
623#endif
624
625#ifndef arch_atomic_fetch_and_release
626static __always_inline int
627arch_atomic_fetch_and_release(int i, atomic_t *v)
628{
629 __atomic_release_fence();
630 return arch_atomic_fetch_and_relaxed(i, v);
631}
632#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
633#endif
634
635#ifndef arch_atomic_fetch_and
636static __always_inline int
637arch_atomic_fetch_and(int i, atomic_t *v)
638{
639 int ret;
640 __atomic_pre_full_fence();
641 ret = arch_atomic_fetch_and_relaxed(i, v);
642 __atomic_post_full_fence();
643 return ret;
644}
645#define arch_atomic_fetch_and arch_atomic_fetch_and
646#endif
647
648#endif /* arch_atomic_fetch_and_relaxed */
649
650#ifndef arch_atomic_andnot
651static __always_inline void
652arch_atomic_andnot(int i, atomic_t *v)
653{
654 arch_atomic_and(~i, v);
655}
656#define arch_atomic_andnot arch_atomic_andnot
657#endif
658
659#ifndef arch_atomic_fetch_andnot_relaxed
660#ifdef arch_atomic_fetch_andnot
661#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
662#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
663#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
664#endif /* arch_atomic_fetch_andnot */
665
666#ifndef arch_atomic_fetch_andnot
667static __always_inline int
668arch_atomic_fetch_andnot(int i, atomic_t *v)
669{
670 return arch_atomic_fetch_and(~i, v);
671}
672#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
673#endif
674
675#ifndef arch_atomic_fetch_andnot_acquire
676static __always_inline int
677arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
678{
679 return arch_atomic_fetch_and_acquire(~i, v);
680}
681#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
682#endif
683
684#ifndef arch_atomic_fetch_andnot_release
685static __always_inline int
686arch_atomic_fetch_andnot_release(int i, atomic_t *v)
687{
688 return arch_atomic_fetch_and_release(~i, v);
689}
690#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
691#endif
692
693#ifndef arch_atomic_fetch_andnot_relaxed
694static __always_inline int
695arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
696{
697 return arch_atomic_fetch_and_relaxed(~i, v);
698}
699#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
700#endif
701
702#else /* arch_atomic_fetch_andnot_relaxed */
703
704#ifndef arch_atomic_fetch_andnot_acquire
705static __always_inline int
706arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
707{
708 int ret = arch_atomic_fetch_andnot_relaxed(i, v);
709 __atomic_acquire_fence();
710 return ret;
711}
712#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
713#endif
714
715#ifndef arch_atomic_fetch_andnot_release
716static __always_inline int
717arch_atomic_fetch_andnot_release(int i, atomic_t *v)
718{
719 __atomic_release_fence();
720 return arch_atomic_fetch_andnot_relaxed(i, v);
721}
722#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
723#endif
724
725#ifndef arch_atomic_fetch_andnot
726static __always_inline int
727arch_atomic_fetch_andnot(int i, atomic_t *v)
728{
729 int ret;
730 __atomic_pre_full_fence();
731 ret = arch_atomic_fetch_andnot_relaxed(i, v);
732 __atomic_post_full_fence();
733 return ret;
734}
735#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
736#endif
737
738#endif /* arch_atomic_fetch_andnot_relaxed */
739
740#ifndef arch_atomic_fetch_or_relaxed
741#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
742#define arch_atomic_fetch_or_release arch_atomic_fetch_or
743#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
744#else /* arch_atomic_fetch_or_relaxed */
745
746#ifndef arch_atomic_fetch_or_acquire
747static __always_inline int
748arch_atomic_fetch_or_acquire(int i, atomic_t *v)
749{
750 int ret = arch_atomic_fetch_or_relaxed(i, v);
751 __atomic_acquire_fence();
752 return ret;
753}
754#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
755#endif
756
757#ifndef arch_atomic_fetch_or_release
758static __always_inline int
759arch_atomic_fetch_or_release(int i, atomic_t *v)
760{
761 __atomic_release_fence();
762 return arch_atomic_fetch_or_relaxed(i, v);
763}
764#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
765#endif
766
767#ifndef arch_atomic_fetch_or
768static __always_inline int
769arch_atomic_fetch_or(int i, atomic_t *v)
770{
771 int ret;
772 __atomic_pre_full_fence();
773 ret = arch_atomic_fetch_or_relaxed(i, v);
774 __atomic_post_full_fence();
775 return ret;
776}
777#define arch_atomic_fetch_or arch_atomic_fetch_or
778#endif
779
780#endif /* arch_atomic_fetch_or_relaxed */
781
782#ifndef arch_atomic_fetch_xor_relaxed
783#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
784#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
785#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
786#else /* arch_atomic_fetch_xor_relaxed */
787
788#ifndef arch_atomic_fetch_xor_acquire
789static __always_inline int
790arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
791{
792 int ret = arch_atomic_fetch_xor_relaxed(i, v);
793 __atomic_acquire_fence();
794 return ret;
795}
796#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
797#endif
798
799#ifndef arch_atomic_fetch_xor_release
800static __always_inline int
801arch_atomic_fetch_xor_release(int i, atomic_t *v)
802{
803 __atomic_release_fence();
804 return arch_atomic_fetch_xor_relaxed(i, v);
805}
806#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
807#endif
808
809#ifndef arch_atomic_fetch_xor
810static __always_inline int
811arch_atomic_fetch_xor(int i, atomic_t *v)
812{
813 int ret;
814 __atomic_pre_full_fence();
815 ret = arch_atomic_fetch_xor_relaxed(i, v);
816 __atomic_post_full_fence();
817 return ret;
818}
819#define arch_atomic_fetch_xor arch_atomic_fetch_xor
820#endif
821
822#endif /* arch_atomic_fetch_xor_relaxed */
823
824#ifndef arch_atomic_xchg_relaxed
825#define arch_atomic_xchg_acquire arch_atomic_xchg
826#define arch_atomic_xchg_release arch_atomic_xchg
827#define arch_atomic_xchg_relaxed arch_atomic_xchg
828#else /* arch_atomic_xchg_relaxed */
829
830#ifndef arch_atomic_xchg_acquire
831static __always_inline int
832arch_atomic_xchg_acquire(atomic_t *v, int i)
833{
834 int ret = arch_atomic_xchg_relaxed(v, i);
835 __atomic_acquire_fence();
836 return ret;
837}
838#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
839#endif
840
841#ifndef arch_atomic_xchg_release
842static __always_inline int
843arch_atomic_xchg_release(atomic_t *v, int i)
844{
845 __atomic_release_fence();
846 return arch_atomic_xchg_relaxed(v, i);
847}
848#define arch_atomic_xchg_release arch_atomic_xchg_release
849#endif
850
851#ifndef arch_atomic_xchg
852static __always_inline int
853arch_atomic_xchg(atomic_t *v, int i)
854{
855 int ret;
856 __atomic_pre_full_fence();
857 ret = arch_atomic_xchg_relaxed(v, i);
858 __atomic_post_full_fence();
859 return ret;
860}
861#define arch_atomic_xchg arch_atomic_xchg
862#endif
863
864#endif /* arch_atomic_xchg_relaxed */
865
866#ifndef arch_atomic_cmpxchg_relaxed
867#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
868#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
869#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
870#else /* arch_atomic_cmpxchg_relaxed */
871
872#ifndef arch_atomic_cmpxchg_acquire
873static __always_inline int
874arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
875{
876 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
877 __atomic_acquire_fence();
878 return ret;
879}
880#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
881#endif
882
883#ifndef arch_atomic_cmpxchg_release
884static __always_inline int
885arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
886{
887 __atomic_release_fence();
888 return arch_atomic_cmpxchg_relaxed(v, old, new);
889}
890#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
891#endif
892
893#ifndef arch_atomic_cmpxchg
894static __always_inline int
895arch_atomic_cmpxchg(atomic_t *v, int old, int new)
896{
897 int ret;
898 __atomic_pre_full_fence();
899 ret = arch_atomic_cmpxchg_relaxed(v, old, new);
900 __atomic_post_full_fence();
901 return ret;
902}
903#define arch_atomic_cmpxchg arch_atomic_cmpxchg
904#endif
905
906#endif /* arch_atomic_cmpxchg_relaxed */
907
908#ifndef arch_atomic_try_cmpxchg_relaxed
909#ifdef arch_atomic_try_cmpxchg
910#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
911#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
912#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
913#endif /* arch_atomic_try_cmpxchg */
914
915#ifndef arch_atomic_try_cmpxchg
916static __always_inline bool
917arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
918{
919 int r, o = *old;
920 r = arch_atomic_cmpxchg(v, o, new);
921 if (unlikely(r != o))
922 *old = r;
923 return likely(r == o);
924}
925#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
926#endif
927
928#ifndef arch_atomic_try_cmpxchg_acquire
929static __always_inline bool
930arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
931{
932 int r, o = *old;
933 r = arch_atomic_cmpxchg_acquire(v, o, new);
934 if (unlikely(r != o))
935 *old = r;
936 return likely(r == o);
937}
938#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
939#endif
940
941#ifndef arch_atomic_try_cmpxchg_release
942static __always_inline bool
943arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
944{
945 int r, o = *old;
946 r = arch_atomic_cmpxchg_release(v, o, new);
947 if (unlikely(r != o))
948 *old = r;
949 return likely(r == o);
950}
951#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
952#endif
953
954#ifndef arch_atomic_try_cmpxchg_relaxed
955static __always_inline bool
956arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
957{
958 int r, o = *old;
959 r = arch_atomic_cmpxchg_relaxed(v, o, new);
960 if (unlikely(r != o))
961 *old = r;
962 return likely(r == o);
963}
964#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
965#endif
966
967#else /* arch_atomic_try_cmpxchg_relaxed */
968
969#ifndef arch_atomic_try_cmpxchg_acquire
970static __always_inline bool
971arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
972{
973 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
974 __atomic_acquire_fence();
975 return ret;
976}
977#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
978#endif
979
980#ifndef arch_atomic_try_cmpxchg_release
981static __always_inline bool
982arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
983{
984 __atomic_release_fence();
985 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
986}
987#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
988#endif
989
990#ifndef arch_atomic_try_cmpxchg
991static __always_inline bool
992arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
993{
994 bool ret;
995 __atomic_pre_full_fence();
996 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
997 __atomic_post_full_fence();
998 return ret;
999}
1000#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1001#endif
1002
1003#endif /* arch_atomic_try_cmpxchg_relaxed */
1004
1005#ifndef arch_atomic_sub_and_test
1006/**
1007 * arch_atomic_sub_and_test - subtract value from variable and test result
1008 * @i: integer value to subtract
1009 * @v: pointer of type atomic_t
1010 *
1011 * Atomically subtracts @i from @v and returns
1012 * true if the result is zero, or false for all
1013 * other cases.
1014 */
1015static __always_inline bool
1016arch_atomic_sub_and_test(int i, atomic_t *v)
1017{
1018 return arch_atomic_sub_return(i, v) == 0;
1019}
1020#define arch_atomic_sub_and_test arch_atomic_sub_and_test
1021#endif
1022
1023#ifndef arch_atomic_dec_and_test
1024/**
1025 * arch_atomic_dec_and_test - decrement and test
1026 * @v: pointer of type atomic_t
1027 *
1028 * Atomically decrements @v by 1 and
1029 * returns true if the result is 0, or false for all other
1030 * cases.
1031 */
1032static __always_inline bool
1033arch_atomic_dec_and_test(atomic_t *v)
1034{
1035 return arch_atomic_dec_return(v) == 0;
1036}
1037#define arch_atomic_dec_and_test arch_atomic_dec_and_test
1038#endif
1039
1040#ifndef arch_atomic_inc_and_test
1041/**
1042 * arch_atomic_inc_and_test - increment and test
1043 * @v: pointer of type atomic_t
1044 *
1045 * Atomically increments @v by 1
1046 * and returns true if the result is zero, or false for all
1047 * other cases.
1048 */
1049static __always_inline bool
1050arch_atomic_inc_and_test(atomic_t *v)
1051{
1052 return arch_atomic_inc_return(v) == 0;
1053}
1054#define arch_atomic_inc_and_test arch_atomic_inc_and_test
1055#endif
1056
1057#ifndef arch_atomic_add_negative
1058/**
1059 * arch_atomic_add_negative - add and test if negative
1060 * @i: integer value to add
1061 * @v: pointer of type atomic_t
1062 *
1063 * Atomically adds @i to @v and returns true
1064 * if the result is negative, or false when
1065 * result is greater than or equal to zero.
1066 */
1067static __always_inline bool
1068arch_atomic_add_negative(int i, atomic_t *v)
1069{
1070 return arch_atomic_add_return(i, v) < 0;
1071}
1072#define arch_atomic_add_negative arch_atomic_add_negative
1073#endif
1074
1075#ifndef arch_atomic_fetch_add_unless
1076/**
1077 * arch_atomic_fetch_add_unless - add unless the number is already a given value
1078 * @v: pointer of type atomic_t
1079 * @a: the amount to add to v...
1080 * @u: ...unless v is equal to u.
1081 *
1082 * Atomically adds @a to @v, so long as @v was not already @u.
1083 * Returns original value of @v
1084 */
1085static __always_inline int
1086arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1087{
1088 int c = arch_atomic_read(v);
1089
1090 do {
1091 if (unlikely(c == u))
1092 break;
1093 } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1094
1095 return c;
1096}
1097#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1098#endif
1099
1100#ifndef arch_atomic_add_unless
1101/**
1102 * arch_atomic_add_unless - add unless the number is already a given value
1103 * @v: pointer of type atomic_t
1104 * @a: the amount to add to v...
1105 * @u: ...unless v is equal to u.
1106 *
1107 * Atomically adds @a to @v, if @v was not already @u.
1108 * Returns true if the addition was done.
1109 */
1110static __always_inline bool
1111arch_atomic_add_unless(atomic_t *v, int a, int u)
1112{
1113 return arch_atomic_fetch_add_unless(v, a, u) != u;
1114}
1115#define arch_atomic_add_unless arch_atomic_add_unless
1116#endif
1117
1118#ifndef arch_atomic_inc_not_zero
1119/**
1120 * arch_atomic_inc_not_zero - increment unless the number is zero
1121 * @v: pointer of type atomic_t
1122 *
1123 * Atomically increments @v by 1, if @v is non-zero.
1124 * Returns true if the increment was done.
1125 */
1126static __always_inline bool
1127arch_atomic_inc_not_zero(atomic_t *v)
1128{
1129 return arch_atomic_add_unless(v, 1, 0);
1130}
1131#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1132#endif
1133
1134#ifndef arch_atomic_inc_unless_negative
1135static __always_inline bool
1136arch_atomic_inc_unless_negative(atomic_t *v)
1137{
1138 int c = arch_atomic_read(v);
1139
1140 do {
1141 if (unlikely(c < 0))
1142 return false;
1143 } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1144
1145 return true;
1146}
1147#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1148#endif
1149
1150#ifndef arch_atomic_dec_unless_positive
1151static __always_inline bool
1152arch_atomic_dec_unless_positive(atomic_t *v)
1153{
1154 int c = arch_atomic_read(v);
1155
1156 do {
1157 if (unlikely(c > 0))
1158 return false;
1159 } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1160
1161 return true;
1162}
1163#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1164#endif
1165
1166#ifndef arch_atomic_dec_if_positive
1167static __always_inline int
1168arch_atomic_dec_if_positive(atomic_t *v)
1169{
1170 int dec, c = arch_atomic_read(v);
1171
1172 do {
1173 dec = c - 1;
1174 if (unlikely(dec < 0))
1175 break;
1176 } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1177
1178 return dec;
1179}
1180#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1181#endif
1182
1183#ifdef CONFIG_GENERIC_ATOMIC64
1184#include <asm-generic/atomic64.h>
1185#endif
1186
1187#ifndef arch_atomic64_read_acquire
1188static __always_inline s64
1189arch_atomic64_read_acquire(const atomic64_t *v)
1190{
1191 return smp_load_acquire(&(v)->counter);
1192}
1193#define arch_atomic64_read_acquire arch_atomic64_read_acquire
1194#endif
1195
1196#ifndef arch_atomic64_set_release
1197static __always_inline void
1198arch_atomic64_set_release(atomic64_t *v, s64 i)
1199{
1200 smp_store_release(&(v)->counter, i);
1201}
1202#define arch_atomic64_set_release arch_atomic64_set_release
1203#endif
1204
1205#ifndef arch_atomic64_add_return_relaxed
1206#define arch_atomic64_add_return_acquire arch_atomic64_add_return
1207#define arch_atomic64_add_return_release arch_atomic64_add_return
1208#define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1209#else /* arch_atomic64_add_return_relaxed */
1210
1211#ifndef arch_atomic64_add_return_acquire
1212static __always_inline s64
1213arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1214{
1215 s64 ret = arch_atomic64_add_return_relaxed(i, v);
1216 __atomic_acquire_fence();
1217 return ret;
1218}
1219#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1220#endif
1221
1222#ifndef arch_atomic64_add_return_release
1223static __always_inline s64
1224arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1225{
1226 __atomic_release_fence();
1227 return arch_atomic64_add_return_relaxed(i, v);
1228}
1229#define arch_atomic64_add_return_release arch_atomic64_add_return_release
1230#endif
1231
1232#ifndef arch_atomic64_add_return
1233static __always_inline s64
1234arch_atomic64_add_return(s64 i, atomic64_t *v)
1235{
1236 s64 ret;
1237 __atomic_pre_full_fence();
1238 ret = arch_atomic64_add_return_relaxed(i, v);
1239 __atomic_post_full_fence();
1240 return ret;
1241}
1242#define arch_atomic64_add_return arch_atomic64_add_return
1243#endif
1244
1245#endif /* arch_atomic64_add_return_relaxed */
1246
1247#ifndef arch_atomic64_fetch_add_relaxed
1248#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1249#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1250#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1251#else /* arch_atomic64_fetch_add_relaxed */
1252
1253#ifndef arch_atomic64_fetch_add_acquire
1254static __always_inline s64
1255arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1256{
1257 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1258 __atomic_acquire_fence();
1259 return ret;
1260}
1261#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1262#endif
1263
1264#ifndef arch_atomic64_fetch_add_release
1265static __always_inline s64
1266arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1267{
1268 __atomic_release_fence();
1269 return arch_atomic64_fetch_add_relaxed(i, v);
1270}
1271#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1272#endif
1273
1274#ifndef arch_atomic64_fetch_add
1275static __always_inline s64
1276arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1277{
1278 s64 ret;
1279 __atomic_pre_full_fence();
1280 ret = arch_atomic64_fetch_add_relaxed(i, v);
1281 __atomic_post_full_fence();
1282 return ret;
1283}
1284#define arch_atomic64_fetch_add arch_atomic64_fetch_add
1285#endif
1286
1287#endif /* arch_atomic64_fetch_add_relaxed */
1288
1289#ifndef arch_atomic64_sub_return_relaxed
1290#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1291#define arch_atomic64_sub_return_release arch_atomic64_sub_return
1292#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1293#else /* arch_atomic64_sub_return_relaxed */
1294
1295#ifndef arch_atomic64_sub_return_acquire
1296static __always_inline s64
1297arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1298{
1299 s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1300 __atomic_acquire_fence();
1301 return ret;
1302}
1303#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1304#endif
1305
1306#ifndef arch_atomic64_sub_return_release
1307static __always_inline s64
1308arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1309{
1310 __atomic_release_fence();
1311 return arch_atomic64_sub_return_relaxed(i, v);
1312}
1313#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1314#endif
1315
1316#ifndef arch_atomic64_sub_return
1317static __always_inline s64
1318arch_atomic64_sub_return(s64 i, atomic64_t *v)
1319{
1320 s64 ret;
1321 __atomic_pre_full_fence();
1322 ret = arch_atomic64_sub_return_relaxed(i, v);
1323 __atomic_post_full_fence();
1324 return ret;
1325}
1326#define arch_atomic64_sub_return arch_atomic64_sub_return
1327#endif
1328
1329#endif /* arch_atomic64_sub_return_relaxed */
1330
1331#ifndef arch_atomic64_fetch_sub_relaxed
1332#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1333#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1334#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1335#else /* arch_atomic64_fetch_sub_relaxed */
1336
1337#ifndef arch_atomic64_fetch_sub_acquire
1338static __always_inline s64
1339arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1340{
1341 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1342 __atomic_acquire_fence();
1343 return ret;
1344}
1345#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1346#endif
1347
1348#ifndef arch_atomic64_fetch_sub_release
1349static __always_inline s64
1350arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1351{
1352 __atomic_release_fence();
1353 return arch_atomic64_fetch_sub_relaxed(i, v);
1354}
1355#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1356#endif
1357
1358#ifndef arch_atomic64_fetch_sub
1359static __always_inline s64
1360arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1361{
1362 s64 ret;
1363 __atomic_pre_full_fence();
1364 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1365 __atomic_post_full_fence();
1366 return ret;
1367}
1368#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1369#endif
1370
1371#endif /* arch_atomic64_fetch_sub_relaxed */
1372
1373#ifndef arch_atomic64_inc
1374static __always_inline void
1375arch_atomic64_inc(atomic64_t *v)
1376{
1377 arch_atomic64_add(1, v);
1378}
1379#define arch_atomic64_inc arch_atomic64_inc
1380#endif
1381
1382#ifndef arch_atomic64_inc_return_relaxed
1383#ifdef arch_atomic64_inc_return
1384#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1385#define arch_atomic64_inc_return_release arch_atomic64_inc_return
1386#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1387#endif /* arch_atomic64_inc_return */
1388
1389#ifndef arch_atomic64_inc_return
1390static __always_inline s64
1391arch_atomic64_inc_return(atomic64_t *v)
1392{
1393 return arch_atomic64_add_return(1, v);
1394}
1395#define arch_atomic64_inc_return arch_atomic64_inc_return
1396#endif
1397
1398#ifndef arch_atomic64_inc_return_acquire
1399static __always_inline s64
1400arch_atomic64_inc_return_acquire(atomic64_t *v)
1401{
1402 return arch_atomic64_add_return_acquire(1, v);
1403}
1404#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1405#endif
1406
1407#ifndef arch_atomic64_inc_return_release
1408static __always_inline s64
1409arch_atomic64_inc_return_release(atomic64_t *v)
1410{
1411 return arch_atomic64_add_return_release(1, v);
1412}
1413#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1414#endif
1415
1416#ifndef arch_atomic64_inc_return_relaxed
1417static __always_inline s64
1418arch_atomic64_inc_return_relaxed(atomic64_t *v)
1419{
1420 return arch_atomic64_add_return_relaxed(1, v);
1421}
1422#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1423#endif
1424
1425#else /* arch_atomic64_inc_return_relaxed */
1426
1427#ifndef arch_atomic64_inc_return_acquire
1428static __always_inline s64
1429arch_atomic64_inc_return_acquire(atomic64_t *v)
1430{
1431 s64 ret = arch_atomic64_inc_return_relaxed(v);
1432 __atomic_acquire_fence();
1433 return ret;
1434}
1435#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1436#endif
1437
1438#ifndef arch_atomic64_inc_return_release
1439static __always_inline s64
1440arch_atomic64_inc_return_release(atomic64_t *v)
1441{
1442 __atomic_release_fence();
1443 return arch_atomic64_inc_return_relaxed(v);
1444}
1445#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1446#endif
1447
1448#ifndef arch_atomic64_inc_return
1449static __always_inline s64
1450arch_atomic64_inc_return(atomic64_t *v)
1451{
1452 s64 ret;
1453 __atomic_pre_full_fence();
1454 ret = arch_atomic64_inc_return_relaxed(v);
1455 __atomic_post_full_fence();
1456 return ret;
1457}
1458#define arch_atomic64_inc_return arch_atomic64_inc_return
1459#endif
1460
1461#endif /* arch_atomic64_inc_return_relaxed */
1462
1463#ifndef arch_atomic64_fetch_inc_relaxed
1464#ifdef arch_atomic64_fetch_inc
1465#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1466#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1467#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1468#endif /* arch_atomic64_fetch_inc */
1469
1470#ifndef arch_atomic64_fetch_inc
1471static __always_inline s64
1472arch_atomic64_fetch_inc(atomic64_t *v)
1473{
1474 return arch_atomic64_fetch_add(1, v);
1475}
1476#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1477#endif
1478
1479#ifndef arch_atomic64_fetch_inc_acquire
1480static __always_inline s64
1481arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1482{
1483 return arch_atomic64_fetch_add_acquire(1, v);
1484}
1485#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1486#endif
1487
1488#ifndef arch_atomic64_fetch_inc_release
1489static __always_inline s64
1490arch_atomic64_fetch_inc_release(atomic64_t *v)
1491{
1492 return arch_atomic64_fetch_add_release(1, v);
1493}
1494#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1495#endif
1496
1497#ifndef arch_atomic64_fetch_inc_relaxed
1498static __always_inline s64
1499arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1500{
1501 return arch_atomic64_fetch_add_relaxed(1, v);
1502}
1503#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1504#endif
1505
1506#else /* arch_atomic64_fetch_inc_relaxed */
1507
1508#ifndef arch_atomic64_fetch_inc_acquire
1509static __always_inline s64
1510arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1511{
1512 s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1513 __atomic_acquire_fence();
1514 return ret;
1515}
1516#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1517#endif
1518
1519#ifndef arch_atomic64_fetch_inc_release
1520static __always_inline s64
1521arch_atomic64_fetch_inc_release(atomic64_t *v)
1522{
1523 __atomic_release_fence();
1524 return arch_atomic64_fetch_inc_relaxed(v);
1525}
1526#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1527#endif
1528
1529#ifndef arch_atomic64_fetch_inc
1530static __always_inline s64
1531arch_atomic64_fetch_inc(atomic64_t *v)
1532{
1533 s64 ret;
1534 __atomic_pre_full_fence();
1535 ret = arch_atomic64_fetch_inc_relaxed(v);
1536 __atomic_post_full_fence();
1537 return ret;
1538}
1539#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1540#endif
1541
1542#endif /* arch_atomic64_fetch_inc_relaxed */
1543
1544#ifndef arch_atomic64_dec
1545static __always_inline void
1546arch_atomic64_dec(atomic64_t *v)
1547{
1548 arch_atomic64_sub(1, v);
1549}
1550#define arch_atomic64_dec arch_atomic64_dec
1551#endif
1552
1553#ifndef arch_atomic64_dec_return_relaxed
1554#ifdef arch_atomic64_dec_return
1555#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1556#define arch_atomic64_dec_return_release arch_atomic64_dec_return
1557#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1558#endif /* arch_atomic64_dec_return */
1559
1560#ifndef arch_atomic64_dec_return
1561static __always_inline s64
1562arch_atomic64_dec_return(atomic64_t *v)
1563{
1564 return arch_atomic64_sub_return(1, v);
1565}
1566#define arch_atomic64_dec_return arch_atomic64_dec_return
1567#endif
1568
1569#ifndef arch_atomic64_dec_return_acquire
1570static __always_inline s64
1571arch_atomic64_dec_return_acquire(atomic64_t *v)
1572{
1573 return arch_atomic64_sub_return_acquire(1, v);
1574}
1575#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1576#endif
1577
1578#ifndef arch_atomic64_dec_return_release
1579static __always_inline s64
1580arch_atomic64_dec_return_release(atomic64_t *v)
1581{
1582 return arch_atomic64_sub_return_release(1, v);
1583}
1584#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1585#endif
1586
1587#ifndef arch_atomic64_dec_return_relaxed
1588static __always_inline s64
1589arch_atomic64_dec_return_relaxed(atomic64_t *v)
1590{
1591 return arch_atomic64_sub_return_relaxed(1, v);
1592}
1593#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1594#endif
1595
1596#else /* arch_atomic64_dec_return_relaxed */
1597
1598#ifndef arch_atomic64_dec_return_acquire
1599static __always_inline s64
1600arch_atomic64_dec_return_acquire(atomic64_t *v)
1601{
1602 s64 ret = arch_atomic64_dec_return_relaxed(v);
1603 __atomic_acquire_fence();
1604 return ret;
1605}
1606#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1607#endif
1608
1609#ifndef arch_atomic64_dec_return_release
1610static __always_inline s64
1611arch_atomic64_dec_return_release(atomic64_t *v)
1612{
1613 __atomic_release_fence();
1614 return arch_atomic64_dec_return_relaxed(v);
1615}
1616#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1617#endif
1618
1619#ifndef arch_atomic64_dec_return
1620static __always_inline s64
1621arch_atomic64_dec_return(atomic64_t *v)
1622{
1623 s64 ret;
1624 __atomic_pre_full_fence();
1625 ret = arch_atomic64_dec_return_relaxed(v);
1626 __atomic_post_full_fence();
1627 return ret;
1628}
1629#define arch_atomic64_dec_return arch_atomic64_dec_return
1630#endif
1631
1632#endif /* arch_atomic64_dec_return_relaxed */
1633
1634#ifndef arch_atomic64_fetch_dec_relaxed
1635#ifdef arch_atomic64_fetch_dec
1636#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1637#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1638#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1639#endif /* arch_atomic64_fetch_dec */
1640
1641#ifndef arch_atomic64_fetch_dec
1642static __always_inline s64
1643arch_atomic64_fetch_dec(atomic64_t *v)
1644{
1645 return arch_atomic64_fetch_sub(1, v);
1646}
1647#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1648#endif
1649
1650#ifndef arch_atomic64_fetch_dec_acquire
1651static __always_inline s64
1652arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1653{
1654 return arch_atomic64_fetch_sub_acquire(1, v);
1655}
1656#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1657#endif
1658
1659#ifndef arch_atomic64_fetch_dec_release
1660static __always_inline s64
1661arch_atomic64_fetch_dec_release(atomic64_t *v)
1662{
1663 return arch_atomic64_fetch_sub_release(1, v);
1664}
1665#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1666#endif
1667
1668#ifndef arch_atomic64_fetch_dec_relaxed
1669static __always_inline s64
1670arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1671{
1672 return arch_atomic64_fetch_sub_relaxed(1, v);
1673}
1674#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1675#endif
1676
1677#else /* arch_atomic64_fetch_dec_relaxed */
1678
1679#ifndef arch_atomic64_fetch_dec_acquire
1680static __always_inline s64
1681arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1682{
1683 s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1684 __atomic_acquire_fence();
1685 return ret;
1686}
1687#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1688#endif
1689
1690#ifndef arch_atomic64_fetch_dec_release
1691static __always_inline s64
1692arch_atomic64_fetch_dec_release(atomic64_t *v)
1693{
1694 __atomic_release_fence();
1695 return arch_atomic64_fetch_dec_relaxed(v);
1696}
1697#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1698#endif
1699
1700#ifndef arch_atomic64_fetch_dec
1701static __always_inline s64
1702arch_atomic64_fetch_dec(atomic64_t *v)
1703{
1704 s64 ret;
1705 __atomic_pre_full_fence();
1706 ret = arch_atomic64_fetch_dec_relaxed(v);
1707 __atomic_post_full_fence();
1708 return ret;
1709}
1710#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1711#endif
1712
1713#endif /* arch_atomic64_fetch_dec_relaxed */
1714
1715#ifndef arch_atomic64_fetch_and_relaxed
1716#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1717#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1718#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1719#else /* arch_atomic64_fetch_and_relaxed */
1720
1721#ifndef arch_atomic64_fetch_and_acquire
1722static __always_inline s64
1723arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1724{
1725 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1726 __atomic_acquire_fence();
1727 return ret;
1728}
1729#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1730#endif
1731
1732#ifndef arch_atomic64_fetch_and_release
1733static __always_inline s64
1734arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1735{
1736 __atomic_release_fence();
1737 return arch_atomic64_fetch_and_relaxed(i, v);
1738}
1739#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1740#endif
1741
1742#ifndef arch_atomic64_fetch_and
1743static __always_inline s64
1744arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1745{
1746 s64 ret;
1747 __atomic_pre_full_fence();
1748 ret = arch_atomic64_fetch_and_relaxed(i, v);
1749 __atomic_post_full_fence();
1750 return ret;
1751}
1752#define arch_atomic64_fetch_and arch_atomic64_fetch_and
1753#endif
1754
1755#endif /* arch_atomic64_fetch_and_relaxed */
1756
1757#ifndef arch_atomic64_andnot
1758static __always_inline void
1759arch_atomic64_andnot(s64 i, atomic64_t *v)
1760{
1761 arch_atomic64_and(~i, v);
1762}
1763#define arch_atomic64_andnot arch_atomic64_andnot
1764#endif
1765
1766#ifndef arch_atomic64_fetch_andnot_relaxed
1767#ifdef arch_atomic64_fetch_andnot
1768#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1769#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1770#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1771#endif /* arch_atomic64_fetch_andnot */
1772
1773#ifndef arch_atomic64_fetch_andnot
1774static __always_inline s64
1775arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1776{
1777 return arch_atomic64_fetch_and(~i, v);
1778}
1779#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1780#endif
1781
1782#ifndef arch_atomic64_fetch_andnot_acquire
1783static __always_inline s64
1784arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1785{
1786 return arch_atomic64_fetch_and_acquire(~i, v);
1787}
1788#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1789#endif
1790
1791#ifndef arch_atomic64_fetch_andnot_release
1792static __always_inline s64
1793arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1794{
1795 return arch_atomic64_fetch_and_release(~i, v);
1796}
1797#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1798#endif
1799
1800#ifndef arch_atomic64_fetch_andnot_relaxed
1801static __always_inline s64
1802arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1803{
1804 return arch_atomic64_fetch_and_relaxed(~i, v);
1805}
1806#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1807#endif
1808
1809#else /* arch_atomic64_fetch_andnot_relaxed */
1810
1811#ifndef arch_atomic64_fetch_andnot_acquire
1812static __always_inline s64
1813arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1814{
1815 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1816 __atomic_acquire_fence();
1817 return ret;
1818}
1819#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1820#endif
1821
1822#ifndef arch_atomic64_fetch_andnot_release
1823static __always_inline s64
1824arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1825{
1826 __atomic_release_fence();
1827 return arch_atomic64_fetch_andnot_relaxed(i, v);
1828}
1829#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1830#endif
1831
1832#ifndef arch_atomic64_fetch_andnot
1833static __always_inline s64
1834arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1835{
1836 s64 ret;
1837 __atomic_pre_full_fence();
1838 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1839 __atomic_post_full_fence();
1840 return ret;
1841}
1842#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1843#endif
1844
1845#endif /* arch_atomic64_fetch_andnot_relaxed */
1846
1847#ifndef arch_atomic64_fetch_or_relaxed
1848#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
1849#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
1850#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
1851#else /* arch_atomic64_fetch_or_relaxed */
1852
1853#ifndef arch_atomic64_fetch_or_acquire
1854static __always_inline s64
1855arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1856{
1857 s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
1858 __atomic_acquire_fence();
1859 return ret;
1860}
1861#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
1862#endif
1863
1864#ifndef arch_atomic64_fetch_or_release
1865static __always_inline s64
1866arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
1867{
1868 __atomic_release_fence();
1869 return arch_atomic64_fetch_or_relaxed(i, v);
1870}
1871#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
1872#endif
1873
1874#ifndef arch_atomic64_fetch_or
1875static __always_inline s64
1876arch_atomic64_fetch_or(s64 i, atomic64_t *v)
1877{
1878 s64 ret;
1879 __atomic_pre_full_fence();
1880 ret = arch_atomic64_fetch_or_relaxed(i, v);
1881 __atomic_post_full_fence();
1882 return ret;
1883}
1884#define arch_atomic64_fetch_or arch_atomic64_fetch_or
1885#endif
1886
1887#endif /* arch_atomic64_fetch_or_relaxed */
1888
1889#ifndef arch_atomic64_fetch_xor_relaxed
1890#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
1891#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
1892#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
1893#else /* arch_atomic64_fetch_xor_relaxed */
1894
1895#ifndef arch_atomic64_fetch_xor_acquire
1896static __always_inline s64
1897arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1898{
1899 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1900 __atomic_acquire_fence();
1901 return ret;
1902}
1903#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
1904#endif
1905
1906#ifndef arch_atomic64_fetch_xor_release
1907static __always_inline s64
1908arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1909{
1910 __atomic_release_fence();
1911 return arch_atomic64_fetch_xor_relaxed(i, v);
1912}
1913#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
1914#endif
1915
1916#ifndef arch_atomic64_fetch_xor
1917static __always_inline s64
1918arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
1919{
1920 s64 ret;
1921 __atomic_pre_full_fence();
1922 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1923 __atomic_post_full_fence();
1924 return ret;
1925}
1926#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
1927#endif
1928
1929#endif /* arch_atomic64_fetch_xor_relaxed */
1930
1931#ifndef arch_atomic64_xchg_relaxed
1932#define arch_atomic64_xchg_acquire arch_atomic64_xchg
1933#define arch_atomic64_xchg_release arch_atomic64_xchg
1934#define arch_atomic64_xchg_relaxed arch_atomic64_xchg
1935#else /* arch_atomic64_xchg_relaxed */
1936
1937#ifndef arch_atomic64_xchg_acquire
1938static __always_inline s64
1939arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
1940{
1941 s64 ret = arch_atomic64_xchg_relaxed(v, i);
1942 __atomic_acquire_fence();
1943 return ret;
1944}
1945#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
1946#endif
1947
1948#ifndef arch_atomic64_xchg_release
1949static __always_inline s64
1950arch_atomic64_xchg_release(atomic64_t *v, s64 i)
1951{
1952 __atomic_release_fence();
1953 return arch_atomic64_xchg_relaxed(v, i);
1954}
1955#define arch_atomic64_xchg_release arch_atomic64_xchg_release
1956#endif
1957
1958#ifndef arch_atomic64_xchg
1959static __always_inline s64
1960arch_atomic64_xchg(atomic64_t *v, s64 i)
1961{
1962 s64 ret;
1963 __atomic_pre_full_fence();
1964 ret = arch_atomic64_xchg_relaxed(v, i);
1965 __atomic_post_full_fence();
1966 return ret;
1967}
1968#define arch_atomic64_xchg arch_atomic64_xchg
1969#endif
1970
1971#endif /* arch_atomic64_xchg_relaxed */
1972
1973#ifndef arch_atomic64_cmpxchg_relaxed
1974#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
1975#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
1976#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
1977#else /* arch_atomic64_cmpxchg_relaxed */
1978
1979#ifndef arch_atomic64_cmpxchg_acquire
1980static __always_inline s64
1981arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1982{
1983 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
1984 __atomic_acquire_fence();
1985 return ret;
1986}
1987#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
1988#endif
1989
1990#ifndef arch_atomic64_cmpxchg_release
1991static __always_inline s64
1992arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1993{
1994 __atomic_release_fence();
1995 return arch_atomic64_cmpxchg_relaxed(v, old, new);
1996}
1997#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
1998#endif
1999
2000#ifndef arch_atomic64_cmpxchg
2001static __always_inline s64
2002arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2003{
2004 s64 ret;
2005 __atomic_pre_full_fence();
2006 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2007 __atomic_post_full_fence();
2008 return ret;
2009}
2010#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2011#endif
2012
2013#endif /* arch_atomic64_cmpxchg_relaxed */
2014
2015#ifndef arch_atomic64_try_cmpxchg_relaxed
2016#ifdef arch_atomic64_try_cmpxchg
2017#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2018#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2019#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2020#endif /* arch_atomic64_try_cmpxchg */
2021
2022#ifndef arch_atomic64_try_cmpxchg
2023static __always_inline bool
2024arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2025{
2026 s64 r, o = *old;
2027 r = arch_atomic64_cmpxchg(v, o, new);
2028 if (unlikely(r != o))
2029 *old = r;
2030 return likely(r == o);
2031}
2032#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2033#endif
2034
2035#ifndef arch_atomic64_try_cmpxchg_acquire
2036static __always_inline bool
2037arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2038{
2039 s64 r, o = *old;
2040 r = arch_atomic64_cmpxchg_acquire(v, o, new);
2041 if (unlikely(r != o))
2042 *old = r;
2043 return likely(r == o);
2044}
2045#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2046#endif
2047
2048#ifndef arch_atomic64_try_cmpxchg_release
2049static __always_inline bool
2050arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2051{
2052 s64 r, o = *old;
2053 r = arch_atomic64_cmpxchg_release(v, o, new);
2054 if (unlikely(r != o))
2055 *old = r;
2056 return likely(r == o);
2057}
2058#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2059#endif
2060
2061#ifndef arch_atomic64_try_cmpxchg_relaxed
2062static __always_inline bool
2063arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2064{
2065 s64 r, o = *old;
2066 r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2067 if (unlikely(r != o))
2068 *old = r;
2069 return likely(r == o);
2070}
2071#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2072#endif
2073
2074#else /* arch_atomic64_try_cmpxchg_relaxed */
2075
2076#ifndef arch_atomic64_try_cmpxchg_acquire
2077static __always_inline bool
2078arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2079{
2080 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2081 __atomic_acquire_fence();
2082 return ret;
2083}
2084#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2085#endif
2086
2087#ifndef arch_atomic64_try_cmpxchg_release
2088static __always_inline bool
2089arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2090{
2091 __atomic_release_fence();
2092 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2093}
2094#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2095#endif
2096
2097#ifndef arch_atomic64_try_cmpxchg
2098static __always_inline bool
2099arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2100{
2101 bool ret;
2102 __atomic_pre_full_fence();
2103 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2104 __atomic_post_full_fence();
2105 return ret;
2106}
2107#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2108#endif
2109
2110#endif /* arch_atomic64_try_cmpxchg_relaxed */
2111
2112#ifndef arch_atomic64_sub_and_test
2113/**
2114 * arch_atomic64_sub_and_test - subtract value from variable and test result
2115 * @i: integer value to subtract
2116 * @v: pointer of type atomic64_t
2117 *
2118 * Atomically subtracts @i from @v and returns
2119 * true if the result is zero, or false for all
2120 * other cases.
2121 */
2122static __always_inline bool
2123arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2124{
2125 return arch_atomic64_sub_return(i, v) == 0;
2126}
2127#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2128#endif
2129
2130#ifndef arch_atomic64_dec_and_test
2131/**
2132 * arch_atomic64_dec_and_test - decrement and test
2133 * @v: pointer of type atomic64_t
2134 *
2135 * Atomically decrements @v by 1 and
2136 * returns true if the result is 0, or false for all other
2137 * cases.
2138 */
2139static __always_inline bool
2140arch_atomic64_dec_and_test(atomic64_t *v)
2141{
2142 return arch_atomic64_dec_return(v) == 0;
2143}
2144#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2145#endif
2146
2147#ifndef arch_atomic64_inc_and_test
2148/**
2149 * arch_atomic64_inc_and_test - increment and test
2150 * @v: pointer of type atomic64_t
2151 *
2152 * Atomically increments @v by 1
2153 * and returns true if the result is zero, or false for all
2154 * other cases.
2155 */
2156static __always_inline bool
2157arch_atomic64_inc_and_test(atomic64_t *v)
2158{
2159 return arch_atomic64_inc_return(v) == 0;
2160}
2161#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2162#endif
2163
2164#ifndef arch_atomic64_add_negative
2165/**
2166 * arch_atomic64_add_negative - add and test if negative
2167 * @i: integer value to add
2168 * @v: pointer of type atomic64_t
2169 *
2170 * Atomically adds @i to @v and returns true
2171 * if the result is negative, or false when
2172 * result is greater than or equal to zero.
2173 */
2174static __always_inline bool
2175arch_atomic64_add_negative(s64 i, atomic64_t *v)
2176{
2177 return arch_atomic64_add_return(i, v) < 0;
2178}
2179#define arch_atomic64_add_negative arch_atomic64_add_negative
2180#endif
2181
2182#ifndef arch_atomic64_fetch_add_unless
2183/**
2184 * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2185 * @v: pointer of type atomic64_t
2186 * @a: the amount to add to v...
2187 * @u: ...unless v is equal to u.
2188 *
2189 * Atomically adds @a to @v, so long as @v was not already @u.
2190 * Returns original value of @v
2191 */
2192static __always_inline s64
2193arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2194{
2195 s64 c = arch_atomic64_read(v);
2196
2197 do {
2198 if (unlikely(c == u))
2199 break;
2200 } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2201
2202 return c;
2203}
2204#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2205#endif
2206
2207#ifndef arch_atomic64_add_unless
2208/**
2209 * arch_atomic64_add_unless - add unless the number is already a given value
2210 * @v: pointer of type atomic64_t
2211 * @a: the amount to add to v...
2212 * @u: ...unless v is equal to u.
2213 *
2214 * Atomically adds @a to @v, if @v was not already @u.
2215 * Returns true if the addition was done.
2216 */
2217static __always_inline bool
2218arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2219{
2220 return arch_atomic64_fetch_add_unless(v, a, u) != u;
2221}
2222#define arch_atomic64_add_unless arch_atomic64_add_unless
2223#endif
2224
2225#ifndef arch_atomic64_inc_not_zero
2226/**
2227 * arch_atomic64_inc_not_zero - increment unless the number is zero
2228 * @v: pointer of type atomic64_t
2229 *
2230 * Atomically increments @v by 1, if @v is non-zero.
2231 * Returns true if the increment was done.
2232 */
2233static __always_inline bool
2234arch_atomic64_inc_not_zero(atomic64_t *v)
2235{
2236 return arch_atomic64_add_unless(v, 1, 0);
2237}
2238#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2239#endif
2240
2241#ifndef arch_atomic64_inc_unless_negative
2242static __always_inline bool
2243arch_atomic64_inc_unless_negative(atomic64_t *v)
2244{
2245 s64 c = arch_atomic64_read(v);
2246
2247 do {
2248 if (unlikely(c < 0))
2249 return false;
2250 } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2251
2252 return true;
2253}
2254#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2255#endif
2256
2257#ifndef arch_atomic64_dec_unless_positive
2258static __always_inline bool
2259arch_atomic64_dec_unless_positive(atomic64_t *v)
2260{
2261 s64 c = arch_atomic64_read(v);
2262
2263 do {
2264 if (unlikely(c > 0))
2265 return false;
2266 } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2267
2268 return true;
2269}
2270#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2271#endif
2272
2273#ifndef arch_atomic64_dec_if_positive
2274static __always_inline s64
2275arch_atomic64_dec_if_positive(atomic64_t *v)
2276{
2277 s64 dec, c = arch_atomic64_read(v);
2278
2279 do {
2280 dec = c - 1;
2281 if (unlikely(dec < 0))
2282 break;
2283 } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2284
2285 return dec;
2286}
2287#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2288#endif
2289
2290#endif /* _LINUX_ATOMIC_FALLBACK_H */
2291// 90cd26cfd69d2250303d654955a0cc12620fb91b
include/linux/atomic-fallback.h+171−175
@@ -6,6 +6,8 @@ · 5 unmodified lines
6#ifndef _LINUX_ATOMIC_FALLBACK_H6#ifndef _LINUX_ATOMIC_FALLBACK_H
7#define _LINUX_ATOMIC_FALLBACK_H7#define _LINUX_ATOMIC_FALLBACK_H
8 8
9#include <linux/compiler.h>
10
9#ifndef xchg_relaxed11#ifndef xchg_relaxed
10#define xchg_relaxed xchg12#define xchg_relaxed xchg
11#define xchg_acquire xchg13#define xchg_acquire xchg
@@ -76,7 +78,7 @@ · 64 unmodified lines
76#endif /* cmpxchg64_relaxed */78#endif /* cmpxchg64_relaxed */
77 79
78#ifndef atomic_read_acquire80#ifndef atomic_read_acquire
79static inline int81static __always_inline int
80atomic_read_acquire(const atomic_t *v)82atomic_read_acquire(const atomic_t *v)
81{83{
82 return smp_load_acquire(&(v)->counter);84 return smp_load_acquire(&(v)->counter);
@@ -85,7 +87,7 @@ atomic_read_acquire(const atomic_t *v) · 2 unmodified lines
85#endif87#endif
86 88
87#ifndef atomic_set_release89#ifndef atomic_set_release
88static inline void90static __always_inline void
89atomic_set_release(atomic_t *v, int i)91atomic_set_release(atomic_t *v, int i)
90{92{
91 smp_store_release(&(v)->counter, i);93 smp_store_release(&(v)->counter, i);
@@ -100,7 +102,7 @@ atomic_set_release(atomic_t *v, int i) · 8 unmodified lines
100#else /* atomic_add_return_relaxed */102#else /* atomic_add_return_relaxed */
101 103
102#ifndef atomic_add_return_acquire104#ifndef atomic_add_return_acquire
103static inline int105static __always_inline int
104atomic_add_return_acquire(int i, atomic_t *v)106atomic_add_return_acquire(int i, atomic_t *v)
105{107{
106 int ret = atomic_add_return_relaxed(i, v);108 int ret = atomic_add_return_relaxed(i, v);
@@ -111,7 +113,7 @@ atomic_add_return_acquire(int i, atomic_t *v) · 4 unmodified lines
111#endif113#endif
112 114
113#ifndef atomic_add_return_release115#ifndef atomic_add_return_release
114static inline int116static __always_inline int
115atomic_add_return_release(int i, atomic_t *v)117atomic_add_return_release(int i, atomic_t *v)
116{118{
117 __atomic_release_fence();119 __atomic_release_fence();
@@ -121,7 +123,7 @@ atomic_add_return_release(int i, atomic_t *v) · 3 unmodified lines
121#endif123#endif
122 124
123#ifndef atomic_add_return125#ifndef atomic_add_return
124static inline int126static __always_inline int
125atomic_add_return(int i, atomic_t *v)127atomic_add_return(int i, atomic_t *v)
126{128{
127 int ret;129 int ret;
@@ -142,7 +144,7 @@ atomic_add_return(int i, atomic_t *v) · 14 unmodified lines
142#else /* atomic_fetch_add_relaxed */144#else /* atomic_fetch_add_relaxed */
143 145
144#ifndef atomic_fetch_add_acquire146#ifndef atomic_fetch_add_acquire
145static inline int147static __always_inline int
146atomic_fetch_add_acquire(int i, atomic_t *v)148atomic_fetch_add_acquire(int i, atomic_t *v)
147{149{
148 int ret = atomic_fetch_add_relaxed(i, v);150 int ret = atomic_fetch_add_relaxed(i, v);
@@ -153,7 +155,7 @@ atomic_fetch_add_acquire(int i, atomic_t *v) · 4 unmodified lines
153#endif155#endif
154 156
155#ifndef atomic_fetch_add_release157#ifndef atomic_fetch_add_release
156static inline int158static __always_inline int
157atomic_fetch_add_release(int i, atomic_t *v)159atomic_fetch_add_release(int i, atomic_t *v)
158{160{
159 __atomic_release_fence();161 __atomic_release_fence();
@@ -163,7 +165,7 @@ atomic_fetch_add_release(int i, atomic_t *v) · 3 unmodified lines
163#endif165#endif
164 166
165#ifndef atomic_fetch_add167#ifndef atomic_fetch_add
166static inline int168static __always_inline int
167atomic_fetch_add(int i, atomic_t *v)169atomic_fetch_add(int i, atomic_t *v)
168{170{
169 int ret;171 int ret;
@@ -184,7 +186,7 @@ atomic_fetch_add(int i, atomic_t *v) · 14 unmodified lines
184#else /* atomic_sub_return_relaxed */186#else /* atomic_sub_return_relaxed */
185 187
186#ifndef atomic_sub_return_acquire188#ifndef atomic_sub_return_acquire
187static inline int189static __always_inline int
188atomic_sub_return_acquire(int i, atomic_t *v)190atomic_sub_return_acquire(int i, atomic_t *v)
189{191{
190 int ret = atomic_sub_return_relaxed(i, v);192 int ret = atomic_sub_return_relaxed(i, v);
@@ -195,7 +197,7 @@ atomic_sub_return_acquire(int i, atomic_t *v) · 4 unmodified lines
195#endif197#endif
196 198
197#ifndef atomic_sub_return_release199#ifndef atomic_sub_return_release
198static inline int200static __always_inline int
199atomic_sub_return_release(int i, atomic_t *v)201atomic_sub_return_release(int i, atomic_t *v)
200{202{
201 __atomic_release_fence();203 __atomic_release_fence();
@@ -205,7 +207,7 @@ atomic_sub_return_release(int i, atomic_t *v) · 3 unmodified lines
205#endif207#endif
206 208
207#ifndef atomic_sub_return209#ifndef atomic_sub_return
208static inline int210static __always_inline int
209atomic_sub_return(int i, atomic_t *v)211atomic_sub_return(int i, atomic_t *v)
210{212{
211 int ret;213 int ret;
@@ -226,7 +228,7 @@ atomic_sub_return(int i, atomic_t *v) · 14 unmodified lines
226#else /* atomic_fetch_sub_relaxed */228#else /* atomic_fetch_sub_relaxed */
227 229
228#ifndef atomic_fetch_sub_acquire230#ifndef atomic_fetch_sub_acquire
229static inline int231static __always_inline int
230atomic_fetch_sub_acquire(int i, atomic_t *v)232atomic_fetch_sub_acquire(int i, atomic_t *v)
231{233{
232 int ret = atomic_fetch_sub_relaxed(i, v);234 int ret = atomic_fetch_sub_relaxed(i, v);
@@ -237,7 +239,7 @@ atomic_fetch_sub_acquire(int i, atomic_t *v) · 4 unmodified lines
237#endif239#endif
238 240
239#ifndef atomic_fetch_sub_release241#ifndef atomic_fetch_sub_release
240static inline int242static __always_inline int
241atomic_fetch_sub_release(int i, atomic_t *v)243atomic_fetch_sub_release(int i, atomic_t *v)
242{244{
243 __atomic_release_fence();245 __atomic_release_fence();
@@ -247,7 +249,7 @@ atomic_fetch_sub_release(int i, atomic_t *v) · 3 unmodified lines
247#endif249#endif
248 250
249#ifndef atomic_fetch_sub251#ifndef atomic_fetch_sub
250static inline int252static __always_inline int
251atomic_fetch_sub(int i, atomic_t *v)253atomic_fetch_sub(int i, atomic_t *v)
252{254{
253 int ret;255 int ret;
@@ -262,7 +264,7 @@ atomic_fetch_sub(int i, atomic_t *v) · 8 unmodified lines
262#endif /* atomic_fetch_sub_relaxed */264#endif /* atomic_fetch_sub_relaxed */
263 265
264#ifndef atomic_inc266#ifndef atomic_inc
265static inline void267static __always_inline void
266atomic_inc(atomic_t *v)268atomic_inc(atomic_t *v)
267{269{
268 atomic_add(1, v);270 atomic_add(1, v);
@@ -278,7 +280,7 @@ atomic_inc(atomic_t *v) · 9 unmodified lines
278#endif /* atomic_inc_return */280#endif /* atomic_inc_return */
279 281
280#ifndef atomic_inc_return282#ifndef atomic_inc_return
281static inline int283static __always_inline int
282atomic_inc_return(atomic_t *v)284atomic_inc_return(atomic_t *v)
283{285{
284 return atomic_add_return(1, v);286 return atomic_add_return(1, v);
@@ -287,7 +289,7 @@ atomic_inc_return(atomic_t *v) · 2 unmodified lines
287#endif289#endif
288 290
289#ifndef atomic_inc_return_acquire291#ifndef atomic_inc_return_acquire
290static inline int292static __always_inline int
291atomic_inc_return_acquire(atomic_t *v)293atomic_inc_return_acquire(atomic_t *v)
292{294{
293 return atomic_add_return_acquire(1, v);295 return atomic_add_return_acquire(1, v);
@@ -296,7 +298,7 @@ atomic_inc_return_acquire(atomic_t *v) · 2 unmodified lines
296#endif298#endif
297 299
298#ifndef atomic_inc_return_release300#ifndef atomic_inc_return_release
299static inline int301static __always_inline int
300atomic_inc_return_release(atomic_t *v)302atomic_inc_return_release(atomic_t *v)
301{303{
302 return atomic_add_return_release(1, v);304 return atomic_add_return_release(1, v);
@@ -305,7 +307,7 @@ atomic_inc_return_release(atomic_t *v) · 2 unmodified lines
305#endif307#endif
306 308
307#ifndef atomic_inc_return_relaxed309#ifndef atomic_inc_return_relaxed
308static inline int310static __always_inline int
309atomic_inc_return_relaxed(atomic_t *v)311atomic_inc_return_relaxed(atomic_t *v)
310{312{
311 return atomic_add_return_relaxed(1, v);313 return atomic_add_return_relaxed(1, v);
@@ -316,7 +318,7 @@ atomic_inc_return_relaxed(atomic_t *v) · 4 unmodified lines
316#else /* atomic_inc_return_relaxed */318#else /* atomic_inc_return_relaxed */
317 319
318#ifndef atomic_inc_return_acquire320#ifndef atomic_inc_return_acquire
319static inline int321static __always_inline int
320atomic_inc_return_acquire(atomic_t *v)322atomic_inc_return_acquire(atomic_t *v)
321{323{
322 int ret = atomic_inc_return_relaxed(v);324 int ret = atomic_inc_return_relaxed(v);
@@ -327,7 +329,7 @@ atomic_inc_return_acquire(atomic_t *v) · 4 unmodified lines
327#endif329#endif
328 330
329#ifndef atomic_inc_return_release331#ifndef atomic_inc_return_release
330static inline int332static __always_inline int
331atomic_inc_return_release(atomic_t *v)333atomic_inc_return_release(atomic_t *v)
332{334{
333 __atomic_release_fence();335 __atomic_release_fence();
@@ -337,7 +339,7 @@ atomic_inc_return_release(atomic_t *v) · 3 unmodified lines
337#endif339#endif
338 340
339#ifndef atomic_inc_return341#ifndef atomic_inc_return
340static inline int342static __always_inline int
341atomic_inc_return(atomic_t *v)343atomic_inc_return(atomic_t *v)
342{344{
343 int ret;345 int ret;
@@ -359,7 +361,7 @@ atomic_inc_return(atomic_t *v) · 15 unmodified lines
359#endif /* atomic_fetch_inc */361#endif /* atomic_fetch_inc */
360 362
361#ifndef atomic_fetch_inc363#ifndef atomic_fetch_inc
362static inline int364static __always_inline int
363atomic_fetch_inc(atomic_t *v)365atomic_fetch_inc(atomic_t *v)
364{366{
365 return atomic_fetch_add(1, v);367 return atomic_fetch_add(1, v);
@@ -368,7 +370,7 @@ atomic_fetch_inc(atomic_t *v) · 2 unmodified lines
368#endif370#endif
369 371
370#ifndef atomic_fetch_inc_acquire372#ifndef atomic_fetch_inc_acquire
371static inline int373static __always_inline int
372atomic_fetch_inc_acquire(atomic_t *v)374atomic_fetch_inc_acquire(atomic_t *v)
373{375{
374 return atomic_fetch_add_acquire(1, v);376 return atomic_fetch_add_acquire(1, v);
@@ -377,7 +379,7 @@ atomic_fetch_inc_acquire(atomic_t *v) · 2 unmodified lines
377#endif379#endif
378 380
379#ifndef atomic_fetch_inc_release381#ifndef atomic_fetch_inc_release
380static inline int382static __always_inline int
381atomic_fetch_inc_release(atomic_t *v)383atomic_fetch_inc_release(atomic_t *v)
382{384{
383 return atomic_fetch_add_release(1, v);385 return atomic_fetch_add_release(1, v);
@@ -386,7 +388,7 @@ atomic_fetch_inc_release(atomic_t *v) · 2 unmodified lines
386#endif388#endif
387 389
388#ifndef atomic_fetch_inc_relaxed390#ifndef atomic_fetch_inc_relaxed
389static inline int391static __always_inline int
390atomic_fetch_inc_relaxed(atomic_t *v)392atomic_fetch_inc_relaxed(atomic_t *v)
391{393{
392 return atomic_fetch_add_relaxed(1, v);394 return atomic_fetch_add_relaxed(1, v);
@@ -397,7 +399,7 @@ atomic_fetch_inc_relaxed(atomic_t *v) · 4 unmodified lines
397#else /* atomic_fetch_inc_relaxed */399#else /* atomic_fetch_inc_relaxed */
398 400
399#ifndef atomic_fetch_inc_acquire401#ifndef atomic_fetch_inc_acquire
400static inline int402static __always_inline int
401atomic_fetch_inc_acquire(atomic_t *v)403atomic_fetch_inc_acquire(atomic_t *v)
402{404{
403 int ret = atomic_fetch_inc_relaxed(v);405 int ret = atomic_fetch_inc_relaxed(v);
@@ -408,7 +410,7 @@ atomic_fetch_inc_acquire(atomic_t *v) · 4 unmodified lines
408#endif410#endif
409 411
410#ifndef atomic_fetch_inc_release412#ifndef atomic_fetch_inc_release
411static inline int413static __always_inline int
412atomic_fetch_inc_release(atomic_t *v)414atomic_fetch_inc_release(atomic_t *v)
413{415{
414 __atomic_release_fence();416 __atomic_release_fence();
@@ -418,7 +420,7 @@ atomic_fetch_inc_release(atomic_t *v) · 3 unmodified lines
418#endif420#endif
419 421
420#ifndef atomic_fetch_inc422#ifndef atomic_fetch_inc
421static inline int423static __always_inline int
422atomic_fetch_inc(atomic_t *v)424atomic_fetch_inc(atomic_t *v)
423{425{
424 int ret;426 int ret;
@@ -433,7 +435,7 @@ atomic_fetch_inc(atomic_t *v) · 8 unmodified lines
433#endif /* atomic_fetch_inc_relaxed */435#endif /* atomic_fetch_inc_relaxed */
434 436
435#ifndef atomic_dec437#ifndef atomic_dec
436static inline void438static __always_inline void
437atomic_dec(atomic_t *v)439atomic_dec(atomic_t *v)
438{440{
439 atomic_sub(1, v);441 atomic_sub(1, v);
@@ -449,7 +451,7 @@ atomic_dec(atomic_t *v) · 9 unmodified lines
449#endif /* atomic_dec_return */451#endif /* atomic_dec_return */
450 452
451#ifndef atomic_dec_return453#ifndef atomic_dec_return
452static inline int454static __always_inline int
453atomic_dec_return(atomic_t *v)455atomic_dec_return(atomic_t *v)
454{456{
455 return atomic_sub_return(1, v);457 return atomic_sub_return(1, v);
@@ -458,7 +460,7 @@ atomic_dec_return(atomic_t *v) · 2 unmodified lines
458#endif460#endif
459 461
460#ifndef atomic_dec_return_acquire462#ifndef atomic_dec_return_acquire
461static inline int463static __always_inline int
462atomic_dec_return_acquire(atomic_t *v)464atomic_dec_return_acquire(atomic_t *v)
463{465{
464 return atomic_sub_return_acquire(1, v);466 return atomic_sub_return_acquire(1, v);
@@ -467,7 +469,7 @@ atomic_dec_return_acquire(atomic_t *v) · 2 unmodified lines
467#endif469#endif
468 470
469#ifndef atomic_dec_return_release471#ifndef atomic_dec_return_release
470static inline int472static __always_inline int
471atomic_dec_return_release(atomic_t *v)473atomic_dec_return_release(atomic_t *v)
472{474{
473 return atomic_sub_return_release(1, v);475 return atomic_sub_return_release(1, v);
@@ -476,7 +478,7 @@ atomic_dec_return_release(atomic_t *v) · 2 unmodified lines
476#endif478#endif
477 479
478#ifndef atomic_dec_return_relaxed480#ifndef atomic_dec_return_relaxed
479static inline int481static __always_inline int
480atomic_dec_return_relaxed(atomic_t *v)482atomic_dec_return_relaxed(atomic_t *v)
481{483{
482 return atomic_sub_return_relaxed(1, v);484 return atomic_sub_return_relaxed(1, v);
@@ -487,7 +489,7 @@ atomic_dec_return_relaxed(atomic_t *v) · 4 unmodified lines
487#else /* atomic_dec_return_relaxed */489#else /* atomic_dec_return_relaxed */
488 490
489#ifndef atomic_dec_return_acquire491#ifndef atomic_dec_return_acquire
490static inline int492static __always_inline int
491atomic_dec_return_acquire(atomic_t *v)493atomic_dec_return_acquire(atomic_t *v)
492{494{
493 int ret = atomic_dec_return_relaxed(v);495 int ret = atomic_dec_return_relaxed(v);
@@ -498,7 +500,7 @@ atomic_dec_return_acquire(atomic_t *v) · 4 unmodified lines
498#endif500#endif
499 501
500#ifndef atomic_dec_return_release502#ifndef atomic_dec_return_release
501static inline int503static __always_inline int
502atomic_dec_return_release(atomic_t *v)504atomic_dec_return_release(atomic_t *v)
503{505{
504 __atomic_release_fence();506 __atomic_release_fence();
@@ -508,7 +510,7 @@ atomic_dec_return_release(atomic_t *v) · 3 unmodified lines
508#endif510#endif
509 511
510#ifndef atomic_dec_return512#ifndef atomic_dec_return
511static inline int513static __always_inline int
512atomic_dec_return(atomic_t *v)514atomic_dec_return(atomic_t *v)
513{515{
514 int ret;516 int ret;
@@ -530,7 +532,7 @@ atomic_dec_return(atomic_t *v) · 15 unmodified lines
530#endif /* atomic_fetch_dec */532#endif /* atomic_fetch_dec */
531 533
532#ifndef atomic_fetch_dec534#ifndef atomic_fetch_dec
533static inline int535static __always_inline int
534atomic_fetch_dec(atomic_t *v)536atomic_fetch_dec(atomic_t *v)
535{537{
536 return atomic_fetch_sub(1, v);538 return atomic_fetch_sub(1, v);
@@ -539,7 +541,7 @@ atomic_fetch_dec(atomic_t *v) · 2 unmodified lines
539#endif541#endif
540 542
541#ifndef atomic_fetch_dec_acquire543#ifndef atomic_fetch_dec_acquire
542static inline int544static __always_inline int
543atomic_fetch_dec_acquire(atomic_t *v)545atomic_fetch_dec_acquire(atomic_t *v)
544{546{
545 return atomic_fetch_sub_acquire(1, v);547 return atomic_fetch_sub_acquire(1, v);
@@ -548,7 +550,7 @@ atomic_fetch_dec_acquire(atomic_t *v) · 2 unmodified lines
548#endif550#endif
549 551
550#ifndef atomic_fetch_dec_release552#ifndef atomic_fetch_dec_release
551static inline int553static __always_inline int
552atomic_fetch_dec_release(atomic_t *v)554atomic_fetch_dec_release(atomic_t *v)
553{555{
554 return atomic_fetch_sub_release(1, v);556 return atomic_fetch_sub_release(1, v);
@@ -557,7 +559,7 @@ atomic_fetch_dec_release(atomic_t *v) · 2 unmodified lines
557#endif559#endif
558 560
559#ifndef atomic_fetch_dec_relaxed561#ifndef atomic_fetch_dec_relaxed
560static inline int562static __always_inline int
561atomic_fetch_dec_relaxed(atomic_t *v)563atomic_fetch_dec_relaxed(atomic_t *v)
562{564{
563 return atomic_fetch_sub_relaxed(1, v);565 return atomic_fetch_sub_relaxed(1, v);
@@ -568,7 +570,7 @@ atomic_fetch_dec_relaxed(atomic_t *v) · 4 unmodified lines
568#else /* atomic_fetch_dec_relaxed */570#else /* atomic_fetch_dec_relaxed */
569 571
570#ifndef atomic_fetch_dec_acquire572#ifndef atomic_fetch_dec_acquire
571static inline int573static __always_inline int
572atomic_fetch_dec_acquire(atomic_t *v)574atomic_fetch_dec_acquire(atomic_t *v)
573{575{
574 int ret = atomic_fetch_dec_relaxed(v);576 int ret = atomic_fetch_dec_relaxed(v);
@@ -579,7 +581,7 @@ atomic_fetch_dec_acquire(atomic_t *v) · 4 unmodified lines
579#endif581#endif
580 582
581#ifndef atomic_fetch_dec_release583#ifndef atomic_fetch_dec_release
582static inline int584static __always_inline int
583atomic_fetch_dec_release(atomic_t *v)585atomic_fetch_dec_release(atomic_t *v)
584{586{
585 __atomic_release_fence();587 __atomic_release_fence();
@@ -589,7 +591,7 @@ atomic_fetch_dec_release(atomic_t *v) · 3 unmodified lines
589#endif591#endif
590 592
591#ifndef atomic_fetch_dec593#ifndef atomic_fetch_dec
592static inline int594static __always_inline int
593atomic_fetch_dec(atomic_t *v)595atomic_fetch_dec(atomic_t *v)
594{596{
595 int ret;597 int ret;
@@ -610,7 +612,7 @@ atomic_fetch_dec(atomic_t *v) · 14 unmodified lines
610#else /* atomic_fetch_and_relaxed */612#else /* atomic_fetch_and_relaxed */
611 613
612#ifndef atomic_fetch_and_acquire614#ifndef atomic_fetch_and_acquire
613static inline int615static __always_inline int
614atomic_fetch_and_acquire(int i, atomic_t *v)616atomic_fetch_and_acquire(int i, atomic_t *v)
615{617{
616 int ret = atomic_fetch_and_relaxed(i, v);618 int ret = atomic_fetch_and_relaxed(i, v);
@@ -621,7 +623,7 @@ atomic_fetch_and_acquire(int i, atomic_t *v) · 4 unmodified lines
621#endif623#endif
622 624
623#ifndef atomic_fetch_and_release625#ifndef atomic_fetch_and_release
624static inline int626static __always_inline int
625atomic_fetch_and_release(int i, atomic_t *v)627atomic_fetch_and_release(int i, atomic_t *v)
626{628{
627 __atomic_release_fence();629 __atomic_release_fence();
@@ -631,7 +633,7 @@ atomic_fetch_and_release(int i, atomic_t *v) · 3 unmodified lines
631#endif633#endif
632 634
633#ifndef atomic_fetch_and635#ifndef atomic_fetch_and
634static inline int636static __always_inline int
635atomic_fetch_and(int i, atomic_t *v)637atomic_fetch_and(int i, atomic_t *v)
636{638{
637 int ret;639 int ret;
@@ -646,7 +648,7 @@ atomic_fetch_and(int i, atomic_t *v) · 8 unmodified lines
646#endif /* atomic_fetch_and_relaxed */648#endif /* atomic_fetch_and_relaxed */
647 649
648#ifndef atomic_andnot650#ifndef atomic_andnot
649static inline void651static __always_inline void
650atomic_andnot(int i, atomic_t *v)652atomic_andnot(int i, atomic_t *v)
651{653{
652 atomic_and(~i, v);654 atomic_and(~i, v);
@@ -662,7 +664,7 @@ atomic_andnot(int i, atomic_t *v) · 9 unmodified lines
662#endif /* atomic_fetch_andnot */664#endif /* atomic_fetch_andnot */
663 665
664#ifndef atomic_fetch_andnot666#ifndef atomic_fetch_andnot
665static inline int667static __always_inline int
666atomic_fetch_andnot(int i, atomic_t *v)668atomic_fetch_andnot(int i, atomic_t *v)
667{669{
668 return atomic_fetch_and(~i, v);670 return atomic_fetch_and(~i, v);
@@ -671,7 +673,7 @@ atomic_fetch_andnot(int i, atomic_t *v) · 2 unmodified lines
671#endif673#endif
672 674
673#ifndef atomic_fetch_andnot_acquire675#ifndef atomic_fetch_andnot_acquire
674static inline int676static __always_inline int
675atomic_fetch_andnot_acquire(int i, atomic_t *v)677atomic_fetch_andnot_acquire(int i, atomic_t *v)
676{678{
677 return atomic_fetch_and_acquire(~i, v);679 return atomic_fetch_and_acquire(~i, v);
@@ -680,7 +682,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) · 2 unmodified lines
680#endif682#endif
681 683
682#ifndef atomic_fetch_andnot_release684#ifndef atomic_fetch_andnot_release
683static inline int685static __always_inline int
684atomic_fetch_andnot_release(int i, atomic_t *v)686atomic_fetch_andnot_release(int i, atomic_t *v)
685{687{
686 return atomic_fetch_and_release(~i, v);688 return atomic_fetch_and_release(~i, v);
@@ -689,7 +691,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) · 2 unmodified lines
689#endif691#endif
690 692
691#ifndef atomic_fetch_andnot_relaxed693#ifndef atomic_fetch_andnot_relaxed
692static inline int694static __always_inline int
693atomic_fetch_andnot_relaxed(int i, atomic_t *v)695atomic_fetch_andnot_relaxed(int i, atomic_t *v)
694{696{
695 return atomic_fetch_and_relaxed(~i, v);697 return atomic_fetch_and_relaxed(~i, v);
@@ -700,7 +702,7 @@ atomic_fetch_andnot_relaxed(int i, atomic_t *v) · 4 unmodified lines
700#else /* atomic_fetch_andnot_relaxed */702#else /* atomic_fetch_andnot_relaxed */
701 703
702#ifndef atomic_fetch_andnot_acquire704#ifndef atomic_fetch_andnot_acquire
703static inline int705static __always_inline int
704atomic_fetch_andnot_acquire(int i, atomic_t *v)706atomic_fetch_andnot_acquire(int i, atomic_t *v)
705{707{
706 int ret = atomic_fetch_andnot_relaxed(i, v);708 int ret = atomic_fetch_andnot_relaxed(i, v);
@@ -711,7 +713,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) · 4 unmodified lines
711#endif713#endif
712 714
713#ifndef atomic_fetch_andnot_release715#ifndef atomic_fetch_andnot_release
714static inline int716static __always_inline int
715atomic_fetch_andnot_release(int i, atomic_t *v)717atomic_fetch_andnot_release(int i, atomic_t *v)
716{718{
717 __atomic_release_fence();719 __atomic_release_fence();
@@ -721,7 +723,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) · 3 unmodified lines
721#endif723#endif
722 724
723#ifndef atomic_fetch_andnot725#ifndef atomic_fetch_andnot
724static inline int726static __always_inline int
725atomic_fetch_andnot(int i, atomic_t *v)727atomic_fetch_andnot(int i, atomic_t *v)
726{728{
727 int ret;729 int ret;
@@ -742,7 +744,7 @@ atomic_fetch_andnot(int i, atomic_t *v) · 14 unmodified lines
742#else /* atomic_fetch_or_relaxed */744#else /* atomic_fetch_or_relaxed */
743 745
744#ifndef atomic_fetch_or_acquire746#ifndef atomic_fetch_or_acquire
745static inline int747static __always_inline int
746atomic_fetch_or_acquire(int i, atomic_t *v)748atomic_fetch_or_acquire(int i, atomic_t *v)
747{749{
748 int ret = atomic_fetch_or_relaxed(i, v);750 int ret = atomic_fetch_or_relaxed(i, v);
@@ -753,7 +755,7 @@ atomic_fetch_or_acquire(int i, atomic_t *v) · 4 unmodified lines
753#endif755#endif
754 756
755#ifndef atomic_fetch_or_release757#ifndef atomic_fetch_or_release
756static inline int758static __always_inline int
757atomic_fetch_or_release(int i, atomic_t *v)759atomic_fetch_or_release(int i, atomic_t *v)
758{760{
759 __atomic_release_fence();761 __atomic_release_fence();
@@ -763,7 +765,7 @@ atomic_fetch_or_release(int i, atomic_t *v) · 3 unmodified lines
763#endif765#endif
764 766
765#ifndef atomic_fetch_or767#ifndef atomic_fetch_or
766static inline int768static __always_inline int
767atomic_fetch_or(int i, atomic_t *v)769atomic_fetch_or(int i, atomic_t *v)
768{770{
769 int ret;771 int ret;
@@ -784,7 +786,7 @@ atomic_fetch_or(int i, atomic_t *v) · 14 unmodified lines
784#else /* atomic_fetch_xor_relaxed */786#else /* atomic_fetch_xor_relaxed */
785 787
786#ifndef atomic_fetch_xor_acquire788#ifndef atomic_fetch_xor_acquire
787static inline int789static __always_inline int
788atomic_fetch_xor_acquire(int i, atomic_t *v)790atomic_fetch_xor_acquire(int i, atomic_t *v)
789{791{
790 int ret = atomic_fetch_xor_relaxed(i, v);792 int ret = atomic_fetch_xor_relaxed(i, v);
@@ -795,7 +797,7 @@ atomic_fetch_xor_acquire(int i, atomic_t *v) · 4 unmodified lines
795#endif797#endif
796 798
797#ifndef atomic_fetch_xor_release799#ifndef atomic_fetch_xor_release
798static inline int800static __always_inline int
799atomic_fetch_xor_release(int i, atomic_t *v)801atomic_fetch_xor_release(int i, atomic_t *v)
800{802{
801 __atomic_release_fence();803 __atomic_release_fence();
@@ -805,7 +807,7 @@ atomic_fetch_xor_release(int i, atomic_t *v) · 3 unmodified lines
805#endif807#endif
806 808
807#ifndef atomic_fetch_xor809#ifndef atomic_fetch_xor
808static inline int810static __always_inline int
809atomic_fetch_xor(int i, atomic_t *v)811atomic_fetch_xor(int i, atomic_t *v)
810{812{
811 int ret;813 int ret;
@@ -826,7 +828,7 @@ atomic_fetch_xor(int i, atomic_t *v) · 14 unmodified lines
826#else /* atomic_xchg_relaxed */828#else /* atomic_xchg_relaxed */
827 829
828#ifndef atomic_xchg_acquire830#ifndef atomic_xchg_acquire
829static inline int831static __always_inline int
830atomic_xchg_acquire(atomic_t *v, int i)832atomic_xchg_acquire(atomic_t *v, int i)
831{833{
832 int ret = atomic_xchg_relaxed(v, i);834 int ret = atomic_xchg_relaxed(v, i);
@@ -837,7 +839,7 @@ atomic_xchg_acquire(atomic_t *v, int i) · 4 unmodified lines
837#endif839#endif
838 840
839#ifndef atomic_xchg_release841#ifndef atomic_xchg_release
840static inline int842static __always_inline int
841atomic_xchg_release(atomic_t *v, int i)843atomic_xchg_release(atomic_t *v, int i)
842{844{
843 __atomic_release_fence();845 __atomic_release_fence();
@@ -847,7 +849,7 @@ atomic_xchg_release(atomic_t *v, int i) · 3 unmodified lines
847#endif849#endif
848 850
849#ifndef atomic_xchg851#ifndef atomic_xchg
850static inline int852static __always_inline int
851atomic_xchg(atomic_t *v, int i)853atomic_xchg(atomic_t *v, int i)
852{854{
853 int ret;855 int ret;
@@ -868,7 +870,7 @@ atomic_xchg(atomic_t *v, int i) · 14 unmodified lines
868#else /* atomic_cmpxchg_relaxed */870#else /* atomic_cmpxchg_relaxed */
869 871
870#ifndef atomic_cmpxchg_acquire872#ifndef atomic_cmpxchg_acquire
871static inline int873static __always_inline int
872atomic_cmpxchg_acquire(atomic_t *v, int old, int new)874atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
873{875{
874 int ret = atomic_cmpxchg_relaxed(v, old, new);876 int ret = atomic_cmpxchg_relaxed(v, old, new);
@@ -879,7 +881,7 @@ atomic_cmpxchg_acquire(atomic_t *v, int old, int new) · 4 unmodified lines
879#endif881#endif
880 882
881#ifndef atomic_cmpxchg_release883#ifndef atomic_cmpxchg_release
882static inline int884static __always_inline int
883atomic_cmpxchg_release(atomic_t *v, int old, int new)885atomic_cmpxchg_release(atomic_t *v, int old, int new)
884{886{
885 __atomic_release_fence();887 __atomic_release_fence();
@@ -889,7 +891,7 @@ atomic_cmpxchg_release(atomic_t *v, int old, int new) · 3 unmodified lines
889#endif891#endif
890 892
891#ifndef atomic_cmpxchg893#ifndef atomic_cmpxchg
892static inline int894static __always_inline int
893atomic_cmpxchg(atomic_t *v, int old, int new)895atomic_cmpxchg(atomic_t *v, int old, int new)
894{896{
895 int ret;897 int ret;
@@ -911,7 +913,7 @@ atomic_cmpxchg(atomic_t *v, int old, int new) · 15 unmodified lines
911#endif /* atomic_try_cmpxchg */913#endif /* atomic_try_cmpxchg */
912 914
913#ifndef atomic_try_cmpxchg915#ifndef atomic_try_cmpxchg
914static inline bool916static __always_inline bool
915atomic_try_cmpxchg(atomic_t *v, int *old, int new)917atomic_try_cmpxchg(atomic_t *v, int *old, int new)
916{918{
917 int r, o = *old;919 int r, o = *old;
@@ -924,7 +926,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) · 6 unmodified lines
924#endif926#endif
925 927
926#ifndef atomic_try_cmpxchg_acquire928#ifndef atomic_try_cmpxchg_acquire
927static inline bool929static __always_inline bool
928atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)930atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
929{931{
930 int r, o = *old;932 int r, o = *old;
@@ -937,7 +939,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) · 6 unmodified lines
937#endif939#endif
938 940
939#ifndef atomic_try_cmpxchg_release941#ifndef atomic_try_cmpxchg_release
940static inline bool942static __always_inline bool
941atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)943atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
942{944{
943 int r, o = *old;945 int r, o = *old;
@@ -950,7 +952,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) · 6 unmodified lines
950#endif952#endif
951 953
952#ifndef atomic_try_cmpxchg_relaxed954#ifndef atomic_try_cmpxchg_relaxed
953static inline bool955static __always_inline bool
954atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)956atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
955{957{
956 int r, o = *old;958 int r, o = *old;
@@ -965,7 +967,7 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) · 8 unmodified lines
965#else /* atomic_try_cmpxchg_relaxed */967#else /* atomic_try_cmpxchg_relaxed */
966 968
967#ifndef atomic_try_cmpxchg_acquire969#ifndef atomic_try_cmpxchg_acquire
968static inline bool970static __always_inline bool
969atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)971atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
970{972{
971 bool ret = atomic_try_cmpxchg_relaxed(v, old, new);973 bool ret = atomic_try_cmpxchg_relaxed(v, old, new);
@@ -976,7 +978,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) · 4 unmodified lines
976#endif978#endif
977 979
978#ifndef atomic_try_cmpxchg_release980#ifndef atomic_try_cmpxchg_release
979static inline bool981static __always_inline bool
980atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)982atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
981{983{
982 __atomic_release_fence();984 __atomic_release_fence();
@@ -986,7 +988,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) · 3 unmodified lines
986#endif988#endif
987 989
988#ifndef atomic_try_cmpxchg990#ifndef atomic_try_cmpxchg
989static inline bool991static __always_inline bool
990atomic_try_cmpxchg(atomic_t *v, int *old, int new)992atomic_try_cmpxchg(atomic_t *v, int *old, int new)
991{993{
992 bool ret;994 bool ret;
@@ -1010,7 +1012,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) · 17 unmodified lines
1010 * true if the result is zero, or false for all1012 * true if the result is zero, or false for all
1011 * other cases.1013 * other cases.
1012 */1014 */
1013static inline bool1015static __always_inline bool
1014atomic_sub_and_test(int i, atomic_t *v)1016atomic_sub_and_test(int i, atomic_t *v)
1015{1017{
1016 return atomic_sub_return(i, v) == 0;1018 return atomic_sub_return(i, v) == 0;
@@ -1027,7 +1029,7 @@ atomic_sub_and_test(int i, atomic_t *v) · 10 unmodified lines
1027 * returns true if the result is 0, or false for all other1029 * returns true if the result is 0, or false for all other
1028 * cases.1030 * cases.
1029 */1031 */
1030static inline bool1032static __always_inline bool
1031atomic_dec_and_test(atomic_t *v)1033atomic_dec_and_test(atomic_t *v)
1032{1034{
1033 return atomic_dec_return(v) == 0;1035 return atomic_dec_return(v) == 0;
@@ -1044,7 +1046,7 @@ atomic_dec_and_test(atomic_t *v) · 10 unmodified lines
1044 * and returns true if the result is zero, or false for all1046 * and returns true if the result is zero, or false for all
1045 * other cases.1047 * other cases.
1046 */1048 */
1047static inline bool1049static __always_inline bool
1048atomic_inc_and_test(atomic_t *v)1050atomic_inc_and_test(atomic_t *v)
1049{1051{
1050 return atomic_inc_return(v) == 0;1052 return atomic_inc_return(v) == 0;
@@ -1062,7 +1064,7 @@ atomic_inc_and_test(atomic_t *v) · 11 unmodified lines
1062 * if the result is negative, or false when1064 * if the result is negative, or false when
1063 * result is greater than or equal to zero.1065 * result is greater than or equal to zero.
1064 */1066 */
1065static inline bool1067static __always_inline bool
1066atomic_add_negative(int i, atomic_t *v)1068atomic_add_negative(int i, atomic_t *v)
1067{1069{
1068 return atomic_add_return(i, v) < 0;1070 return atomic_add_return(i, v) < 0;
@@ -1080,7 +1082,7 @@ atomic_add_negative(int i, atomic_t *v) · 11 unmodified lines
1080 * Atomically adds @a to @v, so long as @v was not already @u.1082 * Atomically adds @a to @v, so long as @v was not already @u.
1081 * Returns original value of @v1083 * Returns original value of @v
1082 */1084 */
1083static inline int1085static __always_inline int
1084atomic_fetch_add_unless(atomic_t *v, int a, int u)1086atomic_fetch_add_unless(atomic_t *v, int a, int u)
1085{1087{
1086 int c = atomic_read(v);1088 int c = atomic_read(v);
@@ -1105,7 +1107,7 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) · 18 unmodified lines
1105 * Atomically adds @a to @v, if @v was not already @u.1107 * Atomically adds @a to @v, if @v was not already @u.
1106 * Returns true if the addition was done.1108 * Returns true if the addition was done.
1107 */1109 */
1108static inline bool1110static __always_inline bool
1109atomic_add_unless(atomic_t *v, int a, int u)1111atomic_add_unless(atomic_t *v, int a, int u)
1110{1112{
1111 return atomic_fetch_add_unless(v, a, u) != u;1113 return atomic_fetch_add_unless(v, a, u) != u;
@@ -1121,7 +1123,7 @@ atomic_add_unless(atomic_t *v, int a, int u) · 9 unmodified lines
1121 * Atomically increments @v by 1, if @v is non-zero.1123 * Atomically increments @v by 1, if @v is non-zero.
1122 * Returns true if the increment was done.1124 * Returns true if the increment was done.
1123 */1125 */
1124static inline bool1126static __always_inline bool
1125atomic_inc_not_zero(atomic_t *v)1127atomic_inc_not_zero(atomic_t *v)
1126{1128{
1127 return atomic_add_unless(v, 1, 0);1129 return atomic_add_unless(v, 1, 0);
@@ -1130,7 +1132,7 @@ atomic_inc_not_zero(atomic_t *v) · 2 unmodified lines
1130#endif1132#endif
1131 1133
1132#ifndef atomic_inc_unless_negative1134#ifndef atomic_inc_unless_negative
1133static inline bool1135static __always_inline bool
1134atomic_inc_unless_negative(atomic_t *v)1136atomic_inc_unless_negative(atomic_t *v)
1135{1137{
1136 int c = atomic_read(v);1138 int c = atomic_read(v);
@@ -1146,7 +1148,7 @@ atomic_inc_unless_negative(atomic_t *v) · 9 unmodified lines
1146#endif1148#endif
1147 1149
1148#ifndef atomic_dec_unless_positive1150#ifndef atomic_dec_unless_positive
1149static inline bool1151static __always_inline bool
1150atomic_dec_unless_positive(atomic_t *v)1152atomic_dec_unless_positive(atomic_t *v)
1151{1153{
1152 int c = atomic_read(v);1154 int c = atomic_read(v);
@@ -1162,7 +1164,7 @@ atomic_dec_unless_positive(atomic_t *v) · 9 unmodified lines
1162#endif1164#endif
1163 1165
1164#ifndef atomic_dec_if_positive1166#ifndef atomic_dec_if_positive
1165static inline int1167static __always_inline int
1166atomic_dec_if_positive(atomic_t *v)1168atomic_dec_if_positive(atomic_t *v)
1167{1169{
1168 int dec, c = atomic_read(v);1170 int dec, c = atomic_read(v);
@@ -1178,15 +1180,12 @@ atomic_dec_if_positive(atomic_t *v) · 9 unmodified lines
1178#define atomic_dec_if_positive atomic_dec_if_positive1180#define atomic_dec_if_positive atomic_dec_if_positive
1179#endif1181#endif
1180 1182
1181#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
1182#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
1183
1184#ifdef CONFIG_GENERIC_ATOMIC641183#ifdef CONFIG_GENERIC_ATOMIC64
1185#include <asm-generic/atomic64.h>1184#include <asm-generic/atomic64.h>
1186#endif1185#endif
1187 1186
1188#ifndef atomic64_read_acquire1187#ifndef atomic64_read_acquire
1189static inline s641188static __always_inline s64
1190atomic64_read_acquire(const atomic64_t *v)1189atomic64_read_acquire(const atomic64_t *v)
1191{1190{
1192 return smp_load_acquire(&(v)->counter);1191 return smp_load_acquire(&(v)->counter);
@@ -1195,7 +1194,7 @@ atomic64_read_acquire(const atomic64_t *v) · 2 unmodified lines
1195#endif1194#endif
1196 1195
1197#ifndef atomic64_set_release1196#ifndef atomic64_set_release
1198static inline void1197static __always_inline void
1199atomic64_set_release(atomic64_t *v, s64 i)1198atomic64_set_release(atomic64_t *v, s64 i)
1200{1199{
1201 smp_store_release(&(v)->counter, i);1200 smp_store_release(&(v)->counter, i);
@@ -1210,7 +1209,7 @@ atomic64_set_release(atomic64_t *v, s64 i) · 8 unmodified lines
1210#else /* atomic64_add_return_relaxed */1209#else /* atomic64_add_return_relaxed */
1211 1210
1212#ifndef atomic64_add_return_acquire1211#ifndef atomic64_add_return_acquire
1213static inline s641212static __always_inline s64
1214atomic64_add_return_acquire(s64 i, atomic64_t *v)1213atomic64_add_return_acquire(s64 i, atomic64_t *v)
1215{1214{
1216 s64 ret = atomic64_add_return_relaxed(i, v);1215 s64 ret = atomic64_add_return_relaxed(i, v);
@@ -1221,7 +1220,7 @@ atomic64_add_return_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1221#endif1220#endif
1222 1221
1223#ifndef atomic64_add_return_release1222#ifndef atomic64_add_return_release
1224static inline s641223static __always_inline s64
1225atomic64_add_return_release(s64 i, atomic64_t *v)1224atomic64_add_return_release(s64 i, atomic64_t *v)
1226{1225{
1227 __atomic_release_fence();1226 __atomic_release_fence();
@@ -1231,7 +1230,7 @@ atomic64_add_return_release(s64 i, atomic64_t *v) · 3 unmodified lines
1231#endif1230#endif
1232 1231
1233#ifndef atomic64_add_return1232#ifndef atomic64_add_return
1234static inline s641233static __always_inline s64
1235atomic64_add_return(s64 i, atomic64_t *v)1234atomic64_add_return(s64 i, atomic64_t *v)
1236{1235{
1237 s64 ret;1236 s64 ret;
@@ -1252,7 +1251,7 @@ atomic64_add_return(s64 i, atomic64_t *v) · 14 unmodified lines
1252#else /* atomic64_fetch_add_relaxed */1251#else /* atomic64_fetch_add_relaxed */
1253 1252
1254#ifndef atomic64_fetch_add_acquire1253#ifndef atomic64_fetch_add_acquire
1255static inline s641254static __always_inline s64
1256atomic64_fetch_add_acquire(s64 i, atomic64_t *v)1255atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1257{1256{
1258 s64 ret = atomic64_fetch_add_relaxed(i, v);1257 s64 ret = atomic64_fetch_add_relaxed(i, v);
@@ -1263,7 +1262,7 @@ atomic64_fetch_add_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1263#endif1262#endif
1264 1263
1265#ifndef atomic64_fetch_add_release1264#ifndef atomic64_fetch_add_release
1266static inline s641265static __always_inline s64
1267atomic64_fetch_add_release(s64 i, atomic64_t *v)1266atomic64_fetch_add_release(s64 i, atomic64_t *v)
1268{1267{
1269 __atomic_release_fence();1268 __atomic_release_fence();
@@ -1273,7 +1272,7 @@ atomic64_fetch_add_release(s64 i, atomic64_t *v) · 3 unmodified lines
1273#endif1272#endif
1274 1273
1275#ifndef atomic64_fetch_add1274#ifndef atomic64_fetch_add
1276static inline s641275static __always_inline s64
1277atomic64_fetch_add(s64 i, atomic64_t *v)1276atomic64_fetch_add(s64 i, atomic64_t *v)
1278{1277{
1279 s64 ret;1278 s64 ret;
@@ -1294,7 +1293,7 @@ atomic64_fetch_add(s64 i, atomic64_t *v) · 14 unmodified lines
1294#else /* atomic64_sub_return_relaxed */1293#else /* atomic64_sub_return_relaxed */
1295 1294
1296#ifndef atomic64_sub_return_acquire1295#ifndef atomic64_sub_return_acquire
1297static inline s641296static __always_inline s64
1298atomic64_sub_return_acquire(s64 i, atomic64_t *v)1297atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1299{1298{
1300 s64 ret = atomic64_sub_return_relaxed(i, v);1299 s64 ret = atomic64_sub_return_relaxed(i, v);
@@ -1305,7 +1304,7 @@ atomic64_sub_return_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1305#endif1304#endif
1306 1305
1307#ifndef atomic64_sub_return_release1306#ifndef atomic64_sub_return_release
1308static inline s641307static __always_inline s64
1309atomic64_sub_return_release(s64 i, atomic64_t *v)1308atomic64_sub_return_release(s64 i, atomic64_t *v)
1310{1309{
1311 __atomic_release_fence();1310 __atomic_release_fence();
@@ -1315,7 +1314,7 @@ atomic64_sub_return_release(s64 i, atomic64_t *v) · 3 unmodified lines
1315#endif1314#endif
1316 1315
1317#ifndef atomic64_sub_return1316#ifndef atomic64_sub_return
1318static inline s641317static __always_inline s64
1319atomic64_sub_return(s64 i, atomic64_t *v)1318atomic64_sub_return(s64 i, atomic64_t *v)
1320{1319{
1321 s64 ret;1320 s64 ret;
@@ -1336,7 +1335,7 @@ atomic64_sub_return(s64 i, atomic64_t *v) · 14 unmodified lines
1336#else /* atomic64_fetch_sub_relaxed */1335#else /* atomic64_fetch_sub_relaxed */
1337 1336
1338#ifndef atomic64_fetch_sub_acquire1337#ifndef atomic64_fetch_sub_acquire
1339static inline s641338static __always_inline s64
1340atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)1339atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1341{1340{
1342 s64 ret = atomic64_fetch_sub_relaxed(i, v);1341 s64 ret = atomic64_fetch_sub_relaxed(i, v);
@@ -1347,7 +1346,7 @@ atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1347#endif1346#endif
1348 1347
1349#ifndef atomic64_fetch_sub_release1348#ifndef atomic64_fetch_sub_release
1350static inline s641349static __always_inline s64
1351atomic64_fetch_sub_release(s64 i, atomic64_t *v)1350atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1352{1351{
1353 __atomic_release_fence();1352 __atomic_release_fence();
@@ -1357,7 +1356,7 @@ atomic64_fetch_sub_release(s64 i, atomic64_t *v) · 3 unmodified lines
1357#endif1356#endif
1358 1357
1359#ifndef atomic64_fetch_sub1358#ifndef atomic64_fetch_sub
1360static inline s641359static __always_inline s64
1361atomic64_fetch_sub(s64 i, atomic64_t *v)1360atomic64_fetch_sub(s64 i, atomic64_t *v)
1362{1361{
1363 s64 ret;1362 s64 ret;
@@ -1372,7 +1371,7 @@ atomic64_fetch_sub(s64 i, atomic64_t *v) · 8 unmodified lines
1372#endif /* atomic64_fetch_sub_relaxed */1371#endif /* atomic64_fetch_sub_relaxed */
1373 1372
1374#ifndef atomic64_inc1373#ifndef atomic64_inc
1375static inline void1374static __always_inline void
1376atomic64_inc(atomic64_t *v)1375atomic64_inc(atomic64_t *v)
1377{1376{
1378 atomic64_add(1, v);1377 atomic64_add(1, v);
@@ -1388,7 +1387,7 @@ atomic64_inc(atomic64_t *v) · 9 unmodified lines
1388#endif /* atomic64_inc_return */1387#endif /* atomic64_inc_return */
1389 1388
1390#ifndef atomic64_inc_return1389#ifndef atomic64_inc_return
1391static inline s641390static __always_inline s64
1392atomic64_inc_return(atomic64_t *v)1391atomic64_inc_return(atomic64_t *v)
1393{1392{
1394 return atomic64_add_return(1, v);1393 return atomic64_add_return(1, v);
@@ -1397,7 +1396,7 @@ atomic64_inc_return(atomic64_t *v) · 2 unmodified lines
1397#endif1396#endif
1398 1397
1399#ifndef atomic64_inc_return_acquire1398#ifndef atomic64_inc_return_acquire
1400static inline s641399static __always_inline s64
1401atomic64_inc_return_acquire(atomic64_t *v)1400atomic64_inc_return_acquire(atomic64_t *v)
1402{1401{
1403 return atomic64_add_return_acquire(1, v);1402 return atomic64_add_return_acquire(1, v);
@@ -1406,7 +1405,7 @@ atomic64_inc_return_acquire(atomic64_t *v) · 2 unmodified lines
1406#endif1405#endif
1407 1406
1408#ifndef atomic64_inc_return_release1407#ifndef atomic64_inc_return_release
1409static inline s641408static __always_inline s64
1410atomic64_inc_return_release(atomic64_t *v)1409atomic64_inc_return_release(atomic64_t *v)
1411{1410{
1412 return atomic64_add_return_release(1, v);1411 return atomic64_add_return_release(1, v);
@@ -1415,7 +1414,7 @@ atomic64_inc_return_release(atomic64_t *v) · 2 unmodified lines
1415#endif1414#endif
1416 1415
1417#ifndef atomic64_inc_return_relaxed1416#ifndef atomic64_inc_return_relaxed
1418static inline s641417static __always_inline s64
1419atomic64_inc_return_relaxed(atomic64_t *v)1418atomic64_inc_return_relaxed(atomic64_t *v)
1420{1419{
1421 return atomic64_add_return_relaxed(1, v);1420 return atomic64_add_return_relaxed(1, v);
@@ -1426,7 +1425,7 @@ atomic64_inc_return_relaxed(atomic64_t *v) · 4 unmodified lines
1426#else /* atomic64_inc_return_relaxed */1425#else /* atomic64_inc_return_relaxed */
1427 1426
1428#ifndef atomic64_inc_return_acquire1427#ifndef atomic64_inc_return_acquire
1429static inline s641428static __always_inline s64
1430atomic64_inc_return_acquire(atomic64_t *v)1429atomic64_inc_return_acquire(atomic64_t *v)
1431{1430{
1432 s64 ret = atomic64_inc_return_relaxed(v);1431 s64 ret = atomic64_inc_return_relaxed(v);
@@ -1437,7 +1436,7 @@ atomic64_inc_return_acquire(atomic64_t *v) · 4 unmodified lines
1437#endif1436#endif
1438 1437
1439#ifndef atomic64_inc_return_release1438#ifndef atomic64_inc_return_release
1440static inline s641439static __always_inline s64
1441atomic64_inc_return_release(atomic64_t *v)1440atomic64_inc_return_release(atomic64_t *v)
1442{1441{
1443 __atomic_release_fence();1442 __atomic_release_fence();
@@ -1447,7 +1446,7 @@ atomic64_inc_return_release(atomic64_t *v) · 3 unmodified lines
1447#endif1446#endif
1448 1447
1449#ifndef atomic64_inc_return1448#ifndef atomic64_inc_return
1450static inline s641449static __always_inline s64
1451atomic64_inc_return(atomic64_t *v)1450atomic64_inc_return(atomic64_t *v)
1452{1451{
1453 s64 ret;1452 s64 ret;
@@ -1469,7 +1468,7 @@ atomic64_inc_return(atomic64_t *v) · 15 unmodified lines
1469#endif /* atomic64_fetch_inc */1468#endif /* atomic64_fetch_inc */
1470 1469
1471#ifndef atomic64_fetch_inc1470#ifndef atomic64_fetch_inc
1472static inline s641471static __always_inline s64
1473atomic64_fetch_inc(atomic64_t *v)1472atomic64_fetch_inc(atomic64_t *v)
1474{1473{
1475 return atomic64_fetch_add(1, v);1474 return atomic64_fetch_add(1, v);
@@ -1478,7 +1477,7 @@ atomic64_fetch_inc(atomic64_t *v) · 2 unmodified lines
1478#endif1477#endif
1479 1478
1480#ifndef atomic64_fetch_inc_acquire1479#ifndef atomic64_fetch_inc_acquire
1481static inline s641480static __always_inline s64
1482atomic64_fetch_inc_acquire(atomic64_t *v)1481atomic64_fetch_inc_acquire(atomic64_t *v)
1483{1482{
1484 return atomic64_fetch_add_acquire(1, v);1483 return atomic64_fetch_add_acquire(1, v);
@@ -1487,7 +1486,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) · 2 unmodified lines
1487#endif1486#endif
1488 1487
1489#ifndef atomic64_fetch_inc_release1488#ifndef atomic64_fetch_inc_release
1490static inline s641489static __always_inline s64
1491atomic64_fetch_inc_release(atomic64_t *v)1490atomic64_fetch_inc_release(atomic64_t *v)
1492{1491{
1493 return atomic64_fetch_add_release(1, v);1492 return atomic64_fetch_add_release(1, v);
@@ -1496,7 +1495,7 @@ atomic64_fetch_inc_release(atomic64_t *v) · 2 unmodified lines
1496#endif1495#endif
1497 1496
1498#ifndef atomic64_fetch_inc_relaxed1497#ifndef atomic64_fetch_inc_relaxed
1499static inline s641498static __always_inline s64
1500atomic64_fetch_inc_relaxed(atomic64_t *v)1499atomic64_fetch_inc_relaxed(atomic64_t *v)
1501{1500{
1502 return atomic64_fetch_add_relaxed(1, v);1501 return atomic64_fetch_add_relaxed(1, v);
@@ -1507,7 +1506,7 @@ atomic64_fetch_inc_relaxed(atomic64_t *v) · 4 unmodified lines
1507#else /* atomic64_fetch_inc_relaxed */1506#else /* atomic64_fetch_inc_relaxed */
1508 1507
1509#ifndef atomic64_fetch_inc_acquire1508#ifndef atomic64_fetch_inc_acquire
1510static inline s641509static __always_inline s64
1511atomic64_fetch_inc_acquire(atomic64_t *v)1510atomic64_fetch_inc_acquire(atomic64_t *v)
1512{1511{
1513 s64 ret = atomic64_fetch_inc_relaxed(v);1512 s64 ret = atomic64_fetch_inc_relaxed(v);
@@ -1518,7 +1517,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) · 4 unmodified lines
1518#endif1517#endif
1519 1518
1520#ifndef atomic64_fetch_inc_release1519#ifndef atomic64_fetch_inc_release
1521static inline s641520static __always_inline s64
1522atomic64_fetch_inc_release(atomic64_t *v)1521atomic64_fetch_inc_release(atomic64_t *v)
1523{1522{
1524 __atomic_release_fence();1523 __atomic_release_fence();
@@ -1528,7 +1527,7 @@ atomic64_fetch_inc_release(atomic64_t *v) · 3 unmodified lines
1528#endif1527#endif
1529 1528
1530#ifndef atomic64_fetch_inc1529#ifndef atomic64_fetch_inc
1531static inline s641530static __always_inline s64
1532atomic64_fetch_inc(atomic64_t *v)1531atomic64_fetch_inc(atomic64_t *v)
1533{1532{
1534 s64 ret;1533 s64 ret;
@@ -1543,7 +1542,7 @@ atomic64_fetch_inc(atomic64_t *v) · 8 unmodified lines
1543#endif /* atomic64_fetch_inc_relaxed */1542#endif /* atomic64_fetch_inc_relaxed */
1544 1543
1545#ifndef atomic64_dec1544#ifndef atomic64_dec
1546static inline void1545static __always_inline void
1547atomic64_dec(atomic64_t *v)1546atomic64_dec(atomic64_t *v)
1548{1547{
1549 atomic64_sub(1, v);1548 atomic64_sub(1, v);
@@ -1559,7 +1558,7 @@ atomic64_dec(atomic64_t *v) · 9 unmodified lines
1559#endif /* atomic64_dec_return */1558#endif /* atomic64_dec_return */
1560 1559
1561#ifndef atomic64_dec_return1560#ifndef atomic64_dec_return
1562static inline s641561static __always_inline s64
1563atomic64_dec_return(atomic64_t *v)1562atomic64_dec_return(atomic64_t *v)
1564{1563{
1565 return atomic64_sub_return(1, v);1564 return atomic64_sub_return(1, v);
@@ -1568,7 +1567,7 @@ atomic64_dec_return(atomic64_t *v) · 2 unmodified lines
1568#endif1567#endif
1569 1568
1570#ifndef atomic64_dec_return_acquire1569#ifndef atomic64_dec_return_acquire
1571static inline s641570static __always_inline s64
1572atomic64_dec_return_acquire(atomic64_t *v)1571atomic64_dec_return_acquire(atomic64_t *v)
1573{1572{
1574 return atomic64_sub_return_acquire(1, v);1573 return atomic64_sub_return_acquire(1, v);
@@ -1577,7 +1576,7 @@ atomic64_dec_return_acquire(atomic64_t *v) · 2 unmodified lines
1577#endif1576#endif
1578 1577
1579#ifndef atomic64_dec_return_release1578#ifndef atomic64_dec_return_release
1580static inline s641579static __always_inline s64
1581atomic64_dec_return_release(atomic64_t *v)1580atomic64_dec_return_release(atomic64_t *v)
1582{1581{
1583 return atomic64_sub_return_release(1, v);1582 return atomic64_sub_return_release(1, v);
@@ -1586,7 +1585,7 @@ atomic64_dec_return_release(atomic64_t *v) · 2 unmodified lines
1586#endif1585#endif
1587 1586
1588#ifndef atomic64_dec_return_relaxed1587#ifndef atomic64_dec_return_relaxed
1589static inline s641588static __always_inline s64
1590atomic64_dec_return_relaxed(atomic64_t *v)1589atomic64_dec_return_relaxed(atomic64_t *v)
1591{1590{
1592 return atomic64_sub_return_relaxed(1, v);1591 return atomic64_sub_return_relaxed(1, v);
@@ -1597,7 +1596,7 @@ atomic64_dec_return_relaxed(atomic64_t *v) · 4 unmodified lines
1597#else /* atomic64_dec_return_relaxed */1596#else /* atomic64_dec_return_relaxed */
1598 1597
1599#ifndef atomic64_dec_return_acquire1598#ifndef atomic64_dec_return_acquire
1600static inline s641599static __always_inline s64
1601atomic64_dec_return_acquire(atomic64_t *v)1600atomic64_dec_return_acquire(atomic64_t *v)
1602{1601{
1603 s64 ret = atomic64_dec_return_relaxed(v);1602 s64 ret = atomic64_dec_return_relaxed(v);
@@ -1608,7 +1607,7 @@ atomic64_dec_return_acquire(atomic64_t *v) · 4 unmodified lines
1608#endif1607#endif
1609 1608
1610#ifndef atomic64_dec_return_release1609#ifndef atomic64_dec_return_release
1611static inline s641610static __always_inline s64
1612atomic64_dec_return_release(atomic64_t *v)1611atomic64_dec_return_release(atomic64_t *v)
1613{1612{
1614 __atomic_release_fence();1613 __atomic_release_fence();
@@ -1618,7 +1617,7 @@ atomic64_dec_return_release(atomic64_t *v) · 3 unmodified lines
1618#endif1617#endif
1619 1618
1620#ifndef atomic64_dec_return1619#ifndef atomic64_dec_return
1621static inline s641620static __always_inline s64
1622atomic64_dec_return(atomic64_t *v)1621atomic64_dec_return(atomic64_t *v)
1623{1622{
1624 s64 ret;1623 s64 ret;
@@ -1640,7 +1639,7 @@ atomic64_dec_return(atomic64_t *v) · 15 unmodified lines
1640#endif /* atomic64_fetch_dec */1639#endif /* atomic64_fetch_dec */
1641 1640
1642#ifndef atomic64_fetch_dec1641#ifndef atomic64_fetch_dec
1643static inline s641642static __always_inline s64
1644atomic64_fetch_dec(atomic64_t *v)1643atomic64_fetch_dec(atomic64_t *v)
1645{1644{
1646 return atomic64_fetch_sub(1, v);1645 return atomic64_fetch_sub(1, v);
@@ -1649,7 +1648,7 @@ atomic64_fetch_dec(atomic64_t *v) · 2 unmodified lines
1649#endif1648#endif
1650 1649
1651#ifndef atomic64_fetch_dec_acquire1650#ifndef atomic64_fetch_dec_acquire
1652static inline s641651static __always_inline s64
1653atomic64_fetch_dec_acquire(atomic64_t *v)1652atomic64_fetch_dec_acquire(atomic64_t *v)
1654{1653{
1655 return atomic64_fetch_sub_acquire(1, v);1654 return atomic64_fetch_sub_acquire(1, v);
@@ -1658,7 +1657,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) · 2 unmodified lines
1658#endif1657#endif
1659 1658
1660#ifndef atomic64_fetch_dec_release1659#ifndef atomic64_fetch_dec_release
1661static inline s641660static __always_inline s64
1662atomic64_fetch_dec_release(atomic64_t *v)1661atomic64_fetch_dec_release(atomic64_t *v)
1663{1662{
1664 return atomic64_fetch_sub_release(1, v);1663 return atomic64_fetch_sub_release(1, v);
@@ -1667,7 +1666,7 @@ atomic64_fetch_dec_release(atomic64_t *v) · 2 unmodified lines
1667#endif1666#endif
1668 1667
1669#ifndef atomic64_fetch_dec_relaxed1668#ifndef atomic64_fetch_dec_relaxed
1670static inline s641669static __always_inline s64
1671atomic64_fetch_dec_relaxed(atomic64_t *v)1670atomic64_fetch_dec_relaxed(atomic64_t *v)
1672{1671{
1673 return atomic64_fetch_sub_relaxed(1, v);1672 return atomic64_fetch_sub_relaxed(1, v);
@@ -1678,7 +1677,7 @@ atomic64_fetch_dec_relaxed(atomic64_t *v) · 4 unmodified lines
1678#else /* atomic64_fetch_dec_relaxed */1677#else /* atomic64_fetch_dec_relaxed */
1679 1678
1680#ifndef atomic64_fetch_dec_acquire1679#ifndef atomic64_fetch_dec_acquire
1681static inline s641680static __always_inline s64
1682atomic64_fetch_dec_acquire(atomic64_t *v)1681atomic64_fetch_dec_acquire(atomic64_t *v)
1683{1682{
1684 s64 ret = atomic64_fetch_dec_relaxed(v);1683 s64 ret = atomic64_fetch_dec_relaxed(v);
@@ -1689,7 +1688,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) · 4 unmodified lines
1689#endif1688#endif
1690 1689
1691#ifndef atomic64_fetch_dec_release1690#ifndef atomic64_fetch_dec_release
1692static inline s641691static __always_inline s64
1693atomic64_fetch_dec_release(atomic64_t *v)1692atomic64_fetch_dec_release(atomic64_t *v)
1694{1693{
1695 __atomic_release_fence();1694 __atomic_release_fence();
@@ -1699,7 +1698,7 @@ atomic64_fetch_dec_release(atomic64_t *v) · 3 unmodified lines
1699#endif1698#endif
1700 1699
1701#ifndef atomic64_fetch_dec1700#ifndef atomic64_fetch_dec
1702static inline s641701static __always_inline s64
1703atomic64_fetch_dec(atomic64_t *v)1702atomic64_fetch_dec(atomic64_t *v)
1704{1703{
1705 s64 ret;1704 s64 ret;
@@ -1720,7 +1719,7 @@ atomic64_fetch_dec(atomic64_t *v) · 14 unmodified lines
1720#else /* atomic64_fetch_and_relaxed */1719#else /* atomic64_fetch_and_relaxed */
1721 1720
1722#ifndef atomic64_fetch_and_acquire1721#ifndef atomic64_fetch_and_acquire
1723static inline s641722static __always_inline s64
1724atomic64_fetch_and_acquire(s64 i, atomic64_t *v)1723atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1725{1724{
1726 s64 ret = atomic64_fetch_and_relaxed(i, v);1725 s64 ret = atomic64_fetch_and_relaxed(i, v);
@@ -1731,7 +1730,7 @@ atomic64_fetch_and_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1731#endif1730#endif
1732 1731
1733#ifndef atomic64_fetch_and_release1732#ifndef atomic64_fetch_and_release
1734static inline s641733static __always_inline s64
1735atomic64_fetch_and_release(s64 i, atomic64_t *v)1734atomic64_fetch_and_release(s64 i, atomic64_t *v)
1736{1735{
1737 __atomic_release_fence();1736 __atomic_release_fence();
@@ -1741,7 +1740,7 @@ atomic64_fetch_and_release(s64 i, atomic64_t *v) · 3 unmodified lines
1741#endif1740#endif
1742 1741
1743#ifndef atomic64_fetch_and1742#ifndef atomic64_fetch_and
1744static inline s641743static __always_inline s64
1745atomic64_fetch_and(s64 i, atomic64_t *v)1744atomic64_fetch_and(s64 i, atomic64_t *v)
1746{1745{
1747 s64 ret;1746 s64 ret;
@@ -1756,7 +1755,7 @@ atomic64_fetch_and(s64 i, atomic64_t *v) · 8 unmodified lines
1756#endif /* atomic64_fetch_and_relaxed */1755#endif /* atomic64_fetch_and_relaxed */
1757 1756
1758#ifndef atomic64_andnot1757#ifndef atomic64_andnot
1759static inline void1758static __always_inline void
1760atomic64_andnot(s64 i, atomic64_t *v)1759atomic64_andnot(s64 i, atomic64_t *v)
1761{1760{
1762 atomic64_and(~i, v);1761 atomic64_and(~i, v);
@@ -1772,7 +1771,7 @@ atomic64_andnot(s64 i, atomic64_t *v) · 9 unmodified lines
1772#endif /* atomic64_fetch_andnot */1771#endif /* atomic64_fetch_andnot */
1773 1772
1774#ifndef atomic64_fetch_andnot1773#ifndef atomic64_fetch_andnot
1775static inline s641774static __always_inline s64
1776atomic64_fetch_andnot(s64 i, atomic64_t *v)1775atomic64_fetch_andnot(s64 i, atomic64_t *v)
1777{1776{
1778 return atomic64_fetch_and(~i, v);1777 return atomic64_fetch_and(~i, v);
@@ -1781,7 +1780,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) · 2 unmodified lines
1781#endif1780#endif
1782 1781
1783#ifndef atomic64_fetch_andnot_acquire1782#ifndef atomic64_fetch_andnot_acquire
1784static inline s641783static __always_inline s64
1785atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)1784atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1786{1785{
1787 return atomic64_fetch_and_acquire(~i, v);1786 return atomic64_fetch_and_acquire(~i, v);
@@ -1790,7 +1789,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) · 2 unmodified lines
1790#endif1789#endif
1791 1790
1792#ifndef atomic64_fetch_andnot_release1791#ifndef atomic64_fetch_andnot_release
1793static inline s641792static __always_inline s64
1794atomic64_fetch_andnot_release(s64 i, atomic64_t *v)1793atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1795{1794{
1796 return atomic64_fetch_and_release(~i, v);1795 return atomic64_fetch_and_release(~i, v);
@@ -1799,7 +1798,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) · 2 unmodified lines
1799#endif1798#endif
1800 1799
1801#ifndef atomic64_fetch_andnot_relaxed1800#ifndef atomic64_fetch_andnot_relaxed
1802static inline s641801static __always_inline s64
1803atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)1802atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1804{1803{
1805 return atomic64_fetch_and_relaxed(~i, v);1804 return atomic64_fetch_and_relaxed(~i, v);
@@ -1810,7 +1809,7 @@ atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) · 4 unmodified lines
1810#else /* atomic64_fetch_andnot_relaxed */1809#else /* atomic64_fetch_andnot_relaxed */
1811 1810
1812#ifndef atomic64_fetch_andnot_acquire1811#ifndef atomic64_fetch_andnot_acquire
1813static inline s641812static __always_inline s64
1814atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)1813atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1815{1814{
1816 s64 ret = atomic64_fetch_andnot_relaxed(i, v);1815 s64 ret = atomic64_fetch_andnot_relaxed(i, v);
@@ -1821,7 +1820,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1821#endif1820#endif
1822 1821
1823#ifndef atomic64_fetch_andnot_release1822#ifndef atomic64_fetch_andnot_release
1824static inline s641823static __always_inline s64
1825atomic64_fetch_andnot_release(s64 i, atomic64_t *v)1824atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1826{1825{
1827 __atomic_release_fence();1826 __atomic_release_fence();
@@ -1831,7 +1830,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) · 3 unmodified lines
1831#endif1830#endif
1832 1831
1833#ifndef atomic64_fetch_andnot1832#ifndef atomic64_fetch_andnot
1834static inline s641833static __always_inline s64
1835atomic64_fetch_andnot(s64 i, atomic64_t *v)1834atomic64_fetch_andnot(s64 i, atomic64_t *v)
1836{1835{
1837 s64 ret;1836 s64 ret;
@@ -1852,7 +1851,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) · 14 unmodified lines
1852#else /* atomic64_fetch_or_relaxed */1851#else /* atomic64_fetch_or_relaxed */
1853 1852
1854#ifndef atomic64_fetch_or_acquire1853#ifndef atomic64_fetch_or_acquire
1855static inline s641854static __always_inline s64
1856atomic64_fetch_or_acquire(s64 i, atomic64_t *v)1855atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1857{1856{
1858 s64 ret = atomic64_fetch_or_relaxed(i, v);1857 s64 ret = atomic64_fetch_or_relaxed(i, v);
@@ -1863,7 +1862,7 @@ atomic64_fetch_or_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1863#endif1862#endif
1864 1863
1865#ifndef atomic64_fetch_or_release1864#ifndef atomic64_fetch_or_release
1866static inline s641865static __always_inline s64
1867atomic64_fetch_or_release(s64 i, atomic64_t *v)1866atomic64_fetch_or_release(s64 i, atomic64_t *v)
1868{1867{
1869 __atomic_release_fence();1868 __atomic_release_fence();
@@ -1873,7 +1872,7 @@ atomic64_fetch_or_release(s64 i, atomic64_t *v) · 3 unmodified lines
1873#endif1872#endif
1874 1873
1875#ifndef atomic64_fetch_or1874#ifndef atomic64_fetch_or
1876static inline s641875static __always_inline s64
1877atomic64_fetch_or(s64 i, atomic64_t *v)1876atomic64_fetch_or(s64 i, atomic64_t *v)
1878{1877{
1879 s64 ret;1878 s64 ret;
@@ -1894,7 +1893,7 @@ atomic64_fetch_or(s64 i, atomic64_t *v) · 14 unmodified lines
1894#else /* atomic64_fetch_xor_relaxed */1893#else /* atomic64_fetch_xor_relaxed */
1895 1894
1896#ifndef atomic64_fetch_xor_acquire1895#ifndef atomic64_fetch_xor_acquire
1897static inline s641896static __always_inline s64
1898atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)1897atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1899{1898{
1900 s64 ret = atomic64_fetch_xor_relaxed(i, v);1899 s64 ret = atomic64_fetch_xor_relaxed(i, v);
@@ -1905,7 +1904,7 @@ atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) · 4 unmodified lines
1905#endif1904#endif
1906 1905
1907#ifndef atomic64_fetch_xor_release1906#ifndef atomic64_fetch_xor_release
1908static inline s641907static __always_inline s64
1909atomic64_fetch_xor_release(s64 i, atomic64_t *v)1908atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1910{1909{
1911 __atomic_release_fence();1910 __atomic_release_fence();
@@ -1915,7 +1914,7 @@ atomic64_fetch_xor_release(s64 i, atomic64_t *v) · 3 unmodified lines
1915#endif1914#endif
1916 1915
1917#ifndef atomic64_fetch_xor1916#ifndef atomic64_fetch_xor
1918static inline s641917static __always_inline s64
1919atomic64_fetch_xor(s64 i, atomic64_t *v)1918atomic64_fetch_xor(s64 i, atomic64_t *v)
1920{1919{
1921 s64 ret;1920 s64 ret;
@@ -1936,7 +1935,7 @@ atomic64_fetch_xor(s64 i, atomic64_t *v) · 14 unmodified lines
1936#else /* atomic64_xchg_relaxed */1935#else /* atomic64_xchg_relaxed */
1937 1936
1938#ifndef atomic64_xchg_acquire1937#ifndef atomic64_xchg_acquire
1939static inline s641938static __always_inline s64
1940atomic64_xchg_acquire(atomic64_t *v, s64 i)1939atomic64_xchg_acquire(atomic64_t *v, s64 i)
1941{1940{
1942 s64 ret = atomic64_xchg_relaxed(v, i);1941 s64 ret = atomic64_xchg_relaxed(v, i);
@@ -1947,7 +1946,7 @@ atomic64_xchg_acquire(atomic64_t *v, s64 i) · 4 unmodified lines
1947#endif1946#endif
1948 1947
1949#ifndef atomic64_xchg_release1948#ifndef atomic64_xchg_release
1950static inline s641949static __always_inline s64
1951atomic64_xchg_release(atomic64_t *v, s64 i)1950atomic64_xchg_release(atomic64_t *v, s64 i)
1952{1951{
1953 __atomic_release_fence();1952 __atomic_release_fence();
@@ -1957,7 +1956,7 @@ atomic64_xchg_release(atomic64_t *v, s64 i) · 3 unmodified lines
1957#endif1956#endif
1958 1957
1959#ifndef atomic64_xchg1958#ifndef atomic64_xchg
1960static inline s641959static __always_inline s64
1961atomic64_xchg(atomic64_t *v, s64 i)1960atomic64_xchg(atomic64_t *v, s64 i)
1962{1961{
1963 s64 ret;1962 s64 ret;
@@ -1978,7 +1977,7 @@ atomic64_xchg(atomic64_t *v, s64 i) · 14 unmodified lines
1978#else /* atomic64_cmpxchg_relaxed */1977#else /* atomic64_cmpxchg_relaxed */
1979 1978
1980#ifndef atomic64_cmpxchg_acquire1979#ifndef atomic64_cmpxchg_acquire
1981static inline s641980static __always_inline s64
1982atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)1981atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1983{1982{
1984 s64 ret = atomic64_cmpxchg_relaxed(v, old, new);1983 s64 ret = atomic64_cmpxchg_relaxed(v, old, new);
@@ -1989,7 +1988,7 @@ atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) · 4 unmodified lines
1989#endif1988#endif
1990 1989
1991#ifndef atomic64_cmpxchg_release1990#ifndef atomic64_cmpxchg_release
1992static inline s641991static __always_inline s64
1993atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)1992atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1994{1993{
1995 __atomic_release_fence();1994 __atomic_release_fence();
@@ -1999,7 +1998,7 @@ atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) · 3 unmodified lines
1999#endif1998#endif
2000 1999
2001#ifndef atomic64_cmpxchg2000#ifndef atomic64_cmpxchg
2002static inline s642001static __always_inline s64
2003atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)2002atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2004{2003{
2005 s64 ret;2004 s64 ret;
@@ -2021,7 +2020,7 @@ atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) · 15 unmodified lines
2021#endif /* atomic64_try_cmpxchg */2020#endif /* atomic64_try_cmpxchg */
2022 2021
2023#ifndef atomic64_try_cmpxchg2022#ifndef atomic64_try_cmpxchg
2024static inline bool2023static __always_inline bool
2025atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)2024atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2026{2025{
2027 s64 r, o = *old;2026 s64 r, o = *old;
@@ -2034,7 +2033,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
2034#endif2033#endif
2035 2034
2036#ifndef atomic64_try_cmpxchg_acquire2035#ifndef atomic64_try_cmpxchg_acquire
2037static inline bool2036static __always_inline bool
2038atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)2037atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2039{2038{
2040 s64 r, o = *old;2039 s64 r, o = *old;
@@ -2047,7 +2046,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
2047#endif2046#endif
2048 2047
2049#ifndef atomic64_try_cmpxchg_release2048#ifndef atomic64_try_cmpxchg_release
2050static inline bool2049static __always_inline bool
2051atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)2050atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2052{2051{
2053 s64 r, o = *old;2052 s64 r, o = *old;
@@ -2060,7 +2059,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) · 6 unmodified lines
2060#endif2059#endif
2061 2060
2062#ifndef atomic64_try_cmpxchg_relaxed2061#ifndef atomic64_try_cmpxchg_relaxed
2063static inline bool2062static __always_inline bool
2064atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)2063atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2065{2064{
2066 s64 r, o = *old;2065 s64 r, o = *old;
@@ -2075,7 +2074,7 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) · 8 unmodified lines
2075#else /* atomic64_try_cmpxchg_relaxed */2074#else /* atomic64_try_cmpxchg_relaxed */
2076 2075
2077#ifndef atomic64_try_cmpxchg_acquire2076#ifndef atomic64_try_cmpxchg_acquire
2078static inline bool2077static __always_inline bool
2079atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)2078atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2080{2079{
2081 bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);2080 bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);
@@ -2086,7 +2085,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) · 4 unmodified lines
2086#endif2085#endif
2087 2086
2088#ifndef atomic64_try_cmpxchg_release2087#ifndef atomic64_try_cmpxchg_release
2089static inline bool2088static __always_inline bool
2090atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)2089atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2091{2090{
2092 __atomic_release_fence();2091 __atomic_release_fence();
@@ -2096,7 +2095,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) · 3 unmodified lines
2096#endif2095#endif
2097 2096
2098#ifndef atomic64_try_cmpxchg2097#ifndef atomic64_try_cmpxchg
2099static inline bool2098static __always_inline bool
2100atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)2099atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2101{2100{
2102 bool ret;2101 bool ret;
@@ -2120,7 +2119,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) · 17 unmodified lines
2120 * true if the result is zero, or false for all2119 * true if the result is zero, or false for all
2121 * other cases.2120 * other cases.
2122 */2121 */
2123static inline bool2122static __always_inline bool
2124atomic64_sub_and_test(s64 i, atomic64_t *v)2123atomic64_sub_and_test(s64 i, atomic64_t *v)
2125{2124{
2126 return atomic64_sub_return(i, v) == 0;2125 return atomic64_sub_return(i, v) == 0;
@@ -2137,7 +2136,7 @@ atomic64_sub_and_test(s64 i, atomic64_t *v) · 10 unmodified lines
2137 * returns true if the result is 0, or false for all other2136 * returns true if the result is 0, or false for all other
2138 * cases.2137 * cases.
2139 */2138 */
2140static inline bool2139static __always_inline bool
2141atomic64_dec_and_test(atomic64_t *v)2140atomic64_dec_and_test(atomic64_t *v)
2142{2141{
2143 return atomic64_dec_return(v) == 0;2142 return atomic64_dec_return(v) == 0;
@@ -2154,7 +2153,7 @@ atomic64_dec_and_test(atomic64_t *v) · 10 unmodified lines
2154 * and returns true if the result is zero, or false for all2153 * and returns true if the result is zero, or false for all
2155 * other cases.2154 * other cases.
2156 */2155 */
2157static inline bool2156static __always_inline bool
2158atomic64_inc_and_test(atomic64_t *v)2157atomic64_inc_and_test(atomic64_t *v)
2159{2158{
2160 return atomic64_inc_return(v) == 0;2159 return atomic64_inc_return(v) == 0;
@@ -2172,7 +2171,7 @@ atomic64_inc_and_test(atomic64_t *v) · 11 unmodified lines
2172 * if the result is negative, or false when2171 * if the result is negative, or false when
2173 * result is greater than or equal to zero.2172 * result is greater than or equal to zero.
2174 */2173 */
2175static inline bool2174static __always_inline bool
2176atomic64_add_negative(s64 i, atomic64_t *v)2175atomic64_add_negative(s64 i, atomic64_t *v)
2177{2176{
2178 return atomic64_add_return(i, v) < 0;2177 return atomic64_add_return(i, v) < 0;
@@ -2190,7 +2189,7 @@ atomic64_add_negative(s64 i, atomic64_t *v) · 11 unmodified lines
2190 * Atomically adds @a to @v, so long as @v was not already @u.2189 * Atomically adds @a to @v, so long as @v was not already @u.
2191 * Returns original value of @v2190 * Returns original value of @v
2192 */2191 */
2193static inline s642192static __always_inline s64
2194atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)2193atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2195{2194{
2196 s64 c = atomic64_read(v);2195 s64 c = atomic64_read(v);
@@ -2215,7 +2214,7 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) · 18 unmodified lines
2215 * Atomically adds @a to @v, if @v was not already @u.2214 * Atomically adds @a to @v, if @v was not already @u.
2216 * Returns true if the addition was done.2215 * Returns true if the addition was done.
2217 */2216 */
2218static inline bool2217static __always_inline bool
2219atomic64_add_unless(atomic64_t *v, s64 a, s64 u)2218atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2220{2219{
2221 return atomic64_fetch_add_unless(v, a, u) != u;2220 return atomic64_fetch_add_unless(v, a, u) != u;
@@ -2231,7 +2230,7 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) · 9 unmodified lines
2231 * Atomically increments @v by 1, if @v is non-zero.2230 * Atomically increments @v by 1, if @v is non-zero.
2232 * Returns true if the increment was done.2231 * Returns true if the increment was done.
2233 */2232 */
2234static inline bool2233static __always_inline bool
2235atomic64_inc_not_zero(atomic64_t *v)2234atomic64_inc_not_zero(atomic64_t *v)
2236{2235{
2237 return atomic64_add_unless(v, 1, 0);2236 return atomic64_add_unless(v, 1, 0);
@@ -2240,7 +2239,7 @@ atomic64_inc_not_zero(atomic64_t *v) · 2 unmodified lines
2240#endif2239#endif
2241 2240
2242#ifndef atomic64_inc_unless_negative2241#ifndef atomic64_inc_unless_negative
2243static inline bool2242static __always_inline bool
2244atomic64_inc_unless_negative(atomic64_t *v)2243atomic64_inc_unless_negative(atomic64_t *v)
2245{2244{
2246 s64 c = atomic64_read(v);2245 s64 c = atomic64_read(v);
@@ -2256,7 +2255,7 @@ atomic64_inc_unless_negative(atomic64_t *v) · 9 unmodified lines
2256#endif2255#endif
2257 2256
2258#ifndef atomic64_dec_unless_positive2257#ifndef atomic64_dec_unless_positive
2259static inline bool2258static __always_inline bool
2260atomic64_dec_unless_positive(atomic64_t *v)2259atomic64_dec_unless_positive(atomic64_t *v)
2261{2260{
2262 s64 c = atomic64_read(v);2261 s64 c = atomic64_read(v);
@@ -2272,7 +2271,7 @@ atomic64_dec_unless_positive(atomic64_t *v) · 9 unmodified lines
2272#endif2271#endif
2273 2272
2274#ifndef atomic64_dec_if_positive2273#ifndef atomic64_dec_if_positive
2275static inline s642274static __always_inline s64
2276atomic64_dec_if_positive(atomic64_t *v)2275atomic64_dec_if_positive(atomic64_t *v)
2277{2276{
2278 s64 dec, c = atomic64_read(v);2277 s64 dec, c = atomic64_read(v);
@@ -2288,8 +2287,5 @@ atomic64_dec_if_positive(atomic64_t *v) · 9 unmodified lines
2288#define atomic64_dec_if_positive atomic64_dec_if_positive2287#define atomic64_dec_if_positive atomic64_dec_if_positive
2289#endif2288#endif
2290 2289
2291#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
2292#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
2293
2294#endif /* _LINUX_ATOMIC_FALLBACK_H */2290#endif /* _LINUX_ATOMIC_FALLBACK_H */
2295// 25de4a2804d70f57e994fe3b419148658bb5378a2291// 1fac0941c79bf0ae100723cc2ac9b94061f0b67a
include/linux/atomic.h+11−0
@@ -25,6 +25,12 @@ · 24 unmodified lines
25 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.25 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
26 */26 */
27 27
28#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
29#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
30
31#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
32#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
33
28/*34/*
29 * The idea here is to build acquire/release variants by adding explicit35 * The idea here is to build acquire/release variants by adding explicit
30 * barriers on top of the relaxed variant. In the case where the relaxed36 * barriers on top of the relaxed variant. In the case where the relaxed
@@ -71,7 +77,12 @@ · 40 unmodified lines
71 __ret; \77 __ret; \
72})78})
73 79
80#ifdef ARCH_ATOMIC
81#include <linux/atomic-arch-fallback.h>
82#include <asm-generic/atomic-instrumented.h>
83#else
74#include <linux/atomic-fallback.h>84#include <linux/atomic-fallback.h>
85#endif
75 86
76#include <asm-generic/atomic-long.h>87#include <asm-generic/atomic-long.h>
77 88
scripts/atomic/fallbacks/acquire+3−3
@@ -1,8 +1,8 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_${pfx}${name}${sfx}_acquire(${params})3${arch}${atomic}_${pfx}${name}${sfx}_acquire(${params})
4{4{
5 ${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});5 ${ret} ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
6 __atomic_acquire_fence();6 __atomic_acquire_fence();
7 return ret;7 return ret;
8}8}
scripts/atomic/fallbacks/add_negative+4−4
@@ -1,6 +1,6 @@
1cat <<EOF1cat <<EOF
2/**2/**
3 * ${atomic}_add_negative - add and test if negative3 * ${arch}${atomic}_add_negative - add and test if negative
4 * @i: integer value to add4 * @i: integer value to add
5 * @v: pointer of type ${atomic}_t5 * @v: pointer of type ${atomic}_t
6 *6 *
@@ -8,9 +8,9 @@ cat <<EOF · 1 unmodified line
8 * if the result is negative, or false when8 * if the result is negative, or false when
9 * result is greater than or equal to zero.9 * result is greater than or equal to zero.
10 */10 */
11static inline bool11static __always_inline bool
12${atomic}_add_negative(${int} i, ${atomic}_t *v)12${arch}${atomic}_add_negative(${int} i, ${atomic}_t *v)
13{13{
14 return ${atomic}_add_return(i, v) < 0;14 return ${arch}${atomic}_add_return(i, v) < 0;
15}15}
16EOF16EOF
scripts/atomic/fallbacks/add_unless+4−4
@@ -1,6 +1,6 @@
1cat << EOF1cat << EOF
2/**2/**
3 * ${atomic}_add_unless - add unless the number is already a given value3 * ${arch}${atomic}_add_unless - add unless the number is already a given value
4 * @v: pointer of type ${atomic}_t4 * @v: pointer of type ${atomic}_t
5 * @a: the amount to add to v...5 * @a: the amount to add to v...
6 * @u: ...unless v is equal to u.6 * @u: ...unless v is equal to u.
@@ -8,9 +8,9 @@ cat << EOF · 1 unmodified line
8 * Atomically adds @a to @v, if @v was not already @u.8 * Atomically adds @a to @v, if @v was not already @u.
9 * Returns true if the addition was done.9 * Returns true if the addition was done.
10 */10 */
11static inline bool11static __always_inline bool
12${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)12${arch}${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13{13{
14 return ${atomic}_fetch_add_unless(v, a, u) != u;14 return ${arch}${atomic}_fetch_add_unless(v, a, u) != u;
15}15}
16EOF16EOF
scripts/atomic/fallbacks/andnot+3−3
@@ -1,7 +1,7 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)3${arch}${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
4{4{
5 ${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v);5 ${retstmt}${arch}${atomic}_${pfx}and${sfx}${order}(~i, v);
6}6}
7EOF7EOF
scripts/atomic/fallbacks/dec+3−3
@@ -1,7 +1,7 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)3${arch}${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
4{4{
5 ${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v);5 ${retstmt}${arch}${atomic}_${pfx}sub${sfx}${order}(1, v);
6}6}
7EOF7EOF
scripts/atomic/fallbacks/dec_and_test+4−4
@@ -1,15 +1,15 @@
1cat <<EOF1cat <<EOF
2/**2/**
3 * ${atomic}_dec_and_test - decrement and test3 * ${arch}${atomic}_dec_and_test - decrement and test
4 * @v: pointer of type ${atomic}_t4 * @v: pointer of type ${atomic}_t
5 *5 *
6 * Atomically decrements @v by 1 and6 * Atomically decrements @v by 1 and
7 * returns true if the result is 0, or false for all other7 * returns true if the result is 0, or false for all other
8 * cases.8 * cases.
9 */9 */
10static inline bool10static __always_inline bool
11${atomic}_dec_and_test(${atomic}_t *v)11${arch}${atomic}_dec_and_test(${atomic}_t *v)
12{12{
13 return ${atomic}_dec_return(v) == 0;13 return ${arch}${atomic}_dec_return(v) == 0;
14}14}
15EOF15EOF
scripts/atomic/fallbacks/dec_if_positive+4−4
@@ -1,14 +1,14 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_dec_if_positive(${atomic}_t *v)3${arch}${atomic}_dec_if_positive(${atomic}_t *v)
4{4{
5 ${int} dec, c = ${atomic}_read(v);5 ${int} dec, c = ${arch}${atomic}_read(v);
6 6
7 do {7 do {
8 dec = c - 1;8 dec = c - 1;
9 if (unlikely(dec < 0))9 if (unlikely(dec < 0))
10 break;10 break;
11 } while (!${atomic}_try_cmpxchg(v, &c, dec));11 } while (!${arch}${atomic}_try_cmpxchg(v, &c, dec));
12 12
13 return dec;13 return dec;
14}14}
scripts/atomic/fallbacks/dec_unless_positive+4−4
@@ -1,13 +1,13 @@
1cat <<EOF1cat <<EOF
2static inline bool2static __always_inline bool
3${atomic}_dec_unless_positive(${atomic}_t *v)3${arch}${atomic}_dec_unless_positive(${atomic}_t *v)
4{4{
5 ${int} c = ${atomic}_read(v);5 ${int} c = ${arch}${atomic}_read(v);
6 6
7 do {7 do {
8 if (unlikely(c > 0))8 if (unlikely(c > 0))
9 return false;9 return false;
10 } while (!${atomic}_try_cmpxchg(v, &c, c - 1));10 } while (!${arch}${atomic}_try_cmpxchg(v, &c, c - 1));
11 11
12 return true;12 return true;
13}13}
scripts/atomic/fallbacks/fence+3−3
@@ -1,10 +1,10 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_${pfx}${name}${sfx}(${params})3${arch}${atomic}_${pfx}${name}${sfx}(${params})
4{4{
5 ${ret} ret;5 ${ret} ret;
6 __atomic_pre_full_fence();6 __atomic_pre_full_fence();
7 ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});7 ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
8 __atomic_post_full_fence();8 __atomic_post_full_fence();
9 return ret;9 return ret;
10}10}
scripts/atomic/fallbacks/fetch_add_unless+5−5
@@ -1,6 +1,6 @@
1cat << EOF1cat << EOF
2/**2/**
3 * ${atomic}_fetch_add_unless - add unless the number is already a given value3 * ${arch}${atomic}_fetch_add_unless - add unless the number is already a given value
4 * @v: pointer of type ${atomic}_t4 * @v: pointer of type ${atomic}_t
5 * @a: the amount to add to v...5 * @a: the amount to add to v...
6 * @u: ...unless v is equal to u.6 * @u: ...unless v is equal to u.
@@ -8,15 +8,15 @@ cat << EOF · 1 unmodified line
8 * Atomically adds @a to @v, so long as @v was not already @u.8 * Atomically adds @a to @v, so long as @v was not already @u.
9 * Returns original value of @v9 * Returns original value of @v
10 */10 */
11static inline ${int}11static __always_inline ${int}
12${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)12${arch}${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13{13{
14 ${int} c = ${atomic}_read(v);14 ${int} c = ${arch}${atomic}_read(v);
15 15
16 do {16 do {
17 if (unlikely(c == u))17 if (unlikely(c == u))
18 break;18 break;
19 } while (!${atomic}_try_cmpxchg(v, &c, c + a));19 } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + a));
20 20
21 return c;21 return c;
22}22}
scripts/atomic/fallbacks/inc+3−3
@@ -1,7 +1,7 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)3${arch}${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
4{4{
5 ${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v);5 ${retstmt}${arch}${atomic}_${pfx}add${sfx}${order}(1, v);
6}6}
7EOF7EOF
scripts/atomic/fallbacks/inc_and_test+4−4
@@ -1,15 +1,15 @@
1cat <<EOF1cat <<EOF
2/**2/**
3 * ${atomic}_inc_and_test - increment and test3 * ${arch}${atomic}_inc_and_test - increment and test
4 * @v: pointer of type ${atomic}_t4 * @v: pointer of type ${atomic}_t
5 *5 *
6 * Atomically increments @v by 16 * Atomically increments @v by 1
7 * and returns true if the result is zero, or false for all7 * and returns true if the result is zero, or false for all
8 * other cases.8 * other cases.
9 */9 */
10static inline bool10static __always_inline bool
11${atomic}_inc_and_test(${atomic}_t *v)11${arch}${atomic}_inc_and_test(${atomic}_t *v)
12{12{
13 return ${atomic}_inc_return(v) == 0;13 return ${arch}${atomic}_inc_return(v) == 0;
14}14}
15EOF15EOF
scripts/atomic/fallbacks/inc_not_zero+4−4
@@ -1,14 +1,14 @@
1cat <<EOF1cat <<EOF
2/**2/**
3 * ${atomic}_inc_not_zero - increment unless the number is zero3 * ${arch}${atomic}_inc_not_zero - increment unless the number is zero
4 * @v: pointer of type ${atomic}_t4 * @v: pointer of type ${atomic}_t
5 *5 *
6 * Atomically increments @v by 1, if @v is non-zero.6 * Atomically increments @v by 1, if @v is non-zero.
7 * Returns true if the increment was done.7 * Returns true if the increment was done.
8 */8 */
9static inline bool9static __always_inline bool
10${atomic}_inc_not_zero(${atomic}_t *v)10${arch}${atomic}_inc_not_zero(${atomic}_t *v)
11{11{
12 return ${atomic}_add_unless(v, 1, 0);12 return ${arch}${atomic}_add_unless(v, 1, 0);
13}13}
14EOF14EOF
scripts/atomic/fallbacks/inc_unless_negative+4−4
@@ -1,13 +1,13 @@
1cat <<EOF1cat <<EOF
2static inline bool2static __always_inline bool
3${atomic}_inc_unless_negative(${atomic}_t *v)3${arch}${atomic}_inc_unless_negative(${atomic}_t *v)
4{4{
5 ${int} c = ${atomic}_read(v);5 ${int} c = ${arch}${atomic}_read(v);
6 6
7 do {7 do {
8 if (unlikely(c < 0))8 if (unlikely(c < 0))
9 return false;9 return false;
10 } while (!${atomic}_try_cmpxchg(v, &c, c + 1));10 } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + 1));
11 11
12 return true;12 return true;
13}13}
scripts/atomic/fallbacks/read_acquire+2−2
@@ -1,6 +1,6 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_read_acquire(const ${atomic}_t *v)3${arch}${atomic}_read_acquire(const ${atomic}_t *v)
4{4{
5 return smp_load_acquire(&(v)->counter);5 return smp_load_acquire(&(v)->counter);
6}6}
scripts/atomic/fallbacks/release+3−3
@@ -1,8 +1,8 @@
1cat <<EOF1cat <<EOF
2static inline ${ret}2static __always_inline ${ret}
3${atomic}_${pfx}${name}${sfx}_release(${params})3${arch}${atomic}_${pfx}${name}${sfx}_release(${params})
4{4{
5 __atomic_release_fence();5 __atomic_release_fence();
6 ${retstmt}${atomic}_${pfx}${name}${sfx}_relaxed(${args});6 ${retstmt}${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
7}7}
8EOF8EOF
scripts/atomic/fallbacks/set_release+2−2
@@ -1,6 +1,6 @@
1cat <<EOF1cat <<EOF
2static inline void2static __always_inline void
3${atomic}_set_release(${atomic}_t *v, ${int} i)3${arch}${atomic}_set_release(${atomic}_t *v, ${int} i)
4{4{
5 smp_store_release(&(v)->counter, i);5 smp_store_release(&(v)->counter, i);
6}6}
scripts/atomic/fallbacks/sub_and_test+4−4
@@ -1,6 +1,6 @@
1cat <<EOF1cat <<EOF
2/**2/**
3 * ${atomic}_sub_and_test - subtract value from variable and test result3 * ${arch}${atomic}_sub_and_test - subtract value from variable and test result
4 * @i: integer value to subtract4 * @i: integer value to subtract
5 * @v: pointer of type ${atomic}_t5 * @v: pointer of type ${atomic}_t
6 *6 *
@@ -8,9 +8,9 @@ cat <<EOF · 1 unmodified line
8 * true if the result is zero, or false for all8 * true if the result is zero, or false for all
9 * other cases.9 * other cases.
10 */10 */
11static inline bool11static __always_inline bool
12${atomic}_sub_and_test(${int} i, ${atomic}_t *v)12${arch}${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
13{13{
14 return ${atomic}_sub_return(i, v) == 0;14 return ${arch}${atomic}_sub_return(i, v) == 0;
15}15}
16EOF16EOF
scripts/atomic/fallbacks/try_cmpxchg+3−3
@@ -1,9 +1,9 @@
1cat <<EOF1cat <<EOF
2static inline bool2static __always_inline bool
3${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)3${arch}${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
4{4{
5 ${int} r, o = *old;5 ${int} r, o = *old;
6 r = ${atomic}_cmpxchg${order}(v, o, new);6 r = ${arch}${atomic}_cmpxchg${order}(v, o, new);
7 if (unlikely(r != o))7 if (unlikely(r != o))
8 *old = r;8 *old = r;
9 return likely(r == o);9 return likely(r == o);
scripts/atomic/gen-atomic-fallback.sh+15−16
@@ -2,10 +2,11 @@ · 1 unmodified line
2# SPDX-License-Identifier: GPL-2.02# SPDX-License-Identifier: GPL-2.0
3 3
4ATOMICDIR=$(dirname $0)4ATOMICDIR=$(dirname $0)
5ARCH=$2
5 6
6. ${ATOMICDIR}/atomic-tbl.sh7. ${ATOMICDIR}/atomic-tbl.sh
7 8
8#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...)9#gen_template_fallback(template, meta, pfx, name, sfx, order, arch, atomic, int, args...)
9gen_template_fallback()10gen_template_fallback()
10{11{
11 local template="$1"; shift12 local template="$1"; shift
@@ -14,10 +15,11 @@ gen_template_fallback() · 2 unmodified lines
14 local name="$1"; shift15 local name="$1"; shift
15 local sfx="$1"; shift16 local sfx="$1"; shift
16 local order="$1"; shift17 local order="$1"; shift
18 local arch="$1"; shift
17 local atomic="$1"; shift19 local atomic="$1"; shift
18 local int="$1"; shift20 local int="$1"; shift
19 21
20 local atomicname="${atomic}_${pfx}${name}${sfx}${order}"22 local atomicname="${arch}${atomic}_${pfx}${name}${sfx}${order}"
21 23
22 local ret="$(gen_ret_type "${meta}" "${int}")"24 local ret="$(gen_ret_type "${meta}" "${int}")"
23 local retstmt="$(gen_ret_stmt "${meta}")"25 local retstmt="$(gen_ret_stmt "${meta}")"
@@ -32,7 +34,7 @@ gen_template_fallback() · 8 unmodified lines
32 fi34 fi
33}35}
34 36
35#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)37#gen_proto_fallback(meta, pfx, name, sfx, order, arch, atomic, int, args...)
36gen_proto_fallback()38gen_proto_fallback()
37{39{
38 local meta="$1"; shift40 local meta="$1"; shift
@@ -56,16 +58,17 @@ cat << EOF · 17 unmodified lines
56EOF58EOF
57}59}
58 60
59#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)61#gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...)
60gen_proto_order_variants()62gen_proto_order_variants()
61{63{
62 local meta="$1"; shift64 local meta="$1"; shift
63 local pfx="$1"; shift65 local pfx="$1"; shift
64 local name="$1"; shift66 local name="$1"; shift
65 local sfx="$1"; shift67 local sfx="$1"; shift
66 local atomic="$1"68 local arch="$1"
69 local atomic="$2"
67 70
68 local basename="${atomic}_${pfx}${name}${sfx}"71 local basename="${arch}${atomic}_${pfx}${name}${sfx}"
69 72
70 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"73 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
71 74
@@ -94,7 +97,7 @@ gen_proto_order_variants() · 22 unmodified lines
94 gen_basic_fallbacks "${basename}"97 gen_basic_fallbacks "${basename}"
95 98
96 if [ ! -z "${template}" ]; then99 if [ ! -z "${template}" ]; then
97 printf "#endif /* ${atomic}_${pfx}${name}${sfx} */\n\n"100 printf "#endif /* ${arch}${atomic}_${pfx}${name}${sfx} */\n\n"
98 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"101 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
99 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"102 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
100 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"103 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
@@ -149,20 +152,19 @@ cat << EOF · 48 unmodified lines
149#ifndef _LINUX_ATOMIC_FALLBACK_H152#ifndef _LINUX_ATOMIC_FALLBACK_H
150#define _LINUX_ATOMIC_FALLBACK_H153#define _LINUX_ATOMIC_FALLBACK_H
151 154
155#include <linux/compiler.h>
156
152EOF157EOF
153 158
154for xchg in "xchg" "cmpxchg" "cmpxchg64"; do159for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do
155 gen_xchg_fallbacks "${xchg}"160 gen_xchg_fallbacks "${xchg}"
156done161done
157 162
158grep '^[a-z]' "$1" | while read name meta args; do163grep '^[a-z]' "$1" | while read name meta args; do
159 gen_proto "${meta}" "${name}" "atomic" "int" ${args}164 gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args}
160done165done
161 166
162cat <<EOF167cat <<EOF
163#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
164#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
165
166#ifdef CONFIG_GENERIC_ATOMIC64168#ifdef CONFIG_GENERIC_ATOMIC64
167#include <asm-generic/atomic64.h>169#include <asm-generic/atomic64.h>
168#endif170#endif
@@ -170,12 +172,9 @@ cat <<EOF · 1 unmodified line
170EOF172EOF
171 173
172grep '^[a-z]' "$1" | while read name meta args; do174grep '^[a-z]' "$1" | while read name meta args; do
173 gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}175 gen_proto "${meta}" "${name}" "${ARCH}" "atomic64" "s64" ${args}
174done176done
175 177
176cat <<EOF178cat <<EOF
177#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
178#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
179
180#endif /* _LINUX_ATOMIC_FALLBACK_H */179#endif /* _LINUX_ATOMIC_FALLBACK_H */
181EOF180EOF
scripts/atomic/gen-atomics.sh+3−2
@@ -10,10 +10,11 @@ LINUXDIR=${ATOMICDIR}/../.. · 9 unmodified lines
10cat <<EOF |10cat <<EOF |
11gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h11gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h
12gen-atomic-long.sh asm-generic/atomic-long.h12gen-atomic-long.sh asm-generic/atomic-long.h
13gen-atomic-fallback.sh linux/atomic-arch-fallback.h arch_
13gen-atomic-fallback.sh linux/atomic-fallback.h14gen-atomic-fallback.sh linux/atomic-fallback.h
14EOF15EOF
15while read script header; do16while read script header args; do
16 /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} > ${LINUXDIR}/include/${header}17 /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header}
17 HASH="$(sha1sum ${LINUXDIR}/include/${header})"18 HASH="$(sha1sum ${LINUXDIR}/include/${header})"
18 HASH="${HASH%% *}"19 HASH="${HASH%% *}"
19 printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header}20 printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header}