History

Merge tag 'locking-core-2023-06-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar: - Introduce cmpxchg128() -- aka. the demise of cmpxchg_double() The cmpxchg128() family of functions is basically & functionally the same as cmpxchg_double(), but with a saner interface. Instead of a 6-parameter horror that forced u128 - u64/u64-halves layout details on the interface and exposed users to complexity, fragility & bugs, use a natural 3-parameter interface with u128 types. - Restructure the generated atomic headers, and add kerneldoc comments for all of the generic atomic{,64,_long}_t operations. The generated definitions are much cleaner now, and come with documentation. - Implement lock_set_cmp_fn() on lockdep, for defining an ordering when taking multiple locks of the same type. This gets rid of one use of lockdep_set_novalidate_class() in the bcache code. - Fix raw_cpu_generic_try_cmpxchg() bug due to an unintended variable shadowing generating garbage code on Clang on certain ARM builds. * tag 'locking-core-2023-06-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (43 commits) locking/atomic: scripts: fix ${atomic}_dec_if_positive() kerneldoc percpu: Fix self-assignment of __old in raw_cpu_generic_try_cmpxchg() locking/atomic: treewide: delete arch_atomic_*() kerneldoc locking/atomic: docs: Add atomic operations to the driver basic API documentation locking/atomic: scripts: generate kerneldoc comments docs: scripts: kernel-doc: accept bitwise negation like ~@var locking/atomic: scripts: simplify raw_atomic*() definitions locking/atomic: scripts: simplify raw_atomic_long*() definitions locking/atomic: scripts: split pfx/name/sfx/order locking/atomic: scripts: restructure fallback ifdeffery locking/atomic: scripts: build raw_atomic_long*() directly locking/atomic: treewide: use raw_atomic*_<op>() locking/atomic: scripts: add trivial raw_atomic*_<op>() locking/atomic: scripts: factor out order template generation locking/atomic: scripts: remove leftover "${mult}" locking/atomic: scripts: remove bogus order parameter locking/atomic: xtensa: add preprocessor symbols locking/atomic: x86: add preprocessor symbols locking/atomic: sparc: add preprocessor symbols locking/atomic: sh: add preprocessor symbols ...

Linus Torvalds committed 3 years ago

Commit bc6cb4d2 parents ed3b792 b33eb50136 files · +9922 −4161Browse files

Unified · Split

136 files
  1. Documentation/core-api/this_cpu_ops.rst +0 −2
  2. Documentation/driver-api/basics.rst +7 −1
  3. arch/alpha/include/asm/atomic.h +0 −35
  4. arch/arc/include/asm/atomic-spinlock.h +9 −0
  5. arch/arc/include/asm/atomic.h +0 −24
  6. arch/arc/include/asm/atomic64-arcv2.h +2 −17
  7. arch/arm/include/asm/assembler.h +17 −0
  8. arch/arm/include/asm/atomic.h +11 −4
  9. arch/arm/include/asm/sync_bitops.h +25 −4
  10. arch/arm/lib/bitops.h +11 −3
  11. arch/arm/lib/testchangebit.S +4 −0
  12. arch/arm/lib/testclearbit.S +4 −0
  13. arch/arm/lib/testsetbit.S +4 −0
  14. arch/arm64/include/asm/atomic.h +0 −28
  15. arch/arm64/include/asm/atomic_ll_sc.h +32 −24
  16. arch/arm64/include/asm/atomic_lse.h +17 −22
  17. arch/arm64/include/asm/cmpxchg.h +14 −34
  18. arch/arm64/include/asm/percpu.h +20 −10
  19. arch/csky/include/asm/atomic.h +0 −35
  20. arch/hexagon/include/asm/atomic.h +9 −60
  21. arch/ia64/include/asm/atomic.h +0 −7
  22. arch/loongarch/include/asm/atomic.h +0 −56
  23. arch/m68k/include/asm/atomic.h +12 −6
  24. arch/mips/include/asm/atomic.h +0 −11
  25. arch/openrisc/include/asm/atomic.h +0 −3
  26. arch/parisc/include/asm/atomic.h +18 −9
  27. arch/powerpc/include/asm/atomic.h +0 −24
  28. arch/powerpc/kernel/smp.c +6 −6
  29. arch/riscv/include/asm/atomic.h +0 −72
  30. arch/s390/include/asm/cmpxchg.h +6 −26
  31. arch/s390/include/asm/cpu_mf.h +1 −1
  32. arch/s390/include/asm/percpu.h +16 −18
  33. arch/s390/kernel/perf_cpum_sf.c +3 −13
  34. arch/sh/include/asm/atomic-grb.h +9 −0
  35. arch/sh/include/asm/atomic-irq.h +9 −0
  36. arch/sh/include/asm/atomic-llsc.h +9 −0
  37. arch/sh/include/asm/atomic.h +0 −3
  38. arch/sparc/include/asm/atomic_32.h +16 −2
  39. arch/sparc/include/asm/atomic_64.h +18 −11
  40. arch/x86/include/asm/atomic.h +0 −87
  41. arch/x86/include/asm/atomic64_32.h +0 −76
  42. arch/x86/include/asm/atomic64_64.h +0 −81
  43. arch/x86/include/asm/cmpxchg.h +0 −25
  44. arch/x86/include/asm/cmpxchg_32.h +1 −1
  45. arch/x86/include/asm/cmpxchg_64.h +66 −1
  46. arch/x86/include/asm/percpu.h +61 −41
  47. arch/x86/kernel/alternative.c +2 −2
  48. arch/x86/kernel/cpu/mce/core.c +8 −8
  49. arch/x86/kernel/nmi.c +1 −1
  50. arch/x86/kernel/pvclock.c +2 −2
  51. arch/x86/kvm/x86.c +1 −1
  52. arch/x86/lib/Makefile +2 −1
  53. arch/x86/lib/cmpxchg16b_emu.S +25 −18
  54. arch/x86/lib/cmpxchg8b_emu.S +53 −14
  55. arch/xtensa/include/asm/atomic.h +9 −3
  56. drivers/iommu/amd/amd_iommu_types.h +7 −2
  57. drivers/iommu/amd/iommu.c +4 −6
  58. drivers/iommu/intel/irq_remapping.c +2 −6
  59. drivers/md/bcache/btree.c +22 −1
  60. drivers/md/bcache/btree.h +2 −2
  61. include/asm-generic/atomic.h +0 −3
  62. include/asm-generic/bitops/atomic.h +6 −6
  63. include/asm-generic/bitops/lock.h +4 −4
  64. include/asm-generic/percpu.h +152 −49
  65. include/crypto/b128ops.h +3 −11
  66. include/linux/atomic/atomic-arch-fallback.h +3576 −1587
  67. include/linux/atomic/atomic-instrumented.h +3194 −361
  68. include/linux/atomic/atomic-long.h +1435 −687
  69. include/linux/context_tracking.h +2 −2
  70. include/linux/context_tracking_state.h +1 −1
  71. include/linux/cpumask.h +1 −1
  72. include/linux/dmar.h +66 −59
  73. include/linux/jump_label.h +1 −1
  74. include/linux/lockdep.h +8 −0
  75. include/linux/lockdep_types.h +8 −0
  76. include/linux/percpu-defs.h +13 −32
  77. include/linux/slub_def.h +9 −3
  78. include/linux/types.h +5 −0
  79. include/uapi/linux/types.h +4 −0
  80. kernel/context_tracking.c +6 −6
  81. kernel/locking/lockdep.c +87 −31
  82. kernel/sched/clock.c +1 −1
  83. lib/crypto/curve25519-hacl64.c +0 −2
  84. lib/crypto/poly1305-donna64.c +0 −2
  85. mm/slab.h +46 −7
  86. mm/slub.c +82 −57
  87. scripts/atomic/atomic-tbl.sh +108 −4
  88. scripts/atomic/atomics.tbl +1 −1
  89. scripts/atomic/fallbacks/acquire +0 −4
  90. scripts/atomic/fallbacks/add_negative +1 −13
  91. scripts/atomic/fallbacks/add_unless +1 −14
  92. scripts/atomic/fallbacks/andnot +1 −5
  93. scripts/atomic/fallbacks/cmpxchg +3 −0
  94. scripts/atomic/fallbacks/dec +1 −5
  95. scripts/atomic/fallbacks/dec_and_test +1 −13
  96. scripts/atomic/fallbacks/dec_if_positive +2 −6
  97. scripts/atomic/fallbacks/dec_unless_positive +2 −6
  98. scripts/atomic/fallbacks/fence +0 −4
  99. scripts/atomic/fallbacks/fetch_add_unless +2 −15
  100. scripts/atomic/fallbacks/inc +1 −5
  101. scripts/atomic/fallbacks/inc_and_test +1 −13
  102. scripts/atomic/fallbacks/inc_not_zero +1 −12
  103. scripts/atomic/fallbacks/inc_unless_negative +2 −6
  104. scripts/atomic/fallbacks/read_acquire +1 −5
  105. scripts/atomic/fallbacks/release +0 −4
  106. scripts/atomic/fallbacks/set_release +1 −5
  107. scripts/atomic/fallbacks/sub_and_test +1 −14
  108. scripts/atomic/fallbacks/try_cmpxchg +1 −5
  109. scripts/atomic/fallbacks/xchg +3 −0
  110. scripts/atomic/gen-atomic-fallback.sh +175 −91
  111. scripts/atomic/gen-atomic-instrumented.sh +16 −26
  112. scripts/atomic/gen-atomic-long.sh +18 −20
  113. scripts/atomic/kerneldoc/add +13 −0
  114. scripts/atomic/kerneldoc/add_negative +13 −0
  115. scripts/atomic/kerneldoc/add_unless +18 −0
  116. scripts/atomic/kerneldoc/and +13 −0
  117. scripts/atomic/kerneldoc/andnot +13 −0
  118. scripts/atomic/kerneldoc/cmpxchg +14 −0
  119. scripts/atomic/kerneldoc/dec +12 −0
  120. scripts/atomic/kerneldoc/dec_and_test +12 −0
  121. scripts/atomic/kerneldoc/dec_if_positive +12 −0
  122. scripts/atomic/kerneldoc/dec_unless_positive +12 −0
  123. scripts/atomic/kerneldoc/inc +12 −0
  124. scripts/atomic/kerneldoc/inc_and_test +12 −0
  125. scripts/atomic/kerneldoc/inc_not_zero +12 −0
  126. scripts/atomic/kerneldoc/inc_unless_negative +12 −0
  127. scripts/atomic/kerneldoc/or +13 −0
  128. scripts/atomic/kerneldoc/read +12 −0
  129. scripts/atomic/kerneldoc/set +13 −0
  130. scripts/atomic/kerneldoc/sub +13 −0
  131. scripts/atomic/kerneldoc/sub_and_test +13 −0
  132. scripts/atomic/kerneldoc/try_cmpxchg +15 −0
  133. scripts/atomic/kerneldoc/xchg +13 −0
  134. scripts/atomic/kerneldoc/xor +13 −0
  135. scripts/kernel-doc +1 −1
  136. scripts/min-tool-version.sh +5 −1
Documentation/core-api/this_cpu_ops.rst+0−2
@@ -53,7 +53,6 @@ preemption and interrupts:: · 52 unmodified lines
53 this_cpu_add_return(pcp, val)53 this_cpu_add_return(pcp, val)
54 this_cpu_xchg(pcp, nval)54 this_cpu_xchg(pcp, nval)
55 this_cpu_cmpxchg(pcp, oval, nval)55 this_cpu_cmpxchg(pcp, oval, nval)
56 this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
57 this_cpu_sub(pcp, val)56 this_cpu_sub(pcp, val)
58 this_cpu_inc(pcp)57 this_cpu_inc(pcp)
59 this_cpu_dec(pcp)58 this_cpu_dec(pcp)
@@ -242,7 +241,6 @@ safe:: · 182 unmodified lines
242 __this_cpu_add_return(pcp, val)241 __this_cpu_add_return(pcp, val)
243 __this_cpu_xchg(pcp, nval)242 __this_cpu_xchg(pcp, nval)
244 __this_cpu_cmpxchg(pcp, oval, nval)243 __this_cpu_cmpxchg(pcp, oval, nval)
245 __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
246 __this_cpu_sub(pcp, val)244 __this_cpu_sub(pcp, val)
247 __this_cpu_inc(pcp)245 __this_cpu_inc(pcp)
248 __this_cpu_dec(pcp)246 __this_cpu_dec(pcp)
Documentation/driver-api/basics.rst+7−1
@@ -84,7 +84,13 @@ Reference counting · 83 unmodified lines
84Atomics84Atomics
85-------85-------
86 86
87.. kernel-doc:: arch/x86/include/asm/atomic.h87.. kernel-doc:: include/linux/atomic/atomic-instrumented.h
88 :internal:
89
90.. kernel-doc:: include/linux/atomic/atomic-arch-fallback.h
91 :internal:
92
93.. kernel-doc:: include/linux/atomic/atomic-long.h
88 :internal:94 :internal:
89 95
90Kernel objects manipulation96Kernel objects manipulation
arch/alpha/include/asm/atomic.h+0−35
@@ -200,25 +200,6 @@ ATOMIC_OPS(xor, xor) · 199 unmodified lines
200#undef ATOMIC_OP_RETURN200#undef ATOMIC_OP_RETURN
201#undef ATOMIC_OP201#undef ATOMIC_OP
202 202
203#define arch_atomic64_cmpxchg(v, old, new) \
204 (arch_cmpxchg(&((v)->counter), old, new))
205#define arch_atomic64_xchg(v, new) \
206 (arch_xchg(&((v)->counter), new))
207
208#define arch_atomic_cmpxchg(v, old, new) \
209 (arch_cmpxchg(&((v)->counter), old, new))
210#define arch_atomic_xchg(v, new) \
211 (arch_xchg(&((v)->counter), new))
212
213/**
214 * arch_atomic_fetch_add_unless - add unless the number is a given value
215 * @v: pointer of type atomic_t
216 * @a: the amount to add to v...
217 * @u: ...unless v is equal to u.
218 *
219 * Atomically adds @a to @v, so long as it was not @u.
220 * Returns the old value of @v.
221 */
222static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)203static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
223{204{
224 int c, new, old;205 int c, new, old;
@@ -242,15 +223,6 @@ static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) · 17 unmodified lines
242}223}
243#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless224#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
244 225
245/**
246 * arch_atomic64_fetch_add_unless - add unless the number is a given value
247 * @v: pointer of type atomic64_t
248 * @a: the amount to add to v...
249 * @u: ...unless v is equal to u.
250 *
251 * Atomically adds @a to @v, so long as it was not @u.
252 * Returns the old value of @v.
253 */
254static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)226static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
255{227{
256 s64 c, new, old;228 s64 c, new, old;
@@ -274,13 +246,6 @@ static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u · 17 unmodified lines
274}246}
275#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless247#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
276 248
277/*
278 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
279 * @v: pointer of type atomic_t
280 *
281 * The function returns the old value of *v minus 1, even if
282 * the atomic variable, v, was not decremented.
283 */
284static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)249static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
285{250{
286 s64 old, tmp;251 s64 old, tmp;
arch/arc/include/asm/atomic-spinlock.h+9−0
@@ -81,6 +81,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 80 unmodified lines
81ATOMIC_OPS(add, +=, add)81ATOMIC_OPS(add, +=, add)
82ATOMIC_OPS(sub, -=, sub)82ATOMIC_OPS(sub, -=, sub)
83 83
84#define arch_atomic_fetch_add arch_atomic_fetch_add
85#define arch_atomic_fetch_sub arch_atomic_fetch_sub
86#define arch_atomic_add_return arch_atomic_add_return
87#define arch_atomic_sub_return arch_atomic_sub_return
88
84#undef ATOMIC_OPS89#undef ATOMIC_OPS
85#define ATOMIC_OPS(op, c_op, asm_op) \90#define ATOMIC_OPS(op, c_op, asm_op) \
86 ATOMIC_OP(op, c_op, asm_op) \91 ATOMIC_OP(op, c_op, asm_op) \
@@ -92,7 +97,11 @@ ATOMIC_OPS(or, |=, or) · 5 unmodified lines
92ATOMIC_OPS(xor, ^=, xor)97ATOMIC_OPS(xor, ^=, xor)
93 98
94#define arch_atomic_andnot arch_atomic_andnot99#define arch_atomic_andnot arch_atomic_andnot
100
101#define arch_atomic_fetch_and arch_atomic_fetch_and
95#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot102#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
103#define arch_atomic_fetch_or arch_atomic_fetch_or
104#define arch_atomic_fetch_xor arch_atomic_fetch_xor
96 105
97#undef ATOMIC_OPS106#undef ATOMIC_OPS
98#undef ATOMIC_FETCH_OP107#undef ATOMIC_FETCH_OP
arch/arc/include/asm/atomic.h+0−24
@@ -22,30 +22,6 @@ · 21 unmodified lines
22#include <asm/atomic-spinlock.h>22#include <asm/atomic-spinlock.h>
23#endif23#endif
24 24
25#define arch_atomic_cmpxchg(v, o, n) \
26({ \
27 arch_cmpxchg(&((v)->counter), (o), (n)); \
28})
29
30#ifdef arch_cmpxchg_relaxed
31#define arch_atomic_cmpxchg_relaxed(v, o, n) \
32({ \
33 arch_cmpxchg_relaxed(&((v)->counter), (o), (n)); \
34})
35#endif
36
37#define arch_atomic_xchg(v, n) \
38({ \
39 arch_xchg(&((v)->counter), (n)); \
40})
41
42#ifdef arch_xchg_relaxed
43#define arch_atomic_xchg_relaxed(v, n) \
44({ \
45 arch_xchg_relaxed(&((v)->counter), (n)); \
46})
47#endif
48
49/*25/*
50 * 64-bit atomics26 * 64-bit atomics
51 */27 */
arch/arc/include/asm/atomic64-arcv2.h+2−17
@@ -159,6 +159,7 @@ arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) · 158 unmodified lines
159 159
160 return prev;160 return prev;
161}161}
162#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
162 163
163static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)164static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
164{165{
@@ -179,14 +180,7 @@ static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) · 14 unmodified lines
179 180
180 return prev;181 return prev;
181}182}
182 183#define arch_atomic64_xchg arch_atomic64_xchg
183/**
184 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
185 * @v: pointer of type atomic64_t
186 *
187 * The function returns the old value of *v minus 1, even if
188 * the atomic variable, v, was not decremented.
189 */
190 184
191static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)185static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
192{186{
@@ -212,15 +206,6 @@ static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) · 19 unmodified lines
212}206}
213#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive207#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
214 208
215/**
216 * arch_atomic64_fetch_add_unless - add unless the number is a given value
217 * @v: pointer of type atomic64_t
218 * @a: the amount to add to v...
219 * @u: ...unless v is equal to u.
220 *
221 * Atomically adds @a to @v, if it was not @u.
222 * Returns the old value of @v
223 */
224static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)209static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
225{210{
226 s64 old, temp;211 s64 old, temp;
arch/arm/include/asm/assembler.h+17−0
@@ -394,6 +394,23 @@ ALT_UP_B(.L0_\@) · 393 unmodified lines
394#endif394#endif
395 .endm395 .endm
396 396
397/*
398 * Raw SMP data memory barrier
399 */
400 .macro __smp_dmb mode
401#if __LINUX_ARM_ARCH__ >= 7
402 .ifeqs "\mode","arm"
403 dmb ish
404 .else
405 W(dmb) ish
406 .endif
407#elif __LINUX_ARM_ARCH__ == 6
408 mcr p15, 0, r0, c7, c10, 5 @ dmb
409#else
410 .error "Incompatible SMP platform"
411#endif
412 .endm
413
397#if defined(CONFIG_CPU_V7M)414#if defined(CONFIG_CPU_V7M)
398 /*415 /*
399 * setmode is used to assert to be in svc mode during boot. For v7-M416 * setmode is used to assert to be in svc mode during boot. For v7-M
arch/arm/include/asm/atomic.h+11−4
@@ -197,6 +197,16 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 196 unmodified lines
197 return val; \197 return val; \
198}198}
199 199
200#define arch_atomic_add_return arch_atomic_add_return
201#define arch_atomic_sub_return arch_atomic_sub_return
202#define arch_atomic_fetch_add arch_atomic_fetch_add
203#define arch_atomic_fetch_sub arch_atomic_fetch_sub
204
205#define arch_atomic_fetch_and arch_atomic_fetch_and
206#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
207#define arch_atomic_fetch_or arch_atomic_fetch_or
208#define arch_atomic_fetch_xor arch_atomic_fetch_xor
209
200static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)210static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
201{211{
202 int ret;212 int ret;
@@ -210,8 +220,7 @@ static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) · 7 unmodified lines
210 220
211 return ret;221 return ret;
212}222}
213 223#define arch_atomic_cmpxchg arch_atomic_cmpxchg
214#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
215 224
216#endif /* __LINUX_ARM_ARCH__ */225#endif /* __LINUX_ARM_ARCH__ */
217 226
@@ -240,8 +249,6 @@ ATOMIC_OPS(xor, ^=, eor) · 22 unmodified lines
240#undef ATOMIC_OP_RETURN249#undef ATOMIC_OP_RETURN
241#undef ATOMIC_OP250#undef ATOMIC_OP
242 251
243#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
244
245#ifndef CONFIG_GENERIC_ATOMIC64252#ifndef CONFIG_GENERIC_ATOMIC64
246typedef struct {253typedef struct {
247 s64 counter;254 s64 counter;
arch/arm/include/asm/sync_bitops.h+25−4
@@ -14,14 +14,35 @@ · 13 unmodified lines
14 * ops which are SMP safe even on a UP kernel.14 * ops which are SMP safe even on a UP kernel.
15 */15 */
16 16
17/*
18 * Unordered
19 */
20
17#define sync_set_bit(nr, p) _set_bit(nr, p)21#define sync_set_bit(nr, p) _set_bit(nr, p)
18#define sync_clear_bit(nr, p) _clear_bit(nr, p)22#define sync_clear_bit(nr, p) _clear_bit(nr, p)
19#define sync_change_bit(nr, p) _change_bit(nr, p)23#define sync_change_bit(nr, p) _change_bit(nr, p)
20#define sync_test_and_set_bit(nr, p) _test_and_set_bit(nr, p)
21#define sync_test_and_clear_bit(nr, p) _test_and_clear_bit(nr, p)
22#define sync_test_and_change_bit(nr, p) _test_and_change_bit(nr, p)
23#define sync_test_bit(nr, addr) test_bit(nr, addr)24#define sync_test_bit(nr, addr) test_bit(nr, addr)
24#define arch_sync_cmpxchg arch_cmpxchg
25 25
26/*
27 * Fully ordered
28 */
29
30int _sync_test_and_set_bit(int nr, volatile unsigned long * p);
31#define sync_test_and_set_bit(nr, p) _sync_test_and_set_bit(nr, p)
32
33int _sync_test_and_clear_bit(int nr, volatile unsigned long * p);
34#define sync_test_and_clear_bit(nr, p) _sync_test_and_clear_bit(nr, p)
35
36int _sync_test_and_change_bit(int nr, volatile unsigned long * p);
37#define sync_test_and_change_bit(nr, p) _sync_test_and_change_bit(nr, p)
38
39#define arch_sync_cmpxchg(ptr, old, new) \
40({ \
41 __typeof__(*(ptr)) __ret; \
42 __smp_mb__before_atomic(); \
43 __ret = arch_cmpxchg_relaxed((ptr), (old), (new)); \
44 __smp_mb__after_atomic(); \
45 __ret; \
46})
26 47
27#endif48#endif
arch/arm/lib/bitops.h+11−3
@@ -28,7 +28,7 @@ UNWIND( .fnend ) · 27 unmodified lines
28ENDPROC(\name )28ENDPROC(\name )
29 .endm29 .endm
30 30
31 .macro testop, name, instr, store31 .macro __testop, name, instr, store, barrier
32ENTRY( \name )32ENTRY( \name )
33UNWIND( .fnstart )33UNWIND( .fnstart )
34 ands ip, r1, #334 ands ip, r1, #3
@@ -38,7 +38,7 @@ UNWIND( .fnstart ) · 3 unmodified lines
38 mov r0, r0, lsr #538 mov r0, r0, lsr #5
39 add r1, r1, r0, lsl #2 @ Get word offset39 add r1, r1, r0, lsl #2 @ Get word offset
40 mov r3, r2, lsl r3 @ create mask40 mov r3, r2, lsl r3 @ create mask
41 smp_dmb41 \barrier
42#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)42#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
43 .arch_extension mp43 .arch_extension mp
44 ALT_SMP(W(pldw) [r1])44 ALT_SMP(W(pldw) [r1])
@@ -50,13 +50,21 @@ UNWIND( .fnstart ) · 5 unmodified lines
50 strex ip, r2, [r1]50 strex ip, r2, [r1]
51 cmp ip, #051 cmp ip, #0
52 bne 1b52 bne 1b
53 smp_dmb53 \barrier
54 cmp r0, #054 cmp r0, #0
55 movne r0, #155 movne r0, #1
562: bx lr562: bx lr
57UNWIND( .fnend )57UNWIND( .fnend )
58ENDPROC(\name )58ENDPROC(\name )
59 .endm59 .endm
60
61 .macro testop, name, instr, store
62 __testop \name, \instr, \store, smp_dmb
63 .endm
64
65 .macro sync_testop, name, instr, store
66 __testop \name, \instr, \store, __smp_dmb
67 .endm
60#else68#else
61 .macro bitop, name, instr69 .macro bitop, name, instr
62ENTRY( \name )70ENTRY( \name )
arch/arm/lib/testchangebit.S+4−0
@@ -10,3 +10,7 @@ · 9 unmodified lines
10 .text10 .text
11 11
12testop _test_and_change_bit, eor, str12testop _test_and_change_bit, eor, str
13
14#if __LINUX_ARM_ARCH__ >= 6
15sync_testop _sync_test_and_change_bit, eor, str
16#endif
arch/arm/lib/testclearbit.S+4−0
@@ -10,3 +10,7 @@ · 9 unmodified lines
10 .text10 .text
11 11
12testop _test_and_clear_bit, bicne, strne12testop _test_and_clear_bit, bicne, strne
13
14#if __LINUX_ARM_ARCH__ >= 6
15sync_testop _sync_test_and_clear_bit, bicne, strne
16#endif
arch/arm/lib/testsetbit.S+4−0
@@ -10,3 +10,7 @@ · 9 unmodified lines
10 .text10 .text
11 11
12testop _test_and_set_bit, orreq, streq12testop _test_and_set_bit, orreq, streq
13
14#if __LINUX_ARM_ARCH__ >= 6
15sync_testop _sync_test_and_set_bit, orreq, streq
16#endif
arch/arm64/include/asm/atomic.h+0−28
@@ -142,24 +142,6 @@ static __always_inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 141 unmodified lines
142#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release142#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
143#define arch_atomic_fetch_xor arch_atomic_fetch_xor143#define arch_atomic_fetch_xor arch_atomic_fetch_xor
144 144
145#define arch_atomic_xchg_relaxed(v, new) \
146 arch_xchg_relaxed(&((v)->counter), (new))
147#define arch_atomic_xchg_acquire(v, new) \
148 arch_xchg_acquire(&((v)->counter), (new))
149#define arch_atomic_xchg_release(v, new) \
150 arch_xchg_release(&((v)->counter), (new))
151#define arch_atomic_xchg(v, new) \
152 arch_xchg(&((v)->counter), (new))
153
154#define arch_atomic_cmpxchg_relaxed(v, old, new) \
155 arch_cmpxchg_relaxed(&((v)->counter), (old), (new))
156#define arch_atomic_cmpxchg_acquire(v, old, new) \
157 arch_cmpxchg_acquire(&((v)->counter), (old), (new))
158#define arch_atomic_cmpxchg_release(v, old, new) \
159 arch_cmpxchg_release(&((v)->counter), (old), (new))
160#define arch_atomic_cmpxchg(v, old, new) \
161 arch_cmpxchg(&((v)->counter), (old), (new))
162
163#define arch_atomic_andnot arch_atomic_andnot145#define arch_atomic_andnot arch_atomic_andnot
164 146
165/*147/*
@@ -209,16 +191,6 @@ static __always_inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 43 unmodified lines
209#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release191#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
210#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor192#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
211 193
212#define arch_atomic64_xchg_relaxed arch_atomic_xchg_relaxed
213#define arch_atomic64_xchg_acquire arch_atomic_xchg_acquire
214#define arch_atomic64_xchg_release arch_atomic_xchg_release
215#define arch_atomic64_xchg arch_atomic_xchg
216
217#define arch_atomic64_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed
218#define arch_atomic64_cmpxchg_acquire arch_atomic_cmpxchg_acquire
219#define arch_atomic64_cmpxchg_release arch_atomic_cmpxchg_release
220#define arch_atomic64_cmpxchg arch_atomic_cmpxchg
221
222#define arch_atomic64_andnot arch_atomic64_andnot194#define arch_atomic64_andnot arch_atomic64_andnot
223 195
224#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive196#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
arch/arm64/include/asm/atomic_ll_sc.h+32−24
@@ -294,38 +294,46 @@ __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) · 293 unmodified lines
294 294
295#undef __CMPXCHG_CASE295#undef __CMPXCHG_CASE
296 296
297#define __CMPXCHG_DBL(name, mb, rel, cl) \297union __u128_halves {
298static __always_inline long \298 u128 full;
299__ll_sc__cmpxchg_double##name(unsigned long old1, \299 struct {
300 unsigned long old2, \300 u64 low, high;
301 unsigned long new1, \301 };
302 unsigned long new2, \302};
303 volatile void *ptr) \303
304#define __CMPXCHG128(name, mb, rel, cl...) \
305static __always_inline u128 \
306__ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
304{ \307{ \
305 unsigned long tmp, ret; \308 union __u128_halves r, o = { .full = (old) }, \
309 n = { .full = (new) }; \
310 unsigned int tmp; \
306 \311 \
307 asm volatile("// __cmpxchg_double" #name "\n" \312 asm volatile("// __cmpxchg128" #name "\n" \
308 " prfm pstl1strm, %2\n" \313 " prfm pstl1strm, %[v]\n" \
309 "1: ldxp %0, %1, %2\n" \314 "1: ldxp %[rl], %[rh], %[v]\n" \
310 " eor %0, %0, %3\n" \315 " cmp %[rl], %[ol]\n" \
311 " eor %1, %1, %4\n" \316 " ccmp %[rh], %[oh], 0, eq\n" \
312 " orr %1, %0, %1\n" \317 " b.ne 2f\n" \
313 " cbnz %1, 2f\n" \318 " st" #rel "xp %w[tmp], %[nl], %[nh], %[v]\n" \
314 " st" #rel "xp %w0, %5, %6, %2\n" \319 " cbnz %w[tmp], 1b\n" \
315 " cbnz %w0, 1b\n" \
316 " " #mb "\n" \320 " " #mb "\n" \
317 "2:" \321 "2:" \
318 : "=&r" (tmp), "=&r" (ret), "+Q" (*(__uint128_t *)ptr) \322 : [v] "+Q" (*(u128 *)ptr), \
319 : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \323 [rl] "=&r" (r.low), [rh] "=&r" (r.high), \
320 : cl); \324 [tmp] "=&r" (tmp) \
325 : [ol] "r" (o.low), [oh] "r" (o.high), \
326 [nl] "r" (n.low), [nh] "r" (n.high) \
327 : "cc", ##cl); \
321 \328 \
322 return ret; \329 return r.full; \
323}330}
324 331
325__CMPXCHG_DBL( , , , )332__CMPXCHG128( , , )
326__CMPXCHG_DBL(_mb, dmb ish, l, "memory")333__CMPXCHG128(_mb, dmb ish, l, "memory")
334
335#undef __CMPXCHG128
327 336
328#undef __CMPXCHG_DBL
329#undef K337#undef K
330 338
331#endif /* __ASM_ATOMIC_LL_SC_H */339#endif /* __ASM_ATOMIC_LL_SC_H */
arch/arm64/include/asm/atomic_lse.h+17−22
@@ -281,40 +281,35 @@ __CMPXCHG_CASE(x, , mb_, 64, al, "memory") · 280 unmodified lines
281 281
282#undef __CMPXCHG_CASE282#undef __CMPXCHG_CASE
283 283
284#define __CMPXCHG_DBL(name, mb, cl...) \284#define __CMPXCHG128(name, mb, cl...) \
285static __always_inline long \285static __always_inline u128 \
286__lse__cmpxchg_double##name(unsigned long old1, \286__lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
287 unsigned long old2, \
288 unsigned long new1, \
289 unsigned long new2, \
290 volatile void *ptr) \
291{ \287{ \
292 unsigned long oldval1 = old1; \288 union __u128_halves r, o = { .full = (old) }, \
293 unsigned long oldval2 = old2; \289 n = { .full = (new) }; \
294 register unsigned long x0 asm ("x0") = old1; \290 register unsigned long x0 asm ("x0") = o.low; \
295 register unsigned long x1 asm ("x1") = old2; \291 register unsigned long x1 asm ("x1") = o.high; \
296 register unsigned long x2 asm ("x2") = new1; \292 register unsigned long x2 asm ("x2") = n.low; \
297 register unsigned long x3 asm ("x3") = new2; \293 register unsigned long x3 asm ("x3") = n.high; \
298 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \294 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
299 \295 \
300 asm volatile( \296 asm volatile( \
301 __LSE_PREAMBLE \297 __LSE_PREAMBLE \
302 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\298 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
303 " eor %[old1], %[old1], %[oldval1]\n" \
304 " eor %[old2], %[old2], %[oldval2]\n" \
305 " orr %[old1], %[old1], %[old2]" \
306 : [old1] "+&r" (x0), [old2] "+&r" (x1), \299 : [old1] "+&r" (x0), [old2] "+&r" (x1), \
307 [v] "+Q" (*(__uint128_t *)ptr) \300 [v] "+Q" (*(u128 *)ptr) \
308 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \301 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
309 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \302 [oldval1] "r" (o.low), [oldval2] "r" (o.high) \
310 : cl); \303 : cl); \
311 \304 \
312 return x0; \305 r.low = x0; r.high = x1; \
306 \
307 return r.full; \
313}308}
314 309
315__CMPXCHG_DBL( , )310__CMPXCHG128( , )
316__CMPXCHG_DBL(_mb, al, "memory")311__CMPXCHG128(_mb, al, "memory")
317 312
318#undef __CMPXCHG_DBL313#undef __CMPXCHG128
319 314
320#endif /* __ASM_ATOMIC_LSE_H */315#endif /* __ASM_ATOMIC_LSE_H */
arch/arm64/include/asm/cmpxchg.h+14−34
@@ -130,21 +130,18 @@ __CMPXCHG_CASE(mb_, 64) · 129 unmodified lines
130 130
131#undef __CMPXCHG_CASE131#undef __CMPXCHG_CASE
132 132
133#define __CMPXCHG_DBL(name) \133#define __CMPXCHG128(name) \
134static inline long __cmpxchg_double##name(unsigned long old1, \134static inline u128 __cmpxchg128##name(volatile u128 *ptr, \
135 unsigned long old2, \135 u128 old, u128 new) \
136 unsigned long new1, \
137 unsigned long new2, \
138 volatile void *ptr) \
139{ \136{ \
140 return __lse_ll_sc_body(_cmpxchg_double##name, \137 return __lse_ll_sc_body(_cmpxchg128##name, \
141 old1, old2, new1, new2, ptr); \138 ptr, old, new); \
142}139}
143 140
144__CMPXCHG_DBL( )141__CMPXCHG128( )
145__CMPXCHG_DBL(_mb)142__CMPXCHG128(_mb)
146 143
147#undef __CMPXCHG_DBL144#undef __CMPXCHG128
148 145
149#define __CMPXCHG_GEN(sfx) \146#define __CMPXCHG_GEN(sfx) \
150static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \147static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
@@ -198,34 +195,17 @@ __CMPXCHG_GEN(_mb) · 47 unmodified lines
198#define arch_cmpxchg64 arch_cmpxchg195#define arch_cmpxchg64 arch_cmpxchg
199#define arch_cmpxchg64_local arch_cmpxchg_local196#define arch_cmpxchg64_local arch_cmpxchg_local
200 197
201/* cmpxchg_double */198/* cmpxchg128 */
202#define system_has_cmpxchg_double() 1199#define system_has_cmpxchg128() 1
203
204#define __cmpxchg_double_check(ptr1, ptr2) \
205({ \
206 if (sizeof(*(ptr1)) != 8) \
207 BUILD_BUG(); \
208 VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
209})
210 200
211#define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \201#define arch_cmpxchg128(ptr, o, n) \
212({ \202({ \
213 int __ret; \203 __cmpxchg128_mb((ptr), (o), (n)); \
214 __cmpxchg_double_check(ptr1, ptr2); \
215 __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
216 (unsigned long)(n1), (unsigned long)(n2), \
217 ptr1); \
218 __ret; \
219})204})
220 205
221#define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \206#define arch_cmpxchg128_local(ptr, o, n) \
222({ \207({ \
223 int __ret; \208 __cmpxchg128((ptr), (o), (n)); \
224 __cmpxchg_double_check(ptr1, ptr2); \
225 __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
226 (unsigned long)(n1), (unsigned long)(n2), \
227 ptr1); \
228 __ret; \
229})209})
230 210
231#define __CMPWAIT_CASE(w, sfx, sz) \211#define __CMPWAIT_CASE(w, sfx, sz) \
arch/arm64/include/asm/percpu.h+20−10
@@ -140,17 +140,11 @@ PERCPU_RET_OP(add, add, ldadd) · 139 unmodified lines
140 * re-enabling preemption for preemptible kernels, but doing that in a way140 * re-enabling preemption for preemptible kernels, but doing that in a way
141 * which builds inside a module would mean messing directly with the preempt141 * which builds inside a module would mean messing directly with the preempt
142 * count. If you do this, peterz and tglx will hunt you down.142 * count. If you do this, peterz and tglx will hunt you down.
143 *
144 * Not to mention it'll break the actual preemption model for missing a
145 * preemption point when TIF_NEED_RESCHED gets set while preemption is
146 * disabled.
143 */147 */
144#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
145({ \
146 int __ret; \
147 preempt_disable_notrace(); \
148 __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
149 raw_cpu_ptr(&(ptr2)), \
150 o1, o2, n1, n2); \
151 preempt_enable_notrace(); \
152 __ret; \
153})
154 148
155#define _pcp_protect(op, pcp, ...) \149#define _pcp_protect(op, pcp, ...) \
156({ \150({ \
@@ -240,6 +234,22 @@ PERCPU_RET_OP(add, add, ldadd) · 83 unmodified lines
240#define this_cpu_cmpxchg_8(pcp, o, n) \234#define this_cpu_cmpxchg_8(pcp, o, n) \
241 _pcp_protect_return(cmpxchg_relaxed, pcp, o, n)235 _pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
242 236
237#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
238
239#define this_cpu_cmpxchg128(pcp, o, n) \
240({ \
241 typedef typeof(pcp) pcp_op_T__; \
242 u128 old__, new__, ret__; \
243 pcp_op_T__ *ptr__; \
244 old__ = o; \
245 new__ = n; \
246 preempt_disable_notrace(); \
247 ptr__ = raw_cpu_ptr(&(pcp)); \
248 ret__ = cmpxchg128_local((void *)ptr__, old__, new__); \
249 preempt_enable_notrace(); \
250 ret__; \
251})
252
243#ifdef __KVM_NVHE_HYPERVISOR__253#ifdef __KVM_NVHE_HYPERVISOR__
244extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);254extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
245#define __per_cpu_offset255#define __per_cpu_offset
arch/csky/include/asm/atomic.h+0−35
@@ -195,41 +195,6 @@ arch_atomic_dec_if_positive(atomic_t *v) · 194 unmodified lines
195}195}
196#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive196#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
197 197
198#define ATOMIC_OP() \
199static __always_inline \
200int arch_atomic_xchg_relaxed(atomic_t *v, int n) \
201{ \
202 return __xchg_relaxed(n, &(v->counter), 4); \
203} \
204static __always_inline \
205int arch_atomic_cmpxchg_relaxed(atomic_t *v, int o, int n) \
206{ \
207 return __cmpxchg_relaxed(&(v->counter), o, n, 4); \
208} \
209static __always_inline \
210int arch_atomic_cmpxchg_acquire(atomic_t *v, int o, int n) \
211{ \
212 return __cmpxchg_acquire(&(v->counter), o, n, 4); \
213} \
214static __always_inline \
215int arch_atomic_cmpxchg(atomic_t *v, int o, int n) \
216{ \
217 return __cmpxchg(&(v->counter), o, n, 4); \
218}
219
220#define ATOMIC_OPS() \
221 ATOMIC_OP()
222
223ATOMIC_OPS()
224
225#define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed
226#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed
227#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
228#define arch_atomic_cmpxchg arch_atomic_cmpxchg
229
230#undef ATOMIC_OPS
231#undef ATOMIC_OP
232
233#else198#else
234#include <asm-generic/atomic.h>199#include <asm-generic/atomic.h>
235#endif200#endif
arch/hexagon/include/asm/atomic.h+9−60
@@ -28,58 +28,8 @@ static inline void arch_atomic_set(atomic_t *v, int new) · 27 unmodified lines
28 28
29#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i))29#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i))
30 30
31/**
32 * arch_atomic_read - reads a word, atomically
33 * @v: pointer to atomic value
34 *
35 * Assumes all word reads on our architecture are atomic.
36 */
37#define arch_atomic_read(v) READ_ONCE((v)->counter)31#define arch_atomic_read(v) READ_ONCE((v)->counter)
38 32
39/**
40 * arch_atomic_xchg - atomic
41 * @v: pointer to memory to change
42 * @new: new value (technically passed in a register -- see xchg)
43 */
44#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
45
46
47/**
48 * arch_atomic_cmpxchg - atomic compare-and-exchange values
49 * @v: pointer to value to change
50 * @old: desired old value to match
51 * @new: new value to put in
52 *
53 * Parameters are then pointer, value-in-register, value-in-register,
54 * and the output is the old value.
55 *
56 * Apparently this is complicated for archs that don't support
57 * the memw_locked like we do (or it's broken or whatever).
58 *
59 * Kind of the lynchpin of the rest of the generically defined routines.
60 * Remember V2 had that bug with dotnew predicate set by memw_locked.
61 *
62 * "old" is "expected" old val, __oldval is actual old value
63 */
64static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
65{
66 int __oldval;
67
68 asm volatile(
69 "1: %0 = memw_locked(%1);\n"
70 " { P0 = cmp.eq(%0,%2);\n"
71 " if (!P0.new) jump:nt 2f; }\n"
72 " memw_locked(%1,P0) = %3;\n"
73 " if (!P0) jump 1b;\n"
74 "2:\n"
75 : "=&r" (__oldval)
76 : "r" (&v->counter), "r" (old), "r" (new)
77 : "memory", "p0"
78 );
79
80 return __oldval;
81}
82
83#define ATOMIC_OP(op) \33#define ATOMIC_OP(op) \
84static inline void arch_atomic_##op(int i, atomic_t *v) \34static inline void arch_atomic_##op(int i, atomic_t *v) \
85{ \35{ \
@@ -135,6 +85,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 49 unmodified lines
135ATOMIC_OPS(add)85ATOMIC_OPS(add)
136ATOMIC_OPS(sub)86ATOMIC_OPS(sub)
137 87
88#define arch_atomic_add_return arch_atomic_add_return
89#define arch_atomic_sub_return arch_atomic_sub_return
90#define arch_atomic_fetch_add arch_atomic_fetch_add
91#define arch_atomic_fetch_sub arch_atomic_fetch_sub
92
138#undef ATOMIC_OPS93#undef ATOMIC_OPS
139#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)94#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
140 95
@@ -142,21 +97,15 @@ ATOMIC_OPS(and) · 1 unmodified line
142ATOMIC_OPS(or)97ATOMIC_OPS(or)
143ATOMIC_OPS(xor)98ATOMIC_OPS(xor)
144 99
100#define arch_atomic_fetch_and arch_atomic_fetch_and
101#define arch_atomic_fetch_or arch_atomic_fetch_or
102#define arch_atomic_fetch_xor arch_atomic_fetch_xor
103
145#undef ATOMIC_OPS104#undef ATOMIC_OPS
146#undef ATOMIC_FETCH_OP105#undef ATOMIC_FETCH_OP
147#undef ATOMIC_OP_RETURN106#undef ATOMIC_OP_RETURN
148#undef ATOMIC_OP107#undef ATOMIC_OP
149 108
150/**
151 * arch_atomic_fetch_add_unless - add unless the number is a given value
152 * @v: pointer to value
153 * @a: amount to add
154 * @u: unless value is equal to u
155 *
156 * Returns old value.
157 *
158 */
159
160static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)109static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
161{110{
162 int __oldval;111 int __oldval;
arch/ia64/include/asm/atomic.h+0−7
@@ -207,13 +207,6 @@ ATOMIC64_FETCH_OP(xor, ^) · 206 unmodified lines
207#undef ATOMIC64_FETCH_OP207#undef ATOMIC64_FETCH_OP
208#undef ATOMIC64_OP208#undef ATOMIC64_OP
209 209
210#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), old, new))
211#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
212
213#define arch_atomic64_cmpxchg(v, old, new) \
214 (arch_cmpxchg(&((v)->counter), old, new))
215#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new))
216
217#define arch_atomic_add(i,v) (void)arch_atomic_add_return((i), (v))210#define arch_atomic_add(i,v) (void)arch_atomic_add_return((i), (v))
218#define arch_atomic_sub(i,v) (void)arch_atomic_sub_return((i), (v))211#define arch_atomic_sub(i,v) (void)arch_atomic_sub_return((i), (v))
219 212
arch/loongarch/include/asm/atomic.h+0−56
@@ -29,21 +29,7 @@ · 28 unmodified lines
29 29
30#define ATOMIC_INIT(i) { (i) }30#define ATOMIC_INIT(i) { (i) }
31 31
32/*
33 * arch_atomic_read - read atomic variable
34 * @v: pointer of type atomic_t
35 *
36 * Atomically reads the value of @v.
37 */
38#define arch_atomic_read(v) READ_ONCE((v)->counter)32#define arch_atomic_read(v) READ_ONCE((v)->counter)
39
40/*
41 * arch_atomic_set - set atomic variable
42 * @v: pointer of type atomic_t
43 * @i: required value
44 *
45 * Atomically sets the value of @v to @i.
46 */
47#define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i))33#define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i))
48 34
49#define ATOMIC_OP(op, I, asm_op) \35#define ATOMIC_OP(op, I, asm_op) \
@@ -139,14 +125,6 @@ static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) · 89 unmodified lines
139}125}
140#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless126#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
141 127
142/*
143 * arch_atomic_sub_if_positive - conditionally subtract integer from atomic variable
144 * @i: integer value to subtract
145 * @v: pointer of type atomic_t
146 *
147 * Atomically test @v and subtract @i if @v is greater or equal than @i.
148 * The function returns the old value of @v minus @i.
149 */
150static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)128static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
151{129{
152 int result;130 int result;
@@ -181,31 +159,13 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v) · 28 unmodified lines
181 return result;159 return result;
182}160}
183 161
184#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
185#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
186
187/*
188 * arch_atomic_dec_if_positive - decrement by 1 if old value positive
189 * @v: pointer of type atomic_t
190 */
191#define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v)162#define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v)
192 163
193#ifdef CONFIG_64BIT164#ifdef CONFIG_64BIT
194 165
195#define ATOMIC64_INIT(i) { (i) }166#define ATOMIC64_INIT(i) { (i) }
196 167
197/*
198 * arch_atomic64_read - read atomic variable
199 * @v: pointer of type atomic64_t
200 *
201 */
202#define arch_atomic64_read(v) READ_ONCE((v)->counter)168#define arch_atomic64_read(v) READ_ONCE((v)->counter)
203
204/*
205 * arch_atomic64_set - set atomic variable
206 * @v: pointer of type atomic64_t
207 * @i: required value
208 */
209#define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))169#define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
210 170
211#define ATOMIC64_OP(op, I, asm_op) \171#define ATOMIC64_OP(op, I, asm_op) \
@@ -300,14 +260,6 @@ static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u) · 88 unmodified lines
300}260}
301#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless261#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
302 262
303/*
304 * arch_atomic64_sub_if_positive - conditionally subtract integer from atomic variable
305 * @i: integer value to subtract
306 * @v: pointer of type atomic64_t
307 *
308 * Atomically test @v and subtract @i if @v is greater or equal than @i.
309 * The function returns the old value of @v minus @i.
310 */
311static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)263static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
312{264{
313 long result;265 long result;
@@ -342,14 +294,6 @@ static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v) · 28 unmodified lines
342 return result;294 return result;
343}295}
344 296
345#define arch_atomic64_cmpxchg(v, o, n) \
346 ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
347#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
348
349/*
350 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
351 * @v: pointer of type atomic64_t
352 */
353#define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v)297#define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v)
354 298
355#endif /* CONFIG_64BIT */299#endif /* CONFIG_64BIT */
arch/m68k/include/asm/atomic.h+12−6
@@ -106,6 +106,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \ · 105 unmodified lines
106ATOMIC_OPS(add, +=, add)106ATOMIC_OPS(add, +=, add)
107ATOMIC_OPS(sub, -=, sub)107ATOMIC_OPS(sub, -=, sub)
108 108
109#define arch_atomic_add_return arch_atomic_add_return
110#define arch_atomic_sub_return arch_atomic_sub_return
111#define arch_atomic_fetch_add arch_atomic_fetch_add
112#define arch_atomic_fetch_sub arch_atomic_fetch_sub
113
109#undef ATOMIC_OPS114#undef ATOMIC_OPS
110#define ATOMIC_OPS(op, c_op, asm_op) \115#define ATOMIC_OPS(op, c_op, asm_op) \
111 ATOMIC_OP(op, c_op, asm_op) \116 ATOMIC_OP(op, c_op, asm_op) \
@@ -115,6 +120,10 @@ ATOMIC_OPS(and, &=, and) · 3 unmodified lines
115ATOMIC_OPS(or, |=, or)120ATOMIC_OPS(or, |=, or)
116ATOMIC_OPS(xor, ^=, eor)121ATOMIC_OPS(xor, ^=, eor)
117 122
123#define arch_atomic_fetch_and arch_atomic_fetch_and
124#define arch_atomic_fetch_or arch_atomic_fetch_or
125#define arch_atomic_fetch_xor arch_atomic_fetch_xor
126
118#undef ATOMIC_OPS127#undef ATOMIC_OPS
119#undef ATOMIC_FETCH_OP128#undef ATOMIC_FETCH_OP
120#undef ATOMIC_OP_RETURN129#undef ATOMIC_OP_RETURN
@@ -158,12 +167,7 @@ static inline int arch_atomic_inc_and_test(atomic_t *v) · 37 unmodified lines
158}167}
159#define arch_atomic_inc_and_test arch_atomic_inc_and_test168#define arch_atomic_inc_and_test arch_atomic_inc_and_test
160 169
161#ifdef CONFIG_RMW_INSNS170#ifndef CONFIG_RMW_INSNS
162
163#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n)))
164#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
165
166#else /* !CONFIG_RMW_INSNS */
167 171
168static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)172static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
169{173{
@@ -177,6 +181,7 @@ static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) · 7 unmodified lines
177 local_irq_restore(flags);181 local_irq_restore(flags);
178 return prev;182 return prev;
179}183}
184#define arch_atomic_cmpxchg arch_atomic_cmpxchg
180 185
181static inline int arch_atomic_xchg(atomic_t *v, int new)186static inline int arch_atomic_xchg(atomic_t *v, int new)
182{187{
@@ -189,6 +194,7 @@ static inline int arch_atomic_xchg(atomic_t *v, int new) · 6 unmodified lines
189 local_irq_restore(flags);194 local_irq_restore(flags);
190 return prev;195 return prev;
191}196}
197#define arch_atomic_xchg arch_atomic_xchg
192 198
193#endif /* !CONFIG_RMW_INSNS */199#endif /* !CONFIG_RMW_INSNS */
194 200
arch/mips/include/asm/atomic.h+0−11
@@ -33,17 +33,6 @@ static __always_inline void arch_##pfx##_set(pfx##_t *v, type i) \ · 32 unmodified lines
33{ \33{ \
34 WRITE_ONCE(v->counter, i); \34 WRITE_ONCE(v->counter, i); \
35} \35} \
36 \
37static __always_inline type \
38arch_##pfx##_cmpxchg(pfx##_t *v, type o, type n) \
39{ \
40 return arch_cmpxchg(&v->counter, o, n); \
41} \
42 \
43static __always_inline type arch_##pfx##_xchg(pfx##_t *v, type n) \
44{ \
45 return arch_xchg(&v->counter, n); \
46}
47 36
48ATOMIC_OPS(atomic, int)37ATOMIC_OPS(atomic, int)
49 38
arch/openrisc/include/asm/atomic.h+0−3
@@ -130,7 +130,4 @@ static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) · 129 unmodified lines
130 130
131#include <asm/cmpxchg.h>131#include <asm/cmpxchg.h>
132 132
133#define arch_atomic_xchg(ptr, v) (arch_xchg(&(ptr)->counter, (v)))
134#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), (old), (new)))
135
136#endif /* __ASM_OPENRISC_ATOMIC_H */133#endif /* __ASM_OPENRISC_ATOMIC_H */
arch/parisc/include/asm/atomic.h+18−9
@@ -73,10 +73,6 @@ static __inline__ int arch_atomic_read(const atomic_t *v) · 72 unmodified lines
73 return READ_ONCE((v)->counter);73 return READ_ONCE((v)->counter);
74}74}
75 75
76/* exported interface */
77#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
78#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
79
80#define ATOMIC_OP(op, c_op) \76#define ATOMIC_OP(op, c_op) \
81static __inline__ void arch_atomic_##op(int i, atomic_t *v) \77static __inline__ void arch_atomic_##op(int i, atomic_t *v) \
82{ \78{ \
@@ -122,6 +118,11 @@ static __inline__ int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 39 unmodified lines
122ATOMIC_OPS(add, +=)118ATOMIC_OPS(add, +=)
123ATOMIC_OPS(sub, -=)119ATOMIC_OPS(sub, -=)
124 120
121#define arch_atomic_add_return arch_atomic_add_return
122#define arch_atomic_sub_return arch_atomic_sub_return
123#define arch_atomic_fetch_add arch_atomic_fetch_add
124#define arch_atomic_fetch_sub arch_atomic_fetch_sub
125
125#undef ATOMIC_OPS126#undef ATOMIC_OPS
126#define ATOMIC_OPS(op, c_op) \127#define ATOMIC_OPS(op, c_op) \
127 ATOMIC_OP(op, c_op) \128 ATOMIC_OP(op, c_op) \
@@ -131,6 +132,10 @@ ATOMIC_OPS(and, &=) · 3 unmodified lines
131ATOMIC_OPS(or, |=)132ATOMIC_OPS(or, |=)
132ATOMIC_OPS(xor, ^=)133ATOMIC_OPS(xor, ^=)
133 134
135#define arch_atomic_fetch_and arch_atomic_fetch_and
136#define arch_atomic_fetch_or arch_atomic_fetch_or
137#define arch_atomic_fetch_xor arch_atomic_fetch_xor
138
134#undef ATOMIC_OPS139#undef ATOMIC_OPS
135#undef ATOMIC_FETCH_OP140#undef ATOMIC_FETCH_OP
136#undef ATOMIC_OP_RETURN141#undef ATOMIC_OP_RETURN
@@ -185,6 +190,11 @@ static __inline__ s64 arch_atomic64_fetch_##op(s64 i, atomic64_t *v) \ · 48 unmodified lines
185ATOMIC64_OPS(add, +=)190ATOMIC64_OPS(add, +=)
186ATOMIC64_OPS(sub, -=)191ATOMIC64_OPS(sub, -=)
187 192
193#define arch_atomic64_add_return arch_atomic64_add_return
194#define arch_atomic64_sub_return arch_atomic64_sub_return
195#define arch_atomic64_fetch_add arch_atomic64_fetch_add
196#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
197
188#undef ATOMIC64_OPS198#undef ATOMIC64_OPS
189#define ATOMIC64_OPS(op, c_op) \199#define ATOMIC64_OPS(op, c_op) \
190 ATOMIC64_OP(op, c_op) \200 ATOMIC64_OP(op, c_op) \
@@ -194,6 +204,10 @@ ATOMIC64_OPS(and, &=) · 3 unmodified lines
194ATOMIC64_OPS(or, |=)204ATOMIC64_OPS(or, |=)
195ATOMIC64_OPS(xor, ^=)205ATOMIC64_OPS(xor, ^=)
196 206
207#define arch_atomic64_fetch_and arch_atomic64_fetch_and
208#define arch_atomic64_fetch_or arch_atomic64_fetch_or
209#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
210
197#undef ATOMIC64_OPS211#undef ATOMIC64_OPS
198#undef ATOMIC64_FETCH_OP212#undef ATOMIC64_FETCH_OP
199#undef ATOMIC64_OP_RETURN213#undef ATOMIC64_OP_RETURN
@@ -218,11 +232,6 @@ arch_atomic64_read(const atomic64_t *v) · 18 unmodified lines
218 return READ_ONCE((v)->counter);232 return READ_ONCE((v)->counter);
219}233}
220 234
221/* exported interface */
222#define arch_atomic64_cmpxchg(v, o, n) \
223 ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
224#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new))
225
226#endif /* !CONFIG_64BIT */235#endif /* !CONFIG_64BIT */
227 236
228 237
arch/powerpc/include/asm/atomic.h+0−24
@@ -126,18 +126,6 @@ ATOMIC_OPS(xor, xor, "", K) · 125 unmodified lines
126#undef ATOMIC_OP_RETURN_RELAXED126#undef ATOMIC_OP_RETURN_RELAXED
127#undef ATOMIC_OP127#undef ATOMIC_OP
128 128
129#define arch_atomic_cmpxchg(v, o, n) \
130 (arch_cmpxchg(&((v)->counter), (o), (n)))
131#define arch_atomic_cmpxchg_relaxed(v, o, n) \
132 arch_cmpxchg_relaxed(&((v)->counter), (o), (n))
133#define arch_atomic_cmpxchg_acquire(v, o, n) \
134 arch_cmpxchg_acquire(&((v)->counter), (o), (n))
135
136#define arch_atomic_xchg(v, new) \
137 (arch_xchg(&((v)->counter), new))
138#define arch_atomic_xchg_relaxed(v, new) \
139 arch_xchg_relaxed(&((v)->counter), (new))
140
141/**129/**
142 * atomic_fetch_add_unless - add unless the number is a given value130 * atomic_fetch_add_unless - add unless the number is a given value
143 * @v: pointer of type atomic_t131 * @v: pointer of type atomic_t
@@ -396,18 +384,6 @@ static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v) · 252 unmodified lines
396}384}
397#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive385#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
398 386
399#define arch_atomic64_cmpxchg(v, o, n) \
400 (arch_cmpxchg(&((v)->counter), (o), (n)))
401#define arch_atomic64_cmpxchg_relaxed(v, o, n) \
402 arch_cmpxchg_relaxed(&((v)->counter), (o), (n))
403#define arch_atomic64_cmpxchg_acquire(v, o, n) \
404 arch_cmpxchg_acquire(&((v)->counter), (o), (n))
405
406#define arch_atomic64_xchg(v, new) \
407 (arch_xchg(&((v)->counter), new))
408#define arch_atomic64_xchg_relaxed(v, new) \
409 arch_xchg_relaxed(&((v)->counter), (new))
410
411/**387/**
412 * atomic64_fetch_add_unless - add unless the number is a given value388 * atomic64_fetch_add_unless - add unless the number is a given value
413 * @v: pointer of type atomic64_t389 * @v: pointer of type atomic64_t
arch/powerpc/kernel/smp.c+6−6
@@ -417,9 +417,9 @@ noinstr static void nmi_ipi_lock_start(unsigned long *flags) · 416 unmodified lines
417{417{
418 raw_local_irq_save(*flags);418 raw_local_irq_save(*flags);
419 hard_irq_disable();419 hard_irq_disable();
420 while (arch_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {420 while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
421 raw_local_irq_restore(*flags);421 raw_local_irq_restore(*flags);
422 spin_until_cond(arch_atomic_read(&__nmi_ipi_lock) == 0);422 spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0);
423 raw_local_irq_save(*flags);423 raw_local_irq_save(*flags);
424 hard_irq_disable();424 hard_irq_disable();
425 }425 }
@@ -427,15 +427,15 @@ noinstr static void nmi_ipi_lock_start(unsigned long *flags) · 1 unmodified line
427 427
428noinstr static void nmi_ipi_lock(void)428noinstr static void nmi_ipi_lock(void)
429{429{
430 while (arch_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)430 while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
431 spin_until_cond(arch_atomic_read(&__nmi_ipi_lock) == 0);431 spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0);
432}432}
433 433
434noinstr static void nmi_ipi_unlock(void)434noinstr static void nmi_ipi_unlock(void)
435{435{
436 smp_mb();436 smp_mb();
437 WARN_ON(arch_atomic_read(&__nmi_ipi_lock) != 1);437 WARN_ON(raw_atomic_read(&__nmi_ipi_lock) != 1);
438 arch_atomic_set(&__nmi_ipi_lock, 0);438 raw_atomic_set(&__nmi_ipi_lock, 0);
439}439}
440 440
441noinstr static void nmi_ipi_unlock_end(unsigned long *flags)441noinstr static void nmi_ipi_unlock_end(unsigned long *flags)
arch/riscv/include/asm/atomic.h+0−72
@@ -238,78 +238,6 @@ static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, · 237 unmodified lines
238#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless238#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
239#endif239#endif
240 240
241/*
242 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
243 * {cmp,}xchg and the operations that return, so they need a full barrier.
244 */
245#define ATOMIC_OP(c_t, prefix, size) \
246static __always_inline \
247c_t arch_atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \
248{ \
249 return __xchg_relaxed(&(v->counter), n, size); \
250} \
251static __always_inline \
252c_t arch_atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \
253{ \
254 return __xchg_acquire(&(v->counter), n, size); \
255} \
256static __always_inline \
257c_t arch_atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \
258{ \
259 return __xchg_release(&(v->counter), n, size); \
260} \
261static __always_inline \
262c_t arch_atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \
263{ \
264 return __arch_xchg(&(v->counter), n, size); \
265} \
266static __always_inline \
267c_t arch_atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \
268 c_t o, c_t n) \
269{ \
270 return __cmpxchg_relaxed(&(v->counter), o, n, size); \
271} \
272static __always_inline \
273c_t arch_atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \
274 c_t o, c_t n) \
275{ \
276 return __cmpxchg_acquire(&(v->counter), o, n, size); \
277} \
278static __always_inline \
279c_t arch_atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \
280 c_t o, c_t n) \
281{ \
282 return __cmpxchg_release(&(v->counter), o, n, size); \
283} \
284static __always_inline \
285c_t arch_atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \
286{ \
287 return __cmpxchg(&(v->counter), o, n, size); \
288}
289
290#ifdef CONFIG_GENERIC_ATOMIC64
291#define ATOMIC_OPS() \
292 ATOMIC_OP(int, , 4)
293#else
294#define ATOMIC_OPS() \
295 ATOMIC_OP(int, , 4) \
296 ATOMIC_OP(s64, 64, 8)
297#endif
298
299ATOMIC_OPS()
300
301#define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed
302#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
303#define arch_atomic_xchg_release arch_atomic_xchg_release
304#define arch_atomic_xchg arch_atomic_xchg
305#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed
306#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
307#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
308#define arch_atomic_cmpxchg arch_atomic_cmpxchg
309
310#undef ATOMIC_OPS
311#undef ATOMIC_OP
312
313static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)241static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
314{242{
315 int prev, rc;243 int prev, rc;
arch/s390/include/asm/cmpxchg.h+6−26
@@ -190,38 +190,18 @@ static __always_inline unsigned long __cmpxchg(unsigned long address, · 189 unmodified lines
190#define arch_cmpxchg_local arch_cmpxchg190#define arch_cmpxchg_local arch_cmpxchg
191#define arch_cmpxchg64_local arch_cmpxchg191#define arch_cmpxchg64_local arch_cmpxchg
192 192
193#define system_has_cmpxchg_double() 1193#define system_has_cmpxchg128() 1
194 194
195static __always_inline int __cmpxchg_double(unsigned long p1, unsigned long p2,195static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
196 unsigned long o1, unsigned long o2,
197 unsigned long n1, unsigned long n2)
198{196{
199 union register_pair old = { .even = o1, .odd = o2, };
200 union register_pair new = { .even = n1, .odd = n2, };
201 int cc;
202
203 asm volatile(197 asm volatile(
204 " cdsg %[old],%[new],%[ptr]\n"198 " cdsg %[old],%[new],%[ptr]\n"
205 " ipm %[cc]\n"199 : [old] "+d" (old), [ptr] "+QS" (*ptr)
206 " srl %[cc],28\n"200 : [new] "d" (new)
207 : [cc] "=&d" (cc), [old] "+&d" (old.pair)
208 : [new] "d" (new.pair),
209 [ptr] "QS" (*(unsigned long *)p1), "Q" (*(unsigned long *)p2)
210 : "memory", "cc");201 : "memory", "cc");
211 return !cc;202 return old;
212}203}
213 204
214#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \205#define arch_cmpxchg128 arch_cmpxchg128
215({ \
216 typeof(p1) __p1 = (p1); \
217 typeof(p2) __p2 = (p2); \
218 \
219 BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
220 BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
221 VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\
222 __cmpxchg_double((unsigned long)__p1, (unsigned long)__p2, \
223 (unsigned long)(o1), (unsigned long)(o2), \
224 (unsigned long)(n1), (unsigned long)(n2)); \
225})
226 206
227#endif /* __ASM_CMPXCHG_H */207#endif /* __ASM_CMPXCHG_H */
arch/s390/include/asm/cpu_mf.h+1−1
@@ -140,7 +140,7 @@ union hws_trailer_header { · 139 unmodified lines
140 unsigned int dsdes:16; /* 48-63: size of diagnostic SDE */140 unsigned int dsdes:16; /* 48-63: size of diagnostic SDE */
141 unsigned long long overflow; /* 64 - Overflow Count */141 unsigned long long overflow; /* 64 - Overflow Count */
142 };142 };
143 __uint128_t val;143 u128 val;
144};144};
145 145
146struct hws_trailer_entry {146struct hws_trailer_entry {
arch/s390/include/asm/percpu.h+16−18
@@ -148,6 +148,22 @@ · 147 unmodified lines
148#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)148#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)149#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
150 150
151#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
152
153#define this_cpu_cmpxchg128(pcp, oval, nval) \
154({ \
155 typedef typeof(pcp) pcp_op_T__; \
156 u128 old__, new__, ret__; \
157 pcp_op_T__ *ptr__; \
158 old__ = oval; \
159 new__ = nval; \
160 preempt_disable_notrace(); \
161 ptr__ = raw_cpu_ptr(&(pcp)); \
162 ret__ = cmpxchg128((void *)ptr__, old__, new__); \
163 preempt_enable_notrace(); \
164 ret__; \
165})
166
151#define arch_this_cpu_xchg(pcp, nval) \167#define arch_this_cpu_xchg(pcp, nval) \
152({ \168({ \
153 typeof(pcp) *ptr__; \169 typeof(pcp) *ptr__; \
@@ -164,24 +180,6 @@ · 10 unmodified lines
164#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)180#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
165#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)181#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
166 182
167#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \
168({ \
169 typeof(pcp1) *p1__; \
170 typeof(pcp2) *p2__; \
171 int ret__; \
172 \
173 preempt_disable_notrace(); \
174 p1__ = raw_cpu_ptr(&(pcp1)); \
175 p2__ = raw_cpu_ptr(&(pcp2)); \
176 ret__ = __cmpxchg_double((unsigned long)p1__, (unsigned long)p2__, \
177 (unsigned long)(o1), (unsigned long)(o2), \
178 (unsigned long)(n1), (unsigned long)(n2)); \
179 preempt_enable_notrace(); \
180 ret__; \
181})
182
183#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
184
185#include <asm-generic/percpu.h>183#include <asm-generic/percpu.h>
186 184
187#endif /* __ARCH_S390_PERCPU__ */185#endif /* __ARCH_S390_PERCPU__ */
arch/s390/kernel/perf_cpum_sf.c+3−13
@@ -1271,16 +1271,6 @@ static void hw_collect_samples(struct perf_event *event, unsigned long *sdbt, · 1270 unmodified lines
1271 }1271 }
1272}1272}
1273 1273
1274static inline __uint128_t __cdsg(__uint128_t *ptr, __uint128_t old, __uint128_t new)
1275{
1276 asm volatile(
1277 " cdsg %[old],%[new],%[ptr]\n"
1278 : [old] "+d" (old), [ptr] "+QS" (*ptr)
1279 : [new] "d" (new)
1280 : "memory", "cc");
1281 return old;
1282}
1283
1284/* hw_perf_event_update() - Process sampling buffer1274/* hw_perf_event_update() - Process sampling buffer
1285 * @event: The perf event1275 * @event: The perf event
1286 * @flush_all: Flag to also flush partially filled sample-data-blocks1276 * @flush_all: Flag to also flush partially filled sample-data-blocks
@@ -1352,7 +1342,7 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) · 65 unmodified lines
1352 new.f = 0;1342 new.f = 0;
1353 new.a = 1;1343 new.a = 1;
1354 new.overflow = 0;1344 new.overflow = 0;
1355 prev.val = __cdsg(&te->header.val, old.val, new.val);1345 prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1356 } while (prev.val != old.val);1346 } while (prev.val != old.val);
1357 1347
1358 /* Advance to next sample-data-block */1348 /* Advance to next sample-data-block */
@@ -1562,7 +1552,7 @@ static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index, · 203 unmodified lines
1562 }1552 }
1563 new.a = 1;1553 new.a = 1;
1564 new.overflow = 0;1554 new.overflow = 0;
1565 prev.val = __cdsg(&te->header.val, old.val, new.val);1555 prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1566 } while (prev.val != old.val);1556 } while (prev.val != old.val);
1567 return true;1557 return true;
1568}1558}
@@ -1636,7 +1626,7 @@ static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range, · 67 unmodified lines
1636 new.a = 1;1626 new.a = 1;
1637 else1627 else
1638 new.a = 0;1628 new.a = 0;
1639 prev.val = __cdsg(&te->header.val, old.val, new.val);1629 prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1640 } while (prev.val != old.val);1630 } while (prev.val != old.val);
1641 *overflow += orig_overflow;1631 *overflow += orig_overflow;
1642 }1632 }
arch/sh/include/asm/atomic-grb.h+9−0
@@ -71,6 +71,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 70 unmodified lines
71ATOMIC_OPS(add)71ATOMIC_OPS(add)
72ATOMIC_OPS(sub)72ATOMIC_OPS(sub)
73 73
74#define arch_atomic_add_return arch_atomic_add_return
75#define arch_atomic_sub_return arch_atomic_sub_return
76#define arch_atomic_fetch_add arch_atomic_fetch_add
77#define arch_atomic_fetch_sub arch_atomic_fetch_sub
78
74#undef ATOMIC_OPS79#undef ATOMIC_OPS
75#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)80#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
76 81
@@ -78,6 +83,10 @@ ATOMIC_OPS(and) · 1 unmodified line
78ATOMIC_OPS(or)83ATOMIC_OPS(or)
79ATOMIC_OPS(xor)84ATOMIC_OPS(xor)
80 85
86#define arch_atomic_fetch_and arch_atomic_fetch_and
87#define arch_atomic_fetch_or arch_atomic_fetch_or
88#define arch_atomic_fetch_xor arch_atomic_fetch_xor
89
81#undef ATOMIC_OPS90#undef ATOMIC_OPS
82#undef ATOMIC_FETCH_OP91#undef ATOMIC_FETCH_OP
83#undef ATOMIC_OP_RETURN92#undef ATOMIC_OP_RETURN
arch/sh/include/asm/atomic-irq.h+9−0
@@ -55,6 +55,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 54 unmodified lines
55ATOMIC_OPS(add, +=)55ATOMIC_OPS(add, +=)
56ATOMIC_OPS(sub, -=)56ATOMIC_OPS(sub, -=)
57 57
58#define arch_atomic_add_return arch_atomic_add_return
59#define arch_atomic_sub_return arch_atomic_sub_return
60#define arch_atomic_fetch_add arch_atomic_fetch_add
61#define arch_atomic_fetch_sub arch_atomic_fetch_sub
62
58#undef ATOMIC_OPS63#undef ATOMIC_OPS
59#define ATOMIC_OPS(op, c_op) \64#define ATOMIC_OPS(op, c_op) \
60 ATOMIC_OP(op, c_op) \65 ATOMIC_OP(op, c_op) \
@@ -64,6 +69,10 @@ ATOMIC_OPS(and, &=) · 3 unmodified lines
64ATOMIC_OPS(or, |=)69ATOMIC_OPS(or, |=)
65ATOMIC_OPS(xor, ^=)70ATOMIC_OPS(xor, ^=)
66 71
72#define arch_atomic_fetch_and arch_atomic_fetch_and
73#define arch_atomic_fetch_or arch_atomic_fetch_or
74#define arch_atomic_fetch_xor arch_atomic_fetch_xor
75
67#undef ATOMIC_OPS76#undef ATOMIC_OPS
68#undef ATOMIC_FETCH_OP77#undef ATOMIC_FETCH_OP
69#undef ATOMIC_OP_RETURN78#undef ATOMIC_OP_RETURN
arch/sh/include/asm/atomic-llsc.h+9−0
@@ -73,6 +73,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 72 unmodified lines
73ATOMIC_OPS(add)73ATOMIC_OPS(add)
74ATOMIC_OPS(sub)74ATOMIC_OPS(sub)
75 75
76#define arch_atomic_add_return arch_atomic_add_return
77#define arch_atomic_sub_return arch_atomic_sub_return
78#define arch_atomic_fetch_add arch_atomic_fetch_add
79#define arch_atomic_fetch_sub arch_atomic_fetch_sub
80
76#undef ATOMIC_OPS81#undef ATOMIC_OPS
77#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)82#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
78 83
@@ -80,6 +85,10 @@ ATOMIC_OPS(and) · 1 unmodified line
80ATOMIC_OPS(or)85ATOMIC_OPS(or)
81ATOMIC_OPS(xor)86ATOMIC_OPS(xor)
82 87
88#define arch_atomic_fetch_and arch_atomic_fetch_and
89#define arch_atomic_fetch_or arch_atomic_fetch_or
90#define arch_atomic_fetch_xor arch_atomic_fetch_xor
91
83#undef ATOMIC_OPS92#undef ATOMIC_OPS
84#undef ATOMIC_FETCH_OP93#undef ATOMIC_FETCH_OP
85#undef ATOMIC_OP_RETURN94#undef ATOMIC_OP_RETURN
arch/sh/include/asm/atomic.h+0−3
@@ -30,9 +30,6 @@ · 29 unmodified lines
30#include <asm/atomic-irq.h>30#include <asm/atomic-irq.h>
31#endif31#endif
32 32
33#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
34#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
35
36#endif /* CONFIG_CPU_J2 */33#endif /* CONFIG_CPU_J2 */
37 34
38#endif /* __ASM_SH_ATOMIC_H */35#endif /* __ASM_SH_ATOMIC_H */
arch/sparc/include/asm/atomic_32.h+16−2
@@ -19,17 +19,31 @@ · 18 unmodified lines
19#include <asm-generic/atomic64.h>19#include <asm-generic/atomic64.h>
20 20
21int arch_atomic_add_return(int, atomic_t *);21int arch_atomic_add_return(int, atomic_t *);
22#define arch_atomic_add_return arch_atomic_add_return
23
22int arch_atomic_fetch_add(int, atomic_t *);24int arch_atomic_fetch_add(int, atomic_t *);
25#define arch_atomic_fetch_add arch_atomic_fetch_add
26
23int arch_atomic_fetch_and(int, atomic_t *);27int arch_atomic_fetch_and(int, atomic_t *);
28#define arch_atomic_fetch_and arch_atomic_fetch_and
29
24int arch_atomic_fetch_or(int, atomic_t *);30int arch_atomic_fetch_or(int, atomic_t *);
31#define arch_atomic_fetch_or arch_atomic_fetch_or
32
25int arch_atomic_fetch_xor(int, atomic_t *);33int arch_atomic_fetch_xor(int, atomic_t *);
34#define arch_atomic_fetch_xor arch_atomic_fetch_xor
35
26int arch_atomic_cmpxchg(atomic_t *, int, int);36int arch_atomic_cmpxchg(atomic_t *, int, int);
37#define arch_atomic_cmpxchg arch_atomic_cmpxchg
38
27int arch_atomic_xchg(atomic_t *, int);39int arch_atomic_xchg(atomic_t *, int);
28int arch_atomic_fetch_add_unless(atomic_t *, int, int);40#define arch_atomic_xchg arch_atomic_xchg
29void arch_atomic_set(atomic_t *, int);
30 41
42int arch_atomic_fetch_add_unless(atomic_t *, int, int);
31#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless43#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
32 44
45void arch_atomic_set(atomic_t *, int);
46
33#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i))47#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i))
34 48
35#define arch_atomic_read(v) READ_ONCE((v)->counter)49#define arch_atomic_read(v) READ_ONCE((v)->counter)
arch/sparc/include/asm/atomic_64.h+18−11
@@ -37,6 +37,16 @@ s64 arch_atomic64_fetch_##op(s64, atomic64_t *); · 36 unmodified lines
37ATOMIC_OPS(add)37ATOMIC_OPS(add)
38ATOMIC_OPS(sub)38ATOMIC_OPS(sub)
39 39
40#define arch_atomic_add_return arch_atomic_add_return
41#define arch_atomic_sub_return arch_atomic_sub_return
42#define arch_atomic_fetch_add arch_atomic_fetch_add
43#define arch_atomic_fetch_sub arch_atomic_fetch_sub
44
45#define arch_atomic64_add_return arch_atomic64_add_return
46#define arch_atomic64_sub_return arch_atomic64_sub_return
47#define arch_atomic64_fetch_add arch_atomic64_fetch_add
48#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
49
40#undef ATOMIC_OPS50#undef ATOMIC_OPS
41#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)51#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
42 52
@@ -44,22 +54,19 @@ ATOMIC_OPS(and) · 1 unmodified line
44ATOMIC_OPS(or)54ATOMIC_OPS(or)
45ATOMIC_OPS(xor)55ATOMIC_OPS(xor)
46 56
57#define arch_atomic_fetch_and arch_atomic_fetch_and
58#define arch_atomic_fetch_or arch_atomic_fetch_or
59#define arch_atomic_fetch_xor arch_atomic_fetch_xor
60
61#define arch_atomic64_fetch_and arch_atomic64_fetch_and
62#define arch_atomic64_fetch_or arch_atomic64_fetch_or
63#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
64
47#undef ATOMIC_OPS65#undef ATOMIC_OPS
48#undef ATOMIC_FETCH_OP66#undef ATOMIC_FETCH_OP
49#undef ATOMIC_OP_RETURN67#undef ATOMIC_OP_RETURN
50#undef ATOMIC_OP68#undef ATOMIC_OP
51 69
52#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
53
54static inline int arch_atomic_xchg(atomic_t *v, int new)
55{
56 return arch_xchg(&v->counter, new);
57}
58
59#define arch_atomic64_cmpxchg(v, o, n) \
60 ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
61#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new))
62
63s64 arch_atomic64_dec_if_positive(atomic64_t *v);70s64 arch_atomic64_dec_if_positive(atomic64_t *v);
64#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive71#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
65 72
arch/x86/include/asm/atomic.h+0−87
@@ -14,12 +14,6 @@ · 13 unmodified lines
14 * resource counting etc..14 * resource counting etc..
15 */15 */
16 16
17/**
18 * arch_atomic_read - read atomic variable
19 * @v: pointer of type atomic_t
20 *
21 * Atomically reads the value of @v.
22 */
23static __always_inline int arch_atomic_read(const atomic_t *v)17static __always_inline int arch_atomic_read(const atomic_t *v)
24{18{
25 /*19 /*
@@ -29,25 +23,11 @@ static __always_inline int arch_atomic_read(const atomic_t *v) · 3 unmodified lines
29 return __READ_ONCE((v)->counter);23 return __READ_ONCE((v)->counter);
30}24}
31 25
32/**
33 * arch_atomic_set - set atomic variable
34 * @v: pointer of type atomic_t
35 * @i: required value
36 *
37 * Atomically sets the value of @v to @i.
38 */
39static __always_inline void arch_atomic_set(atomic_t *v, int i)26static __always_inline void arch_atomic_set(atomic_t *v, int i)
40{27{
41 __WRITE_ONCE(v->counter, i);28 __WRITE_ONCE(v->counter, i);
42}29}
43 30
44/**
45 * arch_atomic_add - add integer to atomic variable
46 * @i: integer value to add
47 * @v: pointer of type atomic_t
48 *
49 * Atomically adds @i to @v.
50 */
51static __always_inline void arch_atomic_add(int i, atomic_t *v)31static __always_inline void arch_atomic_add(int i, atomic_t *v)
52{32{
53 asm volatile(LOCK_PREFIX "addl %1,%0"33 asm volatile(LOCK_PREFIX "addl %1,%0"
@@ -55,13 +35,6 @@ static __always_inline void arch_atomic_add(int i, atomic_t *v) · 1 unmodified line
55 : "ir" (i) : "memory");35 : "ir" (i) : "memory");
56}36}
57 37
58/**
59 * arch_atomic_sub - subtract integer from atomic variable
60 * @i: integer value to subtract
61 * @v: pointer of type atomic_t
62 *
63 * Atomically subtracts @i from @v.
64 */
65static __always_inline void arch_atomic_sub(int i, atomic_t *v)38static __always_inline void arch_atomic_sub(int i, atomic_t *v)
66{39{
67 asm volatile(LOCK_PREFIX "subl %1,%0"40 asm volatile(LOCK_PREFIX "subl %1,%0"
@@ -69,27 +42,12 @@ static __always_inline void arch_atomic_sub(int i, atomic_t *v) · 1 unmodified line
69 : "ir" (i) : "memory");42 : "ir" (i) : "memory");
70}43}
71 44
72/**
73 * arch_atomic_sub_and_test - subtract value from variable and test result
74 * @i: integer value to subtract
75 * @v: pointer of type atomic_t
76 *
77 * Atomically subtracts @i from @v and returns
78 * true if the result is zero, or false for all
79 * other cases.
80 */
81static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)45static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
82{46{
83 return GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, e, "er", i);47 return GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, e, "er", i);
84}48}
85#define arch_atomic_sub_and_test arch_atomic_sub_and_test49#define arch_atomic_sub_and_test arch_atomic_sub_and_test
86 50
87/**
88 * arch_atomic_inc - increment atomic variable
89 * @v: pointer of type atomic_t
90 *
91 * Atomically increments @v by 1.
92 */
93static __always_inline void arch_atomic_inc(atomic_t *v)51static __always_inline void arch_atomic_inc(atomic_t *v)
94{52{
95 asm volatile(LOCK_PREFIX "incl %0"53 asm volatile(LOCK_PREFIX "incl %0"
@@ -97,12 +55,6 @@ static __always_inline void arch_atomic_inc(atomic_t *v) · 1 unmodified line
97}55}
98#define arch_atomic_inc arch_atomic_inc56#define arch_atomic_inc arch_atomic_inc
99 57
100/**
101 * arch_atomic_dec - decrement atomic variable
102 * @v: pointer of type atomic_t
103 *
104 * Atomically decrements @v by 1.
105 */
106static __always_inline void arch_atomic_dec(atomic_t *v)58static __always_inline void arch_atomic_dec(atomic_t *v)
107{59{
108 asm volatile(LOCK_PREFIX "decl %0"60 asm volatile(LOCK_PREFIX "decl %0"
@@ -110,69 +62,30 @@ static __always_inline void arch_atomic_dec(atomic_t *v) · 1 unmodified line
110}62}
111#define arch_atomic_dec arch_atomic_dec63#define arch_atomic_dec arch_atomic_dec
112 64
113/**
114 * arch_atomic_dec_and_test - decrement and test
115 * @v: pointer of type atomic_t
116 *
117 * Atomically decrements @v by 1 and
118 * returns true if the result is 0, or false for all other
119 * cases.
120 */
121static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)65static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
122{66{
123 return GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, e);67 return GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, e);
124}68}
125#define arch_atomic_dec_and_test arch_atomic_dec_and_test69#define arch_atomic_dec_and_test arch_atomic_dec_and_test
126 70
127/**
128 * arch_atomic_inc_and_test - increment and test
129 * @v: pointer of type atomic_t
130 *
131 * Atomically increments @v by 1
132 * and returns true if the result is zero, or false for all
133 * other cases.
134 */
135static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)71static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
136{72{
137 return GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, e);73 return GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, e);
138}74}
139#define arch_atomic_inc_and_test arch_atomic_inc_and_test75#define arch_atomic_inc_and_test arch_atomic_inc_and_test
140 76
141/**
142 * arch_atomic_add_negative - add and test if negative
143 * @i: integer value to add
144 * @v: pointer of type atomic_t
145 *
146 * Atomically adds @i to @v and returns true
147 * if the result is negative, or false when
148 * result is greater than or equal to zero.
149 */
150static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)77static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
151{78{
152 return GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, s, "er", i);79 return GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, s, "er", i);
153}80}
154#define arch_atomic_add_negative arch_atomic_add_negative81#define arch_atomic_add_negative arch_atomic_add_negative
155 82
156/**
157 * arch_atomic_add_return - add integer and return
158 * @i: integer value to add
159 * @v: pointer of type atomic_t
160 *
161 * Atomically adds @i to @v and returns @i + @v
162 */
163static __always_inline int arch_atomic_add_return(int i, atomic_t *v)83static __always_inline int arch_atomic_add_return(int i, atomic_t *v)
164{84{
165 return i + xadd(&v->counter, i);85 return i + xadd(&v->counter, i);
166}86}
167#define arch_atomic_add_return arch_atomic_add_return87#define arch_atomic_add_return arch_atomic_add_return
168 88
169/**
170 * arch_atomic_sub_return - subtract integer and return
171 * @v: pointer of type atomic_t
172 * @i: integer value to subtract
173 *
174 * Atomically subtracts @i from @v and returns @v - @i
175 */
176static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)89static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)
177{90{
178 return arch_atomic_add_return(-i, v);91 return arch_atomic_add_return(-i, v);
arch/x86/include/asm/atomic64_32.h+0−76
@@ -61,30 +61,12 @@ ATOMIC64_DECL(add_unless); · 60 unmodified lines
61#undef __ATOMIC64_DECL61#undef __ATOMIC64_DECL
62#undef ATOMIC64_EXPORT62#undef ATOMIC64_EXPORT
63 63
64/**
65 * arch_atomic64_cmpxchg - cmpxchg atomic64 variable
66 * @v: pointer to type atomic64_t
67 * @o: expected value
68 * @n: new value
69 *
70 * Atomically sets @v to @n if it was equal to @o and returns
71 * the old value.
72 */
73
74static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)64static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
75{65{
76 return arch_cmpxchg64(&v->counter, o, n);66 return arch_cmpxchg64(&v->counter, o, n);
77}67}
78#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg68#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
79 69
80/**
81 * arch_atomic64_xchg - xchg atomic64 variable
82 * @v: pointer to type atomic64_t
83 * @n: value to assign
84 *
85 * Atomically xchgs the value of @v to @n and returns
86 * the old value.
87 */
88static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)70static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)
89{71{
90 s64 o;72 s64 o;
@@ -97,13 +79,6 @@ static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) · 6 unmodified lines
97}79}
98#define arch_atomic64_xchg arch_atomic64_xchg80#define arch_atomic64_xchg arch_atomic64_xchg
99 81
100/**
101 * arch_atomic64_set - set atomic64 variable
102 * @v: pointer to type atomic64_t
103 * @i: value to assign
104 *
105 * Atomically sets the value of @v to @n.
106 */
107static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)82static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
108{83{
109 unsigned high = (unsigned)(i >> 32);84 unsigned high = (unsigned)(i >> 32);
@@ -113,12 +88,6 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) · 3 unmodified lines
113 : "eax", "edx", "memory");88 : "eax", "edx", "memory");
114}89}
115 90
116/**
117 * arch_atomic64_read - read atomic64 variable
118 * @v: pointer to type atomic64_t
119 *
120 * Atomically reads the value of @v and returns it.
121 */
122static __always_inline s64 arch_atomic64_read(const atomic64_t *v)91static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
123{92{
124 s64 r;93 s64 r;
@@ -126,13 +95,6 @@ static __always_inline s64 arch_atomic64_read(const atomic64_t *v) · 1 unmodified line
126 return r;95 return r;
127}96}
128 97
129/**
130 * arch_atomic64_add_return - add and return
131 * @i: integer value to add
132 * @v: pointer to type atomic64_t
133 *
134 * Atomically adds @i to @v and returns @i + *@v
135 */
136static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)98static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
137{99{
138 alternative_atomic64(add_return,100 alternative_atomic64(add_return,
@@ -142,9 +104,6 @@ static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) · 3 unmodified lines
142}104}
143#define arch_atomic64_add_return arch_atomic64_add_return105#define arch_atomic64_add_return arch_atomic64_add_return
144 106
145/*
146 * Other variants with different arithmetic operators:
147 */
148static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)107static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
149{108{
150 alternative_atomic64(sub_return,109 alternative_atomic64(sub_return,
@@ -172,13 +131,6 @@ static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v) · 21 unmodified lines
172}131}
173#define arch_atomic64_dec_return arch_atomic64_dec_return132#define arch_atomic64_dec_return arch_atomic64_dec_return
174 133
175/**
176 * arch_atomic64_add - add integer to atomic64 variable
177 * @i: integer value to add
178 * @v: pointer to type atomic64_t
179 *
180 * Atomically adds @i to @v.
181 */
182static __always_inline s64 arch_atomic64_add(s64 i, atomic64_t *v)134static __always_inline s64 arch_atomic64_add(s64 i, atomic64_t *v)
183{135{
184 __alternative_atomic64(add, add_return,136 __alternative_atomic64(add, add_return,
@@ -187,13 +139,6 @@ static __always_inline s64 arch_atomic64_add(s64 i, atomic64_t *v) · 2 unmodified lines
187 return i;139 return i;
188}140}
189 141
190/**
191 * arch_atomic64_sub - subtract the atomic64 variable
192 * @i: integer value to subtract
193 * @v: pointer to type atomic64_t
194 *
195 * Atomically subtracts @i from @v.
196 */
197static __always_inline s64 arch_atomic64_sub(s64 i, atomic64_t *v)142static __always_inline s64 arch_atomic64_sub(s64 i, atomic64_t *v)
198{143{
199 __alternative_atomic64(sub, sub_return,144 __alternative_atomic64(sub, sub_return,
@@ -202,12 +147,6 @@ static __always_inline s64 arch_atomic64_sub(s64 i, atomic64_t *v) · 2 unmodified lines
202 return i;147 return i;
203}148}
204 149
205/**
206 * arch_atomic64_inc - increment atomic64 variable
207 * @v: pointer to type atomic64_t
208 *
209 * Atomically increments @v by 1.
210 */
211static __always_inline void arch_atomic64_inc(atomic64_t *v)150static __always_inline void arch_atomic64_inc(atomic64_t *v)
212{151{
213 __alternative_atomic64(inc, inc_return, /* no output */,152 __alternative_atomic64(inc, inc_return, /* no output */,
@@ -215,12 +154,6 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v) · 1 unmodified line
215}154}
216#define arch_atomic64_inc arch_atomic64_inc155#define arch_atomic64_inc arch_atomic64_inc
217 156
218/**
219 * arch_atomic64_dec - decrement atomic64 variable
220 * @v: pointer to type atomic64_t
221 *
222 * Atomically decrements @v by 1.
223 */
224static __always_inline void arch_atomic64_dec(atomic64_t *v)157static __always_inline void arch_atomic64_dec(atomic64_t *v)
225{158{
226 __alternative_atomic64(dec, dec_return, /* no output */,159 __alternative_atomic64(dec, dec_return, /* no output */,
@@ -228,15 +161,6 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v) · 1 unmodified line
228}161}
229#define arch_atomic64_dec arch_atomic64_dec162#define arch_atomic64_dec arch_atomic64_dec
230 163
231/**
232 * arch_atomic64_add_unless - add unless the number is a given value
233 * @v: pointer of type atomic64_t
234 * @a: the amount to add to v...
235 * @u: ...unless v is equal to u.
236 *
237 * Atomically adds @a to @v, so long as it was not @u.
238 * Returns non-zero if the add was done, zero otherwise.
239 */
240static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)164static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
241{165{
242 unsigned low = (unsigned)u;166 unsigned low = (unsigned)u;
arch/x86/include/asm/atomic64_64.h+0−81
@@ -10,37 +10,16 @@ · 9 unmodified lines
10 10
11#define ATOMIC64_INIT(i) { (i) }11#define ATOMIC64_INIT(i) { (i) }
12 12
13/**
14 * arch_atomic64_read - read atomic64 variable
15 * @v: pointer of type atomic64_t
16 *
17 * Atomically reads the value of @v.
18 * Doesn't imply a read memory barrier.
19 */
20static __always_inline s64 arch_atomic64_read(const atomic64_t *v)13static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
21{14{
22 return __READ_ONCE((v)->counter);15 return __READ_ONCE((v)->counter);
23}16}
24 17
25/**
26 * arch_atomic64_set - set atomic64 variable
27 * @v: pointer to type atomic64_t
28 * @i: required value
29 *
30 * Atomically sets the value of @v to @i.
31 */
32static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)18static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
33{19{
34 __WRITE_ONCE(v->counter, i);20 __WRITE_ONCE(v->counter, i);
35}21}
36 22
37/**
38 * arch_atomic64_add - add integer to atomic64 variable
39 * @i: integer value to add
40 * @v: pointer to type atomic64_t
41 *
42 * Atomically adds @i to @v.
43 */
44static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)23static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
45{24{
46 asm volatile(LOCK_PREFIX "addq %1,%0"25 asm volatile(LOCK_PREFIX "addq %1,%0"
@@ -48,13 +27,6 @@ static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v) · 1 unmodified line
48 : "er" (i), "m" (v->counter) : "memory");27 : "er" (i), "m" (v->counter) : "memory");
49}28}
50 29
51/**
52 * arch_atomic64_sub - subtract the atomic64 variable
53 * @i: integer value to subtract
54 * @v: pointer to type atomic64_t
55 *
56 * Atomically subtracts @i from @v.
57 */
58static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)30static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
59{31{
60 asm volatile(LOCK_PREFIX "subq %1,%0"32 asm volatile(LOCK_PREFIX "subq %1,%0"
@@ -62,27 +34,12 @@ static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v) · 1 unmodified line
62 : "er" (i), "m" (v->counter) : "memory");34 : "er" (i), "m" (v->counter) : "memory");
63}35}
64 36
65/**
66 * arch_atomic64_sub_and_test - subtract value from variable and test result
67 * @i: integer value to subtract
68 * @v: pointer to type atomic64_t
69 *
70 * Atomically subtracts @i from @v and returns
71 * true if the result is zero, or false for all
72 * other cases.
73 */
74static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)37static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
75{38{
76 return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);39 return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);
77}40}
78#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test41#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
79 42
80/**
81 * arch_atomic64_inc - increment atomic64 variable
82 * @v: pointer to type atomic64_t
83 *
84 * Atomically increments @v by 1.
85 */
86static __always_inline void arch_atomic64_inc(atomic64_t *v)43static __always_inline void arch_atomic64_inc(atomic64_t *v)
87{44{
88 asm volatile(LOCK_PREFIX "incq %0"45 asm volatile(LOCK_PREFIX "incq %0"
@@ -91,12 +48,6 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v) · 2 unmodified lines
91}48}
92#define arch_atomic64_inc arch_atomic64_inc49#define arch_atomic64_inc arch_atomic64_inc
93 50
94/**
95 * arch_atomic64_dec - decrement atomic64 variable
96 * @v: pointer to type atomic64_t
97 *
98 * Atomically decrements @v by 1.
99 */
100static __always_inline void arch_atomic64_dec(atomic64_t *v)51static __always_inline void arch_atomic64_dec(atomic64_t *v)
101{52{
102 asm volatile(LOCK_PREFIX "decq %0"53 asm volatile(LOCK_PREFIX "decq %0"
@@ -105,56 +56,24 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v) · 2 unmodified lines
105}56}
106#define arch_atomic64_dec arch_atomic64_dec57#define arch_atomic64_dec arch_atomic64_dec
107 58
108/**
109 * arch_atomic64_dec_and_test - decrement and test
110 * @v: pointer to type atomic64_t
111 *
112 * Atomically decrements @v by 1 and
113 * returns true if the result is 0, or false for all other
114 * cases.
115 */
116static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v)59static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v)
117{60{
118 return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);61 return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);
119}62}
120#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test63#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
121 64
122/**
123 * arch_atomic64_inc_and_test - increment and test
124 * @v: pointer to type atomic64_t
125 *
126 * Atomically increments @v by 1
127 * and returns true if the result is zero, or false for all
128 * other cases.
129 */
130static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v)65static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v)
131{66{
132 return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);67 return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);
133}68}
134#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test69#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
135 70
136/**
137 * arch_atomic64_add_negative - add and test if negative
138 * @i: integer value to add
139 * @v: pointer to type atomic64_t
140 *
141 * Atomically adds @i to @v and returns true
142 * if the result is negative, or false when
143 * result is greater than or equal to zero.
144 */
145static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)71static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)
146{72{
147 return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);73 return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);
148}74}
149#define arch_atomic64_add_negative arch_atomic64_add_negative75#define arch_atomic64_add_negative arch_atomic64_add_negative
150 76
151/**
152 * arch_atomic64_add_return - add and return
153 * @i: integer value to add
154 * @v: pointer to type atomic64_t
155 *
156 * Atomically adds @i to @v and returns @i + @v
157 */
158static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)77static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
159{78{
160 return i + xadd(&v->counter, i);79 return i + xadd(&v->counter, i);
arch/x86/include/asm/cmpxchg.h+0−25
@@ -239,29 +239,4 @@ extern void __add_wrong_size(void) · 238 unmodified lines
239#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)239#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
240#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)240#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
241 241
242#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
243({ \
244 bool __ret; \
245 __typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \
246 __typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \
247 BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
248 BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
249 VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \
250 VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \
251 asm volatile(pfx "cmpxchg%c5b %1" \
252 CC_SET(e) \
253 : CC_OUT(e) (__ret), \
254 "+m" (*(p1)), "+m" (*(p2)), \
255 "+a" (__old1), "+d" (__old2) \
256 : "i" (2 * sizeof(long)), \
257 "b" (__new1), "c" (__new2)); \
258 __ret; \
259})
260
261#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \
262 __cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)
263
264#define arch_cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
265 __cmpxchg_double(, p1, p2, o1, o2, n1, n2)
266
267#endif /* ASM_X86_CMPXCHG_H */242#endif /* ASM_X86_CMPXCHG_H */
arch/x86/include/asm/cmpxchg_32.h+1−1
@@ -103,6 +103,6 @@ static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new) · 102 unmodified lines
103 103
104#endif104#endif
105 105
106#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)106#define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8)
107 107
108#endif /* _ASM_X86_CMPXCHG_32_H */108#endif /* _ASM_X86_CMPXCHG_32_H */
arch/x86/include/asm/cmpxchg_64.h+66−1
@@ -20,6 +20,71 @@ · 19 unmodified lines
20 arch_try_cmpxchg((ptr), (po), (n)); \20 arch_try_cmpxchg((ptr), (po), (n)); \
21})21})
22 22
23#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16)23union __u128_halves {
24 u128 full;
25 struct {
26 u64 low, high;
27 };
28};
29
30#define __arch_cmpxchg128(_ptr, _old, _new, _lock) \
31({ \
32 union __u128_halves o = { .full = (_old), }, \
33 n = { .full = (_new), }; \
34 \
35 asm volatile(_lock "cmpxchg16b %[ptr]" \
36 : [ptr] "+m" (*(_ptr)), \
37 "+a" (o.low), "+d" (o.high) \
38 : "b" (n.low), "c" (n.high) \
39 : "memory"); \
40 \
41 o.full; \
42})
43
44static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
45{
46 return __arch_cmpxchg128(ptr, old, new, LOCK_PREFIX);
47}
48#define arch_cmpxchg128 arch_cmpxchg128
49
50static __always_inline u128 arch_cmpxchg128_local(volatile u128 *ptr, u128 old, u128 new)
51{
52 return __arch_cmpxchg128(ptr, old, new,);
53}
54#define arch_cmpxchg128_local arch_cmpxchg128_local
55
56#define __arch_try_cmpxchg128(_ptr, _oldp, _new, _lock) \
57({ \
58 union __u128_halves o = { .full = *(_oldp), }, \
59 n = { .full = (_new), }; \
60 bool ret; \
61 \
62 asm volatile(_lock "cmpxchg16b %[ptr]" \
63 CC_SET(e) \
64 : CC_OUT(e) (ret), \
65 [ptr] "+m" (*ptr), \
66 "+a" (o.low), "+d" (o.high) \
67 : "b" (n.low), "c" (n.high) \
68 : "memory"); \
69 \
70 if (unlikely(!ret)) \
71 *(_oldp) = o.full; \
72 \
73 likely(ret); \
74})
75
76static __always_inline bool arch_try_cmpxchg128(volatile u128 *ptr, u128 *oldp, u128 new)
77{
78 return __arch_try_cmpxchg128(ptr, oldp, new, LOCK_PREFIX);
79}
80#define arch_try_cmpxchg128 arch_try_cmpxchg128
81
82static __always_inline bool arch_try_cmpxchg128_local(volatile u128 *ptr, u128 *oldp, u128 new)
83{
84 return __arch_try_cmpxchg128(ptr, oldp, new,);
85}
86#define arch_try_cmpxchg128_local arch_try_cmpxchg128_local
87
88#define system_has_cmpxchg128() boot_cpu_has(X86_FEATURE_CX16)
24 89
25#endif /* _ASM_X86_CMPXCHG_64_H */90#endif /* _ASM_X86_CMPXCHG_64_H */
arch/x86/include/asm/percpu.h+61−41
@@ -210,6 +210,67 @@ do { \ · 209 unmodified lines
210 (typeof(_var))(unsigned long) pco_old__; \210 (typeof(_var))(unsigned long) pco_old__; \
211})211})
212 212
213#if defined(CONFIG_X86_32) && !defined(CONFIG_UML)
214#define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \
215({ \
216 union { \
217 u64 var; \
218 struct { \
219 u32 low, high; \
220 }; \
221 } old__, new__; \
222 \
223 old__.var = _oval; \
224 new__.var = _nval; \
225 \
226 asm qual (ALTERNATIVE("leal %P[var], %%esi; call this_cpu_cmpxchg8b_emu", \
227 "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
228 : [var] "+m" (_var), \
229 "+a" (old__.low), \
230 "+d" (old__.high) \
231 : "b" (new__.low), \
232 "c" (new__.high) \
233 : "memory", "esi"); \
234 \
235 old__.var; \
236})
237
238#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval)
239#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
240#endif
241
242#ifdef CONFIG_X86_64
243#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval);
244#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
245
246#define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval) \
247({ \
248 union { \
249 u128 var; \
250 struct { \
251 u64 low, high; \
252 }; \
253 } old__, new__; \
254 \
255 old__.var = _oval; \
256 new__.var = _nval; \
257 \
258 asm qual (ALTERNATIVE("leaq %P[var], %%rsi; call this_cpu_cmpxchg16b_emu", \
259 "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
260 : [var] "+m" (_var), \
261 "+a" (old__.low), \
262 "+d" (old__.high) \
263 : "b" (new__.low), \
264 "c" (new__.high) \
265 : "memory", "rsi"); \
266 \
267 old__.var; \
268})
269
270#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval)
271#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
272#endif
273
213/*274/*
214 * this_cpu_read() makes gcc load the percpu variable every time it is275 * this_cpu_read() makes gcc load the percpu variable every time it is
215 * accessed while this_cpu_read_stable() allows the value to be cached.276 * accessed while this_cpu_read_stable() allows the value to be cached.
@@ -290,23 +351,6 @@ do { \ · 74 unmodified lines
290#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)351#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
291#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)352#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
292 353
293#ifdef CONFIG_X86_CMPXCHG64
294#define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2) \
295({ \
296 bool __ret; \
297 typeof(pcp1) __o1 = (o1), __n1 = (n1); \
298 typeof(pcp2) __o2 = (o2), __n2 = (n2); \
299 asm volatile("cmpxchg8b "__percpu_arg(1) \
300 CC_SET(z) \
301 : CC_OUT(z) (__ret), "+m" (pcp1), "+m" (pcp2), "+a" (__o1), "+d" (__o2) \
302 : "b" (__n1), "c" (__n2)); \
303 __ret; \
304})
305
306#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
307#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
308#endif /* CONFIG_X86_CMPXCHG64 */
309
310/*354/*
311 * Per cpu atomic 64 bit operations are only available under 64 bit.355 * Per cpu atomic 64 bit operations are only available under 64 bit.
312 * 32 bit must fall back to generic operations.356 * 32 bit must fall back to generic operations.
@@ -329,30 +373,6 @@ do { \ · 16 unmodified lines
329#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)373#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
330#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval)374#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval)
331#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)375#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
332
333/*
334 * Pretty complex macro to generate cmpxchg16 instruction. The instruction
335 * is not supported on early AMD64 processors so we must be able to emulate
336 * it in software. The address used in the cmpxchg16 instruction must be
337 * aligned to a 16 byte boundary.
338 */
339#define percpu_cmpxchg16b_double(pcp1, pcp2, o1, o2, n1, n2) \
340({ \
341 bool __ret; \
342 typeof(pcp1) __o1 = (o1), __n1 = (n1); \
343 typeof(pcp2) __o2 = (o2), __n2 = (n2); \
344 alternative_io("leaq %P1,%%rsi\n\tcall this_cpu_cmpxchg16b_emu\n\t", \
345 "cmpxchg16b " __percpu_arg(1) "\n\tsetz %0\n\t", \
346 X86_FEATURE_CX16, \
347 ASM_OUTPUT2("=a" (__ret), "+m" (pcp1), \
348 "+m" (pcp2), "+d" (__o2)), \
349 "b" (__n1), "c" (__n2), "a" (__o1) : "rsi"); \
350 __ret; \
351})
352
353#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
354#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
355
356#endif376#endif
357 377
358static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,378static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,
arch/x86/kernel/alternative.c+2−2
@@ -1967,7 +1967,7 @@ struct bp_patching_desc *try_get_desc(void) · 1966 unmodified lines
1967{1967{
1968 struct bp_patching_desc *desc = &bp_desc;1968 struct bp_patching_desc *desc = &bp_desc;
1969 1969
1970 if (!arch_atomic_inc_not_zero(&desc->refs))1970 if (!raw_atomic_inc_not_zero(&desc->refs))
1971 return NULL;1971 return NULL;
1972 1972
1973 return desc;1973 return desc;
@@ -1978,7 +1978,7 @@ static __always_inline void put_desc(void) · 4 unmodified lines
1978 struct bp_patching_desc *desc = &bp_desc;1978 struct bp_patching_desc *desc = &bp_desc;
1979 1979
1980 smp_mb__before_atomic();1980 smp_mb__before_atomic();
1981 arch_atomic_dec(&desc->refs);1981 raw_atomic_dec(&desc->refs);
1982}1982}
1983 1983
1984static __always_inline void *text_poke_addr(struct text_poke_loc *tp)1984static __always_inline void *text_poke_addr(struct text_poke_loc *tp)
arch/x86/kernel/cpu/mce/core.c+8−8
@@ -1022,12 +1022,12 @@ static noinstr int mce_start(int *no_way_out) · 1021 unmodified lines
1022 if (!timeout)1022 if (!timeout)
1023 return ret;1023 return ret;
1024 1024
1025 arch_atomic_add(*no_way_out, &global_nwo);1025 raw_atomic_add(*no_way_out, &global_nwo);
1026 /*1026 /*
1027 * Rely on the implied barrier below, such that global_nwo1027 * Rely on the implied barrier below, such that global_nwo
1028 * is updated before mce_callin.1028 * is updated before mce_callin.
1029 */1029 */
1030 order = arch_atomic_inc_return(&mce_callin);1030 order = raw_atomic_inc_return(&mce_callin);
1031 arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);1031 arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);
1032 1032
1033 /* Enable instrumentation around calls to external facilities */1033 /* Enable instrumentation around calls to external facilities */
@@ -1036,10 +1036,10 @@ static noinstr int mce_start(int *no_way_out) · 2 unmodified lines
1036 /*1036 /*
1037 * Wait for everyone.1037 * Wait for everyone.
1038 */1038 */
1039 while (arch_atomic_read(&mce_callin) != num_online_cpus()) {1039 while (raw_atomic_read(&mce_callin) != num_online_cpus()) {
1040 if (mce_timed_out(&timeout,1040 if (mce_timed_out(&timeout,
1041 "Timeout: Not all CPUs entered broadcast exception handler")) {1041 "Timeout: Not all CPUs entered broadcast exception handler")) {
1042 arch_atomic_set(&global_nwo, 0);1042 raw_atomic_set(&global_nwo, 0);
1043 goto out;1043 goto out;
1044 }1044 }
1045 ndelay(SPINUNIT);1045 ndelay(SPINUNIT);
@@ -1054,7 +1054,7 @@ static noinstr int mce_start(int *no_way_out) · 8 unmodified lines
1054 /*1054 /*
1055 * Monarch: Starts executing now, the others wait.1055 * Monarch: Starts executing now, the others wait.
1056 */1056 */
1057 arch_atomic_set(&mce_executing, 1);1057 raw_atomic_set(&mce_executing, 1);
1058 } else {1058 } else {
1059 /*1059 /*
1060 * Subject: Now start the scanning loop one by one in1060 * Subject: Now start the scanning loop one by one in
@@ -1062,10 +1062,10 @@ static noinstr int mce_start(int *no_way_out) · 1 unmodified line
1062 * This way when there are any shared banks it will be1062 * This way when there are any shared banks it will be
1063 * only seen by one CPU before cleared, avoiding duplicates.1063 * only seen by one CPU before cleared, avoiding duplicates.
1064 */1064 */
1065 while (arch_atomic_read(&mce_executing) < order) {1065 while (raw_atomic_read(&mce_executing) < order) {
1066 if (mce_timed_out(&timeout,1066 if (mce_timed_out(&timeout,
1067 "Timeout: Subject CPUs unable to finish machine check processing")) {1067 "Timeout: Subject CPUs unable to finish machine check processing")) {
1068 arch_atomic_set(&global_nwo, 0);1068 raw_atomic_set(&global_nwo, 0);
1069 goto out;1069 goto out;
1070 }1070 }
1071 ndelay(SPINUNIT);1071 ndelay(SPINUNIT);
@@ -1075,7 +1075,7 @@ static noinstr int mce_start(int *no_way_out) · 3 unmodified lines
1075 /*1075 /*
1076 * Cache the global no_way_out state.1076 * Cache the global no_way_out state.
1077 */1077 */
1078 *no_way_out = arch_atomic_read(&global_nwo);1078 *no_way_out = raw_atomic_read(&global_nwo);
1079 1079
1080 ret = order;1080 ret = order;
1081 1081
arch/x86/kernel/nmi.c+1−1
@@ -496,7 +496,7 @@ DEFINE_IDTENTRY_RAW(exc_nmi) · 495 unmodified lines
496 */496 */
497 sev_es_nmi_complete();497 sev_es_nmi_complete();
498 if (IS_ENABLED(CONFIG_NMI_CHECK_CPU))498 if (IS_ENABLED(CONFIG_NMI_CHECK_CPU))
499 arch_atomic_long_inc(&nsp->idt_calls);499 raw_atomic_long_inc(&nsp->idt_calls);
500 500
501 if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))501 if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))
502 return;502 return;
arch/x86/kernel/pvclock.c+2−2
@@ -101,11 +101,11 @@ u64 __pvclock_clocksource_read(struct pvclock_vcpu_time_info *src, bool dowd) · 100 unmodified lines
101 * updating at the same time, and one of them could be slightly behind,101 * updating at the same time, and one of them could be slightly behind,
102 * making the assumption that last_value always go forward fail to hold.102 * making the assumption that last_value always go forward fail to hold.
103 */103 */
104 last = arch_atomic64_read(&last_value);104 last = raw_atomic64_read(&last_value);
105 do {105 do {
106 if (ret <= last)106 if (ret <= last)
107 return last;107 return last;
108 } while (!arch_atomic64_try_cmpxchg(&last_value, &last, ret));108 } while (!raw_atomic64_try_cmpxchg(&last_value, &last, ret));
109 109
110 return ret;110 return ret;
111}111}
arch/x86/kvm/x86.c+1−1
@@ -13161,7 +13161,7 @@ EXPORT_SYMBOL_GPL(kvm_arch_end_assignment); · 13160 unmodified lines
13161 13161
13162bool noinstr kvm_arch_has_assigned_device(struct kvm *kvm)13162bool noinstr kvm_arch_has_assigned_device(struct kvm *kvm)
13163{13163{
13164 return arch_atomic_read(&kvm->arch.assigned_device_count);13164 return raw_atomic_read(&kvm->arch.assigned_device_count);
13165}13165}
13166EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);13166EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);
13167 13167
arch/x86/lib/Makefile+2−1
@@ -61,8 +61,9 @@ ifeq ($(CONFIG_X86_32),y) · 60 unmodified lines
61 lib-y += strstr_32.o61 lib-y += strstr_32.o
62 lib-y += string_32.o62 lib-y += string_32.o
63 lib-y += memmove_32.o63 lib-y += memmove_32.o
64 lib-y += cmpxchg8b_emu.o
64ifneq ($(CONFIG_X86_CMPXCHG64),y)65ifneq ($(CONFIG_X86_CMPXCHG64),y)
65 lib-y += cmpxchg8b_emu.o atomic64_386_32.o66 lib-y += atomic64_386_32.o
66endif67endif
67else68else
68 obj-y += iomap_copy_64.o69 obj-y += iomap_copy_64.o
arch/x86/lib/cmpxchg16b_emu.S+25−18
@@ -1,47 +1,54 @@
1/* SPDX-License-Identifier: GPL-2.0-only */1/* SPDX-License-Identifier: GPL-2.0-only */
2#include <linux/linkage.h>2#include <linux/linkage.h>
3#include <asm/percpu.h>3#include <asm/percpu.h>
4#include <asm/processor-flags.h>
4 5
5.text6.text
6 7
7/*8/*
9 * Emulate 'cmpxchg16b %gs:(%rsi)'
10 *
8 * Inputs:11 * Inputs:
9 * %rsi : memory location to compare12 * %rsi : memory location to compare
10 * %rax : low 64 bits of old value13 * %rax : low 64 bits of old value
11 * %rdx : high 64 bits of old value14 * %rdx : high 64 bits of old value
12 * %rbx : low 64 bits of new value15 * %rbx : low 64 bits of new value
13 * %rcx : high 64 bits of new value16 * %rcx : high 64 bits of new value
14 * %al : Operation successful17 *
18 * Notably this is not LOCK prefixed and is not safe against NMIs
15 */19 */
16SYM_FUNC_START(this_cpu_cmpxchg16b_emu)20SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
17 21
18#
19# Emulate 'cmpxchg16b %gs:(%rsi)' except we return the result in %al not
20# via the ZF. Caller will access %al to get result.
21#
22# Note that this is only useful for a cpuops operation. Meaning that we
23# do *not* have a fully atomic operation but just an operation that is
24# *atomic* on a single cpu (as provided by the this_cpu_xx class of
25# macros).
26#
27 pushfq22 pushfq
28 cli23 cli
29 24
30 cmpq PER_CPU_VAR((%rsi)), %rax25 /* if (*ptr == old) */
31 jne .Lnot_same26 cmpq PER_CPU_VAR(0(%rsi)), %rax
32 cmpq PER_CPU_VAR(8(%rsi)), %rdx27 jne .Lnot_same
33 jne .Lnot_same28 cmpq PER_CPU_VAR(8(%rsi)), %rdx
29 jne .Lnot_same
34 30
35 movq %rbx, PER_CPU_VAR((%rsi))31 /* *ptr = new */
36 movq %rcx, PER_CPU_VAR(8(%rsi))32 movq %rbx, PER_CPU_VAR(0(%rsi))
33 movq %rcx, PER_CPU_VAR(8(%rsi))
34
35 /* set ZF in EFLAGS to indicate success */
36 orl $X86_EFLAGS_ZF, (%rsp)
37 37
38 popfq38 popfq
39 mov $1, %al
40 RET39 RET
41 40
42.Lnot_same:41.Lnot_same:
42 /* *ptr != old */
43
44 /* old = *ptr */
45 movq PER_CPU_VAR(0(%rsi)), %rax
46 movq PER_CPU_VAR(8(%rsi)), %rdx
47
48 /* clear ZF in EFLAGS to indicate failure */
49 andl $(~X86_EFLAGS_ZF), (%rsp)
50
43 popfq51 popfq
44 xor %al,%al
45 RET52 RET
46 53
47SYM_FUNC_END(this_cpu_cmpxchg16b_emu)54SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
arch/x86/lib/cmpxchg8b_emu.S+53−14
@@ -2,10 +2,16 @@ · 1 unmodified line
2 2
3#include <linux/linkage.h>3#include <linux/linkage.h>
4#include <asm/export.h>4#include <asm/export.h>
5#include <asm/percpu.h>
6#include <asm/processor-flags.h>
5 7
6.text8.text
7 9
10#ifndef CONFIG_X86_CMPXCHG64
11
8/*12/*
13 * Emulate 'cmpxchg8b (%esi)' on UP
14 *
9 * Inputs:15 * Inputs:
10 * %esi : memory location to compare16 * %esi : memory location to compare
11 * %eax : low 32 bits of old value17 * %eax : low 32 bits of old value
@@ -15,32 +21,65 @@ · 3 unmodified lines
15 */21 */
16SYM_FUNC_START(cmpxchg8b_emu)22SYM_FUNC_START(cmpxchg8b_emu)
17 23
18#
19# Emulate 'cmpxchg8b (%esi)' on UP except we don't
20# set the whole ZF thing (caller will just compare
21# eax:edx with the expected value)
22#
23 pushfl24 pushfl
24 cli25 cli
25 26
26 cmpl (%esi), %eax27 cmpl 0(%esi), %eax
27 jne .Lnot_same28 jne .Lnot_same
28 cmpl 4(%esi), %edx29 cmpl 4(%esi), %edx
29 jne .Lhalf_same30 jne .Lnot_same
31
32 movl %ebx, 0(%esi)
33 movl %ecx, 4(%esi)
30 34
31 movl %ebx, (%esi)35 orl $X86_EFLAGS_ZF, (%esp)
32 movl %ecx, 4(%esi)
33 36
34 popfl37 popfl
35 RET38 RET
36 39
37.Lnot_same:40.Lnot_same:
38 movl (%esi), %eax41 movl 0(%esi), %eax
39.Lhalf_same:42 movl 4(%esi), %edx
40 movl 4(%esi), %edx43
44 andl $(~X86_EFLAGS_ZF), (%esp)
41 45
42 popfl46 popfl
43 RET47 RET
44 48
45SYM_FUNC_END(cmpxchg8b_emu)49SYM_FUNC_END(cmpxchg8b_emu)
46EXPORT_SYMBOL(cmpxchg8b_emu)50EXPORT_SYMBOL(cmpxchg8b_emu)
51
52#endif
53
54#ifndef CONFIG_UML
55
56SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
57
58 pushfl
59 cli
60
61 cmpl PER_CPU_VAR(0(%esi)), %eax
62 jne .Lnot_same2
63 cmpl PER_CPU_VAR(4(%esi)), %edx
64 jne .Lnot_same2
65
66 movl %ebx, PER_CPU_VAR(0(%esi))
67 movl %ecx, PER_CPU_VAR(4(%esi))
68
69 orl $X86_EFLAGS_ZF, (%esp)
70
71 popfl
72 RET
73
74.Lnot_same2:
75 movl PER_CPU_VAR(0(%esi)), %eax
76 movl PER_CPU_VAR(4(%esi)), %edx
77
78 andl $(~X86_EFLAGS_ZF), (%esp)
79
80 popfl
81 RET
82
83SYM_FUNC_END(this_cpu_cmpxchg8b_emu)
84
85#endif
arch/xtensa/include/asm/atomic.h+9−3
@@ -245,6 +245,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \ · 244 unmodified lines
245ATOMIC_OPS(add)245ATOMIC_OPS(add)
246ATOMIC_OPS(sub)246ATOMIC_OPS(sub)
247 247
248#define arch_atomic_add_return arch_atomic_add_return
249#define arch_atomic_sub_return arch_atomic_sub_return
250#define arch_atomic_fetch_add arch_atomic_fetch_add
251#define arch_atomic_fetch_sub arch_atomic_fetch_sub
252
248#undef ATOMIC_OPS253#undef ATOMIC_OPS
249#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)254#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
250 255
@@ -252,12 +257,13 @@ ATOMIC_OPS(and) · 1 unmodified line
252ATOMIC_OPS(or)257ATOMIC_OPS(or)
253ATOMIC_OPS(xor)258ATOMIC_OPS(xor)
254 259
260#define arch_atomic_fetch_and arch_atomic_fetch_and
261#define arch_atomic_fetch_or arch_atomic_fetch_or
262#define arch_atomic_fetch_xor arch_atomic_fetch_xor
263
255#undef ATOMIC_OPS264#undef ATOMIC_OPS
256#undef ATOMIC_FETCH_OP265#undef ATOMIC_FETCH_OP
257#undef ATOMIC_OP_RETURN266#undef ATOMIC_OP_RETURN
258#undef ATOMIC_OP267#undef ATOMIC_OP
259 268
260#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n)))
261#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
262
263#endif /* _XTENSA_ATOMIC_H */269#endif /* _XTENSA_ATOMIC_H */
drivers/iommu/amd/amd_iommu_types.h+7−2
@@ -986,8 +986,13 @@ union irte_ga_hi { · 985 unmodified lines
986};986};
987 987
988struct irte_ga {988struct irte_ga {
989 union irte_ga_lo lo;989 union {
990 union irte_ga_hi hi;990 struct {
991 union irte_ga_lo lo;
992 union irte_ga_hi hi;
993 };
994 u128 irte;
995 };
991};996};
992 997
993struct irq_2_irte {998struct irq_2_irte {
drivers/iommu/amd/iommu.c+4−6
@@ -3023,10 +3023,10 @@ out: · 3022 unmodified lines
3023static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,3023static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
3024 struct irte_ga *irte, struct amd_ir_data *data)3024 struct irte_ga *irte, struct amd_ir_data *data)
3025{3025{
3026 bool ret;
3027 struct irq_remap_table *table;3026 struct irq_remap_table *table;
3028 unsigned long flags;
3029 struct irte_ga *entry;3027 struct irte_ga *entry;
3028 unsigned long flags;
3029 u128 old;
3030 3030
3031 table = get_irq_table(iommu, devid);3031 table = get_irq_table(iommu, devid);
3032 if (!table)3032 if (!table)
@@ -3037,16 +3037,14 @@ static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, · 4 unmodified lines
3037 entry = (struct irte_ga *)table->table;3037 entry = (struct irte_ga *)table->table;
3038 entry = &entry[index];3038 entry = &entry[index];
3039 3039
3040 ret = cmpxchg_double(&entry->lo.val, &entry->hi.val,
3041 entry->lo.val, entry->hi.val,
3042 irte->lo.val, irte->hi.val);
3043 /*3040 /*
3044 * We use cmpxchg16 to atomically update the 128-bit IRTE,3041 * We use cmpxchg16 to atomically update the 128-bit IRTE,
3045 * and it cannot be updated by the hardware or other processors3042 * and it cannot be updated by the hardware or other processors
3046 * behind us, so the return value of cmpxchg16 should be the3043 * behind us, so the return value of cmpxchg16 should be the
3047 * same as the old value.3044 * same as the old value.
3048 */3045 */
3049 WARN_ON(!ret);3046 old = entry->irte;
3047 WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte));
3050 3048
3051 if (data)3049 if (data)
3052 data->ref = entry;3050 data->ref = entry;
drivers/iommu/intel/irq_remapping.c+2−6
@@ -175,18 +175,14 @@ static int modify_irte(struct irq_2_iommu *irq_iommu, · 174 unmodified lines
175 irte = &iommu->ir_table->base[index];175 irte = &iommu->ir_table->base[index];
176 176
177 if ((irte->pst == 1) || (irte_modified->pst == 1)) {177 if ((irte->pst == 1) || (irte_modified->pst == 1)) {
178 bool ret;
179
180 ret = cmpxchg_double(&irte->low, &irte->high,
181 irte->low, irte->high,
182 irte_modified->low, irte_modified->high);
183 /*178 /*
184 * We use cmpxchg16 to atomically update the 128-bit IRTE,179 * We use cmpxchg16 to atomically update the 128-bit IRTE,
185 * and it cannot be updated by the hardware or other processors180 * and it cannot be updated by the hardware or other processors
186 * behind us, so the return value of cmpxchg16 should be the181 * behind us, so the return value of cmpxchg16 should be the
187 * same as the old value.182 * same as the old value.
188 */183 */
189 WARN_ON(!ret);184 u128 old = irte->irte;
185 WARN_ON(!try_cmpxchg128(&irte->irte, &old, irte_modified->irte));
190 } else {186 } else {
191 WRITE_ONCE(irte->low, irte_modified->low);187 WRITE_ONCE(irte->low, irte_modified->low);
192 WRITE_ONCE(irte->high, irte_modified->high);188 WRITE_ONCE(irte->high, irte_modified->high);
drivers/md/bcache/btree.c+22−1
@@ -559,6 +559,27 @@ static void mca_data_alloc(struct btree *b, struct bkey *k, gfp_t gfp) · 558 unmodified lines
559 }559 }
560}560}
561 561
562#define cmp_int(l, r) ((l > r) - (l < r))
563
564#ifdef CONFIG_PROVE_LOCKING
565static int btree_lock_cmp_fn(const struct lockdep_map *_a,
566 const struct lockdep_map *_b)
567{
568 const struct btree *a = container_of(_a, struct btree, lock.dep_map);
569 const struct btree *b = container_of(_b, struct btree, lock.dep_map);
570
571 return -cmp_int(a->level, b->level) ?: bkey_cmp(&a->key, &b->key);
572}
573
574static void btree_lock_print_fn(const struct lockdep_map *map)
575{
576 const struct btree *b = container_of(map, struct btree, lock.dep_map);
577
578 printk(KERN_CONT " l=%u %llu:%llu", b->level,
579 KEY_INODE(&b->key), KEY_OFFSET(&b->key));
580}
581#endif
582
562static struct btree *mca_bucket_alloc(struct cache_set *c,583static struct btree *mca_bucket_alloc(struct cache_set *c,
563 struct bkey *k, gfp_t gfp)584 struct bkey *k, gfp_t gfp)
564{585{
@@ -572,7 +593,7 @@ static struct btree *mca_bucket_alloc(struct cache_set *c, · 7 unmodified lines
572 return NULL;593 return NULL;
573 594
574 init_rwsem(&b->lock);595 init_rwsem(&b->lock);
575 lockdep_set_novalidate_class(&b->lock);596 lock_set_cmp_fn(&b->lock, btree_lock_cmp_fn, btree_lock_print_fn);
576 mutex_init(&b->write_lock);597 mutex_init(&b->write_lock);
577 lockdep_set_novalidate_class(&b->write_lock);598 lockdep_set_novalidate_class(&b->write_lock);
578 INIT_LIST_HEAD(&b->list);599 INIT_LIST_HEAD(&b->list);
drivers/md/bcache/btree.h+2−2
@@ -247,8 +247,8 @@ static inline void bch_btree_op_init(struct btree_op *op, int write_lock_level) · 246 unmodified lines
247 247
248static inline void rw_lock(bool w, struct btree *b, int level)248static inline void rw_lock(bool w, struct btree *b, int level)
249{249{
250 w ? down_write_nested(&b->lock, level + 1)250 w ? down_write(&b->lock)
251 : down_read_nested(&b->lock, level + 1);251 : down_read(&b->lock);
252 if (w)252 if (w)
253 b->seq++;253 b->seq++;
254}254}
include/asm-generic/atomic.h+0−3
@@ -130,7 +130,4 @@ ATOMIC_OP(xor, ^) · 129 unmodified lines
130#define arch_atomic_read(v) READ_ONCE((v)->counter)130#define arch_atomic_read(v) READ_ONCE((v)->counter)
131#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))131#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
132 132
133#define arch_atomic_xchg(ptr, v) (arch_xchg(&(ptr)->counter, (u32)(v)))
134#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), (u32)(old), (u32)(new)))
135
136#endif /* __ASM_GENERIC_ATOMIC_H */133#endif /* __ASM_GENERIC_ATOMIC_H */
include/asm-generic/bitops/atomic.h+6−6
@@ -15,21 +15,21 @@ static __always_inline void · 14 unmodified lines
15arch_set_bit(unsigned int nr, volatile unsigned long *p)15arch_set_bit(unsigned int nr, volatile unsigned long *p)
16{16{
17 p += BIT_WORD(nr);17 p += BIT_WORD(nr);
18 arch_atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p);18 raw_atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p);
19}19}
20 20
21static __always_inline void21static __always_inline void
22arch_clear_bit(unsigned int nr, volatile unsigned long *p)22arch_clear_bit(unsigned int nr, volatile unsigned long *p)
23{23{
24 p += BIT_WORD(nr);24 p += BIT_WORD(nr);
25 arch_atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p);25 raw_atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p);
26}26}
27 27
28static __always_inline void28static __always_inline void
29arch_change_bit(unsigned int nr, volatile unsigned long *p)29arch_change_bit(unsigned int nr, volatile unsigned long *p)
30{30{
31 p += BIT_WORD(nr);31 p += BIT_WORD(nr);
32 arch_atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p);32 raw_atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p);
33}33}
34 34
35static __always_inline int35static __always_inline int
@@ -39,7 +39,7 @@ arch_test_and_set_bit(unsigned int nr, volatile unsigned long *p) · 3 unmodified lines
39 unsigned long mask = BIT_MASK(nr);39 unsigned long mask = BIT_MASK(nr);
40 40
41 p += BIT_WORD(nr);41 p += BIT_WORD(nr);
42 old = arch_atomic_long_fetch_or(mask, (atomic_long_t *)p);42 old = raw_atomic_long_fetch_or(mask, (atomic_long_t *)p);
43 return !!(old & mask);43 return !!(old & mask);
44}44}
45 45
@@ -50,7 +50,7 @@ arch_test_and_clear_bit(unsigned int nr, volatile unsigned long *p) · 4 unmodified lines
50 unsigned long mask = BIT_MASK(nr);50 unsigned long mask = BIT_MASK(nr);
51 51
52 p += BIT_WORD(nr);52 p += BIT_WORD(nr);
53 old = arch_atomic_long_fetch_andnot(mask, (atomic_long_t *)p);53 old = raw_atomic_long_fetch_andnot(mask, (atomic_long_t *)p);
54 return !!(old & mask);54 return !!(old & mask);
55}55}
56 56
@@ -61,7 +61,7 @@ arch_test_and_change_bit(unsigned int nr, volatile unsigned long *p) · 4 unmodified lines
61 unsigned long mask = BIT_MASK(nr);61 unsigned long mask = BIT_MASK(nr);
62 62
63 p += BIT_WORD(nr);63 p += BIT_WORD(nr);
64 old = arch_atomic_long_fetch_xor(mask, (atomic_long_t *)p);64 old = raw_atomic_long_fetch_xor(mask, (atomic_long_t *)p);
65 return !!(old & mask);65 return !!(old & mask);
66}66}
67 67
include/asm-generic/bitops/lock.h+4−4
@@ -25,7 +25,7 @@ arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p) · 24 unmodified lines
25 if (READ_ONCE(*p) & mask)25 if (READ_ONCE(*p) & mask)
26 return 1;26 return 1;
27 27
28 old = arch_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p);28 old = raw_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p);
29 return !!(old & mask);29 return !!(old & mask);
30}30}
31 31
@@ -41,7 +41,7 @@ static __always_inline void · 9 unmodified lines
41arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p)41arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
42{42{
43 p += BIT_WORD(nr);43 p += BIT_WORD(nr);
44 arch_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p);44 raw_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p);
45}45}
46 46
47/**47/**
@@ -63,7 +63,7 @@ arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *p) · 15 unmodified lines
63 p += BIT_WORD(nr);63 p += BIT_WORD(nr);
64 old = READ_ONCE(*p);64 old = READ_ONCE(*p);
65 old &= ~BIT_MASK(nr);65 old &= ~BIT_MASK(nr);
66 arch_atomic_long_set_release((atomic_long_t *)p, old);66 raw_atomic_long_set_release((atomic_long_t *)p, old);
67}67}
68 68
69/**69/**
@@ -83,7 +83,7 @@ static inline bool arch_clear_bit_unlock_is_negative_byte(unsigned int nr, · 13 unmodified lines
83 unsigned long mask = BIT_MASK(nr);83 unsigned long mask = BIT_MASK(nr);
84 84
85 p += BIT_WORD(nr);85 p += BIT_WORD(nr);
86 old = arch_atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p);86 old = raw_atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p);
87 return !!(old & BIT(7));87 return !!(old & BIT(7));
88}88}
89#define arch_clear_bit_unlock_is_negative_byte arch_clear_bit_unlock_is_negative_byte89#define arch_clear_bit_unlock_is_negative_byte arch_clear_bit_unlock_is_negative_byte
include/asm-generic/percpu.h+152−49
@@ -89,27 +89,35 @@ do { \ · 88 unmodified lines
89 __ret; \89 __ret; \
90})90})
91 91
92#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \92#define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg) \
93({ \
94 typeof(pcp) __val, __old = *(ovalp); \
95 __val = _cmpxchg(pcp, __old, nval); \
96 if (__val != __old) \
97 *(ovalp) = __val; \
98 __val == __old; \
99})
100
101#define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
93({ \102({ \
94 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \103 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
95 typeof(pcp) __ret; \104 typeof(pcp) __val = *__p, ___old = *(ovalp); \
96 __ret = *__p; \105 bool __ret; \
97 if (__ret == (oval)) \106 if (__val == ___old) { \
98 *__p = nval; \107 *__p = nval; \
108 __ret = true; \
109 } else { \
110 *(ovalp) = __val; \
111 __ret = false; \
112 } \
99 __ret; \113 __ret; \
100})114})
101 115
102#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \116#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
103({ \117({ \
104 typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \118 typeof(pcp) __old = (oval); \
105 typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2)); \119 raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \
106 int __ret = 0; \120 __old; \
107 if (*__p1 == (oval1) && *__p2 == (oval2)) { \
108 *__p1 = nval1; \
109 *__p2 = nval2; \
110 __ret = 1; \
111 } \
112 (__ret); \
113})121})
114 122
115#define __this_cpu_generic_read_nopreempt(pcp) \123#define __this_cpu_generic_read_nopreempt(pcp) \
@@ -170,23 +178,22 @@ do { \ · 54 unmodified lines
170 __ret; \178 __ret; \
171})179})
172 180
173#define this_cpu_generic_cmpxchg(pcp, oval, nval) \181#define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
174({ \182({ \
175 typeof(pcp) __ret; \183 bool __ret; \
176 unsigned long __flags; \184 unsigned long __flags; \
177 raw_local_irq_save(__flags); \185 raw_local_irq_save(__flags); \
178 __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \186 __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \
179 raw_local_irq_restore(__flags); \187 raw_local_irq_restore(__flags); \
180 __ret; \188 __ret; \
181})189})
182 190
183#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \191#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
184({ \192({ \
185 int __ret; \193 typeof(pcp) __ret; \
186 unsigned long __flags; \194 unsigned long __flags; \
187 raw_local_irq_save(__flags); \195 raw_local_irq_save(__flags); \
188 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \196 __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
189 oval1, oval2, nval1, nval2); \
190 raw_local_irq_restore(__flags); \197 raw_local_irq_restore(__flags); \
191 __ret; \198 __ret; \
192})199})
@@ -282,6 +289,62 @@ do { \ · 89 unmodified lines
282#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)289#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
283#endif290#endif
284 291
292#ifndef raw_cpu_try_cmpxchg_1
293#ifdef raw_cpu_cmpxchg_1
294#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
295 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1)
296#else
297#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
298 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
299#endif
300#endif
301#ifndef raw_cpu_try_cmpxchg_2
302#ifdef raw_cpu_cmpxchg_2
303#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
304 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2)
305#else
306#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
307 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
308#endif
309#endif
310#ifndef raw_cpu_try_cmpxchg_4
311#ifdef raw_cpu_cmpxchg_4
312#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
313 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4)
314#else
315#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
316 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
317#endif
318#endif
319#ifndef raw_cpu_try_cmpxchg_8
320#ifdef raw_cpu_cmpxchg_8
321#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
322 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8)
323#else
324#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
325 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
326#endif
327#endif
328
329#ifndef raw_cpu_try_cmpxchg64
330#ifdef raw_cpu_cmpxchg64
331#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
332 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64)
333#else
334#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
335 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
336#endif
337#endif
338#ifndef raw_cpu_try_cmpxchg128
339#ifdef raw_cpu_cmpxchg128
340#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
341 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128)
342#else
343#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
344 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
345#endif
346#endif
347
285#ifndef raw_cpu_cmpxchg_1348#ifndef raw_cpu_cmpxchg_1
286#define raw_cpu_cmpxchg_1(pcp, oval, nval) \349#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
287 raw_cpu_generic_cmpxchg(pcp, oval, nval)350 raw_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -299,21 +362,13 @@ do { \ · 11 unmodified lines
299 raw_cpu_generic_cmpxchg(pcp, oval, nval)362 raw_cpu_generic_cmpxchg(pcp, oval, nval)
300#endif363#endif
301 364
302#ifndef raw_cpu_cmpxchg_double_1365#ifndef raw_cpu_cmpxchg64
303#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \366#define raw_cpu_cmpxchg64(pcp, oval, nval) \
304 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)367 raw_cpu_generic_cmpxchg(pcp, oval, nval)
305#endif
306#ifndef raw_cpu_cmpxchg_double_2
307#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
308 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
309#endif
310#ifndef raw_cpu_cmpxchg_double_4
311#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
312 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
313#endif368#endif
314#ifndef raw_cpu_cmpxchg_double_8369#ifndef raw_cpu_cmpxchg128
315#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \370#define raw_cpu_cmpxchg128(pcp, oval, nval) \
316 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)371 raw_cpu_generic_cmpxchg(pcp, oval, nval)
317#endif372#endif
318 373
319#ifndef this_cpu_read_1374#ifndef this_cpu_read_1
@@ -407,6 +462,62 @@ do { \ · 87 unmodified lines
407#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)462#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
408#endif463#endif
409 464
465#ifndef this_cpu_try_cmpxchg_1
466#ifdef this_cpu_cmpxchg_1
467#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
468 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1)
469#else
470#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
471 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
472#endif
473#endif
474#ifndef this_cpu_try_cmpxchg_2
475#ifdef this_cpu_cmpxchg_2
476#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
477 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2)
478#else
479#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
480 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
481#endif
482#endif
483#ifndef this_cpu_try_cmpxchg_4
484#ifdef this_cpu_cmpxchg_4
485#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
486 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4)
487#else
488#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
489 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
490#endif
491#endif
492#ifndef this_cpu_try_cmpxchg_8
493#ifdef this_cpu_cmpxchg_8
494#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
495 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8)
496#else
497#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
498 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
499#endif
500#endif
501
502#ifndef this_cpu_try_cmpxchg64
503#ifdef this_cpu_cmpxchg64
504#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
505 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64)
506#else
507#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
508 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
509#endif
510#endif
511#ifndef this_cpu_try_cmpxchg128
512#ifdef this_cpu_cmpxchg128
513#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
514 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128)
515#else
516#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
517 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
518#endif
519#endif
520
410#ifndef this_cpu_cmpxchg_1521#ifndef this_cpu_cmpxchg_1
411#define this_cpu_cmpxchg_1(pcp, oval, nval) \522#define this_cpu_cmpxchg_1(pcp, oval, nval) \
412 this_cpu_generic_cmpxchg(pcp, oval, nval)523 this_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -424,21 +535,13 @@ do { \ · 11 unmodified lines
424 this_cpu_generic_cmpxchg(pcp, oval, nval)535 this_cpu_generic_cmpxchg(pcp, oval, nval)
425#endif536#endif
426 537
427#ifndef this_cpu_cmpxchg_double_1538#ifndef this_cpu_cmpxchg64
428#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \539#define this_cpu_cmpxchg64(pcp, oval, nval) \
429 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)540 this_cpu_generic_cmpxchg(pcp, oval, nval)
430#endif
431#ifndef this_cpu_cmpxchg_double_2
432#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
433 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
434#endif
435#ifndef this_cpu_cmpxchg_double_4
436#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
437 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
438#endif541#endif
439#ifndef this_cpu_cmpxchg_double_8542#ifndef this_cpu_cmpxchg128
440#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \543#define this_cpu_cmpxchg128(pcp, oval, nval) \
441 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)544 this_cpu_generic_cmpxchg(pcp, oval, nval)
442#endif545#endif
443 546
444#endif /* _ASM_GENERIC_PERCPU_H_ */547#endif /* _ASM_GENERIC_PERCPU_H_ */
include/crypto/b128ops.h+3−11
@@ -49,10 +49,6 @@ · 48 unmodified lines
49 49
50#include <linux/types.h>50#include <linux/types.h>
51 51
52typedef struct {
53 u64 a, b;
54} u128;
55
56typedef struct {52typedef struct {
57 __be64 a, b;53 __be64 a, b;
58} be128;54} be128;
@@ -61,20 +57,16 @@ typedef struct { · 2 unmodified lines
61 __le64 b, a;57 __le64 b, a;
62} le128;58} le128;
63 59
64static inline void u128_xor(u128 *r, const u128 *p, const u128 *q)60static inline void be128_xor(be128 *r, const be128 *p, const be128 *q)
65{61{
66 r->a = p->a ^ q->a;62 r->a = p->a ^ q->a;
67 r->b = p->b ^ q->b;63 r->b = p->b ^ q->b;
68}64}
69 65
70static inline void be128_xor(be128 *r, const be128 *p, const be128 *q)
71{
72 u128_xor((u128 *)r, (u128 *)p, (u128 *)q);
73}
74
75static inline void le128_xor(le128 *r, const le128 *p, const le128 *q)66static inline void le128_xor(le128 *r, const le128 *p, const le128 *q)
76{67{
77 u128_xor((u128 *)r, (u128 *)p, (u128 *)q);68 r->a = p->a ^ q->a;
69 r->b = p->b ^ q->b;
78}70}
79 71
80#endif /* _CRYPTO_B128OPS_H */72#endif /* _CRYPTO_B128OPS_H */
include/linux/atomic/atomic-arch-fallback.h+3576−1587
@@ -8,2664 +8,4653 @@ · 7 unmodified lines
8 8
9#include <linux/compiler.h>9#include <linux/compiler.h>
10 10
11#ifndef arch_xchg_relaxed11#if defined(arch_xchg)
12#define arch_xchg_acquire arch_xchg12#define raw_xchg arch_xchg
13#define arch_xchg_release arch_xchg13#elif defined(arch_xchg_relaxed)
14#define arch_xchg_relaxed arch_xchg14#define raw_xchg(...) \
15#else /* arch_xchg_relaxed */15 __atomic_op_fence(arch_xchg, __VA_ARGS__)
16 16#else
17#ifndef arch_xchg_acquire17extern void raw_xchg_not_implemented(void);
18#define arch_xchg_acquire(...) \18#define raw_xchg(...) raw_xchg_not_implemented()
19 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20#endif19#endif
21 20
22#ifndef arch_xchg_release21#if defined(arch_xchg_acquire)
23#define arch_xchg_release(...) \22#define raw_xchg_acquire arch_xchg_acquire
24 __atomic_op_release(arch_xchg, __VA_ARGS__)23#elif defined(arch_xchg_relaxed)
24#define raw_xchg_acquire(...) \
25 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
26#elif defined(arch_xchg)
27#define raw_xchg_acquire arch_xchg
28#else
29extern void raw_xchg_acquire_not_implemented(void);
30#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
25#endif31#endif
26 32
27#ifndef arch_xchg33#if defined(arch_xchg_release)
28#define arch_xchg(...) \34#define raw_xchg_release arch_xchg_release
29 __atomic_op_fence(arch_xchg, __VA_ARGS__)35#elif defined(arch_xchg_relaxed)
36#define raw_xchg_release(...) \
37 __atomic_op_release(arch_xchg, __VA_ARGS__)
38#elif defined(arch_xchg)
39#define raw_xchg_release arch_xchg
40#else
41extern void raw_xchg_release_not_implemented(void);
42#define raw_xchg_release(...) raw_xchg_release_not_implemented()
43#endif
44
45#if defined(arch_xchg_relaxed)
46#define raw_xchg_relaxed arch_xchg_relaxed
47#elif defined(arch_xchg)
48#define raw_xchg_relaxed arch_xchg
49#else
50extern void raw_xchg_relaxed_not_implemented(void);
51#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
52#endif
53
54#if defined(arch_cmpxchg)
55#define raw_cmpxchg arch_cmpxchg
56#elif defined(arch_cmpxchg_relaxed)
57#define raw_cmpxchg(...) \
58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
59#else
60extern void raw_cmpxchg_not_implemented(void);
61#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
30#endif62#endif
31 63
32#endif /* arch_xchg_relaxed */64#if defined(arch_cmpxchg_acquire)
33 65#define raw_cmpxchg_acquire arch_cmpxchg_acquire
34#ifndef arch_cmpxchg_relaxed66#elif defined(arch_cmpxchg_relaxed)
35#define arch_cmpxchg_acquire arch_cmpxchg67#define raw_cmpxchg_acquire(...) \
36#define arch_cmpxchg_release arch_cmpxchg
37#define arch_cmpxchg_relaxed arch_cmpxchg
38#else /* arch_cmpxchg_relaxed */
39
40#ifndef arch_cmpxchg_acquire
41#define arch_cmpxchg_acquire(...) \
42 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)68 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
69#elif defined(arch_cmpxchg)
70#define raw_cmpxchg_acquire arch_cmpxchg
71#else
72extern void raw_cmpxchg_acquire_not_implemented(void);
73#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
43#endif74#endif
44 75
45#ifndef arch_cmpxchg_release76#if defined(arch_cmpxchg_release)
46#define arch_cmpxchg_release(...) \77#define raw_cmpxchg_release arch_cmpxchg_release
78#elif defined(arch_cmpxchg_relaxed)
79#define raw_cmpxchg_release(...) \
47 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)80 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
81#elif defined(arch_cmpxchg)
82#define raw_cmpxchg_release arch_cmpxchg
83#else
84extern void raw_cmpxchg_release_not_implemented(void);
85#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
86#endif
87
88#if defined(arch_cmpxchg_relaxed)
89#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
90#elif defined(arch_cmpxchg)
91#define raw_cmpxchg_relaxed arch_cmpxchg
92#else
93extern void raw_cmpxchg_relaxed_not_implemented(void);
94#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
95#endif
96
97#if defined(arch_cmpxchg64)
98#define raw_cmpxchg64 arch_cmpxchg64
99#elif defined(arch_cmpxchg64_relaxed)
100#define raw_cmpxchg64(...) \
101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
102#else
103extern void raw_cmpxchg64_not_implemented(void);
104#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
48#endif105#endif
49 106
50#ifndef arch_cmpxchg107#if defined(arch_cmpxchg64_acquire)
51#define arch_cmpxchg(...) \108#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
52 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)109#elif defined(arch_cmpxchg64_relaxed)
53#endif110#define raw_cmpxchg64_acquire(...) \
54
55#endif /* arch_cmpxchg_relaxed */
56
57#ifndef arch_cmpxchg64_relaxed
58#define arch_cmpxchg64_acquire arch_cmpxchg64
59#define arch_cmpxchg64_release arch_cmpxchg64
60#define arch_cmpxchg64_relaxed arch_cmpxchg64
61#else /* arch_cmpxchg64_relaxed */
62
63#ifndef arch_cmpxchg64_acquire
64#define arch_cmpxchg64_acquire(...) \
65 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)111 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
112#elif defined(arch_cmpxchg64)
113#define raw_cmpxchg64_acquire arch_cmpxchg64
114#else
115extern void raw_cmpxchg64_acquire_not_implemented(void);
116#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
66#endif117#endif
67 118
68#ifndef arch_cmpxchg64_release119#if defined(arch_cmpxchg64_release)
69#define arch_cmpxchg64_release(...) \120#define raw_cmpxchg64_release arch_cmpxchg64_release
121#elif defined(arch_cmpxchg64_relaxed)
122#define raw_cmpxchg64_release(...) \
70 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)123 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
124#elif defined(arch_cmpxchg64)
125#define raw_cmpxchg64_release arch_cmpxchg64
126#else
127extern void raw_cmpxchg64_release_not_implemented(void);
128#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
129#endif
130
131#if defined(arch_cmpxchg64_relaxed)
132#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
133#elif defined(arch_cmpxchg64)
134#define raw_cmpxchg64_relaxed arch_cmpxchg64
135#else
136extern void raw_cmpxchg64_relaxed_not_implemented(void);
137#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
138#endif
139
140#if defined(arch_cmpxchg128)
141#define raw_cmpxchg128 arch_cmpxchg128
142#elif defined(arch_cmpxchg128_relaxed)
143#define raw_cmpxchg128(...) \
144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
145#else
146extern void raw_cmpxchg128_not_implemented(void);
147#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
148#endif
149
150#if defined(arch_cmpxchg128_acquire)
151#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
152#elif defined(arch_cmpxchg128_relaxed)
153#define raw_cmpxchg128_acquire(...) \
154 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
155#elif defined(arch_cmpxchg128)
156#define raw_cmpxchg128_acquire arch_cmpxchg128
157#else
158extern void raw_cmpxchg128_acquire_not_implemented(void);
159#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
160#endif
161
162#if defined(arch_cmpxchg128_release)
163#define raw_cmpxchg128_release arch_cmpxchg128_release
164#elif defined(arch_cmpxchg128_relaxed)
165#define raw_cmpxchg128_release(...) \
166 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
167#elif defined(arch_cmpxchg128)
168#define raw_cmpxchg128_release arch_cmpxchg128
169#else
170extern void raw_cmpxchg128_release_not_implemented(void);
171#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
172#endif
173
174#if defined(arch_cmpxchg128_relaxed)
175#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
176#elif defined(arch_cmpxchg128)
177#define raw_cmpxchg128_relaxed arch_cmpxchg128
178#else
179extern void raw_cmpxchg128_relaxed_not_implemented(void);
180#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
181#endif
182
183#if defined(arch_try_cmpxchg)
184#define raw_try_cmpxchg arch_try_cmpxchg
185#elif defined(arch_try_cmpxchg_relaxed)
186#define raw_try_cmpxchg(...) \
187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
188#else
189#define raw_try_cmpxchg(_ptr, _oldp, _new) \
190({ \
191 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
192 ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
193 if (unlikely(___r != ___o)) \
194 *___op = ___r; \
195 likely(___r == ___o); \
196})
71#endif197#endif
72 198
73#ifndef arch_cmpxchg64199#if defined(arch_try_cmpxchg_acquire)
74#define arch_cmpxchg64(...) \200#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
75 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)201#elif defined(arch_try_cmpxchg_relaxed)
202#define raw_try_cmpxchg_acquire(...) \
203 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
204#elif defined(arch_try_cmpxchg)
205#define raw_try_cmpxchg_acquire arch_try_cmpxchg
206#else
207#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
208({ \
209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
210 ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
211 if (unlikely(___r != ___o)) \
212 *___op = ___r; \
213 likely(___r == ___o); \
214})
76#endif215#endif
77 216
78#endif /* arch_cmpxchg64_relaxed */217#if defined(arch_try_cmpxchg_release)
79 218#define raw_try_cmpxchg_release arch_try_cmpxchg_release
80#ifndef arch_try_cmpxchg_relaxed219#elif defined(arch_try_cmpxchg_relaxed)
81#ifdef arch_try_cmpxchg220#define raw_try_cmpxchg_release(...) \
82#define arch_try_cmpxchg_acquire arch_try_cmpxchg221 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
83#define arch_try_cmpxchg_release arch_try_cmpxchg222#elif defined(arch_try_cmpxchg)
84#define arch_try_cmpxchg_relaxed arch_try_cmpxchg223#define raw_try_cmpxchg_release arch_try_cmpxchg
85#endif /* arch_try_cmpxchg */224#else
86 225#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
87#ifndef arch_try_cmpxchg
88#define arch_try_cmpxchg(_ptr, _oldp, _new) \
89({ \226({ \
90 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \227 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
91 ___r = arch_cmpxchg((_ptr), ___o, (_new)); \228 ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
92 if (unlikely(___r != ___o)) \229 if (unlikely(___r != ___o)) \
93 *___op = ___r; \230 *___op = ___r; \
94 likely(___r == ___o); \231 likely(___r == ___o); \
95})232})
96#endif /* arch_try_cmpxchg */233#endif
97 234
98#ifndef arch_try_cmpxchg_acquire235#if defined(arch_try_cmpxchg_relaxed)
99#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \236#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
237#elif defined(arch_try_cmpxchg)
238#define raw_try_cmpxchg_relaxed arch_try_cmpxchg
239#else
240#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
100({ \241({ \
101 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \242 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
102 ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \243 ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
103 if (unlikely(___r != ___o)) \244 if (unlikely(___r != ___o)) \
104 *___op = ___r; \245 *___op = ___r; \
105 likely(___r == ___o); \246 likely(___r == ___o); \
106})247})
107#endif /* arch_try_cmpxchg_acquire */248#endif
108 249
109#ifndef arch_try_cmpxchg_release250#if defined(arch_try_cmpxchg64)
110#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \251#define raw_try_cmpxchg64 arch_try_cmpxchg64
252#elif defined(arch_try_cmpxchg64_relaxed)
253#define raw_try_cmpxchg64(...) \
254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
255#else
256#define raw_try_cmpxchg64(_ptr, _oldp, _new) \
111({ \257({ \
112 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \258 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
113 ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \259 ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
114 if (unlikely(___r != ___o)) \260 if (unlikely(___r != ___o)) \
115 *___op = ___r; \261 *___op = ___r; \
116 likely(___r == ___o); \262 likely(___r == ___o); \
117})263})
118#endif /* arch_try_cmpxchg_release */264#endif
119 265
120#ifndef arch_try_cmpxchg_relaxed266#if defined(arch_try_cmpxchg64_acquire)
121#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \267#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
268#elif defined(arch_try_cmpxchg64_relaxed)
269#define raw_try_cmpxchg64_acquire(...) \
270 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
271#elif defined(arch_try_cmpxchg64)
272#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
273#else
274#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
122({ \275({ \
123 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
124 ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \277 ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
125 if (unlikely(___r != ___o)) \278 if (unlikely(___r != ___o)) \
126 *___op = ___r; \279 *___op = ___r; \
127 likely(___r == ___o); \280 likely(___r == ___o); \
128})281})
129#endif /* arch_try_cmpxchg_relaxed */
130
131#else /* arch_try_cmpxchg_relaxed */
132
133#ifndef arch_try_cmpxchg_acquire
134#define arch_try_cmpxchg_acquire(...) \
135 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
136#endif282#endif
137 283
138#ifndef arch_try_cmpxchg_release284#if defined(arch_try_cmpxchg64_release)
139#define arch_try_cmpxchg_release(...) \285#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
140 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)286#elif defined(arch_try_cmpxchg64_relaxed)
287#define raw_try_cmpxchg64_release(...) \
288 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
289#elif defined(arch_try_cmpxchg64)
290#define raw_try_cmpxchg64_release arch_try_cmpxchg64
291#else
292#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
293({ \
294 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
295 ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
296 if (unlikely(___r != ___o)) \
297 *___op = ___r; \
298 likely(___r == ___o); \
299})
141#endif300#endif
142 301
143#ifndef arch_try_cmpxchg302#if defined(arch_try_cmpxchg64_relaxed)
144#define arch_try_cmpxchg(...) \303#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
145 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)304#elif defined(arch_try_cmpxchg64)
305#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
306#else
307#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
308({ \
309 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
310 ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
311 if (unlikely(___r != ___o)) \
312 *___op = ___r; \
313 likely(___r == ___o); \
314})
146#endif315#endif
147 316
148#endif /* arch_try_cmpxchg_relaxed */317#if defined(arch_try_cmpxchg128)
149 318#define raw_try_cmpxchg128 arch_try_cmpxchg128
150#ifndef arch_try_cmpxchg64_relaxed319#elif defined(arch_try_cmpxchg128_relaxed)
151#ifdef arch_try_cmpxchg64320#define raw_try_cmpxchg128(...) \
152#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
153#define arch_try_cmpxchg64_release arch_try_cmpxchg64322#else
154#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64323#define raw_try_cmpxchg128(_ptr, _oldp, _new) \
155#endif /* arch_try_cmpxchg64 */
156
157#ifndef arch_try_cmpxchg64
158#define arch_try_cmpxchg64(_ptr, _oldp, _new) \
159({ \324({ \
160 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \325 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
161 ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \326 ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
162 if (unlikely(___r != ___o)) \327 if (unlikely(___r != ___o)) \
163 *___op = ___r; \328 *___op = ___r; \
164 likely(___r == ___o); \329 likely(___r == ___o); \
165})330})
166#endif /* arch_try_cmpxchg64 */331#endif
167 332
168#ifndef arch_try_cmpxchg64_acquire333#if defined(arch_try_cmpxchg128_acquire)
169#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \334#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
335#elif defined(arch_try_cmpxchg128_relaxed)
336#define raw_try_cmpxchg128_acquire(...) \
337 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
338#elif defined(arch_try_cmpxchg128)
339#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
340#else
341#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
170({ \342({ \
171 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
172 ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \344 ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
173 if (unlikely(___r != ___o)) \345 if (unlikely(___r != ___o)) \
174 *___op = ___r; \346 *___op = ___r; \
175 likely(___r == ___o); \347 likely(___r == ___o); \
176})348})
177#endif /* arch_try_cmpxchg64_acquire */349#endif
178 350
179#ifndef arch_try_cmpxchg64_release351#if defined(arch_try_cmpxchg128_release)
180#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \352#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
353#elif defined(arch_try_cmpxchg128_relaxed)
354#define raw_try_cmpxchg128_release(...) \
355 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
356#elif defined(arch_try_cmpxchg128)
357#define raw_try_cmpxchg128_release arch_try_cmpxchg128
358#else
359#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
181({ \360({ \
182 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
183 ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \362 ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
184 if (unlikely(___r != ___o)) \363 if (unlikely(___r != ___o)) \
185 *___op = ___r; \364 *___op = ___r; \
186 likely(___r == ___o); \365 likely(___r == ___o); \
187})366})
188#endif /* arch_try_cmpxchg64_release */367#endif
189 368
190#ifndef arch_try_cmpxchg64_relaxed369#if defined(arch_try_cmpxchg128_relaxed)
191#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \370#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
371#elif defined(arch_try_cmpxchg128)
372#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
373#else
374#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
192({ \375({ \
193 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \376 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
194 ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \377 ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
195 if (unlikely(___r != ___o)) \378 if (unlikely(___r != ___o)) \
196 *___op = ___r; \379 *___op = ___r; \
197 likely(___r == ___o); \380 likely(___r == ___o); \
198})381})
199#endif /* arch_try_cmpxchg64_relaxed */
200
201#else /* arch_try_cmpxchg64_relaxed */
202
203#ifndef arch_try_cmpxchg64_acquire
204#define arch_try_cmpxchg64_acquire(...) \
205 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
206#endif382#endif
207 383
208#ifndef arch_try_cmpxchg64_release384#define raw_cmpxchg_local arch_cmpxchg_local
209#define arch_try_cmpxchg64_release(...) \
210 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
211#endif
212 385
213#ifndef arch_try_cmpxchg64386#ifdef arch_try_cmpxchg_local
214#define arch_try_cmpxchg64(...) \387#define raw_try_cmpxchg_local arch_try_cmpxchg_local
215 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)388#else
389#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
390({ \
391 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
392 ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
393 if (unlikely(___r != ___o)) \
394 *___op = ___r; \
395 likely(___r == ___o); \
396})
216#endif397#endif
217 398
218#endif /* arch_try_cmpxchg64_relaxed */399#define raw_cmpxchg64_local arch_cmpxchg64_local
219 400
220#ifndef arch_try_cmpxchg_local401#ifdef arch_try_cmpxchg64_local
221#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \402#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
403#else
404#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
222({ \405({ \
223 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \406 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
224 ___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \407 ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
225 if (unlikely(___r != ___o)) \408 if (unlikely(___r != ___o)) \
226 *___op = ___r; \409 *___op = ___r; \
227 likely(___r == ___o); \410 likely(___r == ___o); \
228})411})
229#endif /* arch_try_cmpxchg_local */412#endif
230 413
231#ifndef arch_try_cmpxchg64_local414#define raw_cmpxchg128_local arch_cmpxchg128_local
232#define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \415
416#ifdef arch_try_cmpxchg128_local
417#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
418#else
419#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
233({ \420({ \
234 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \421 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
235 ___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \422 ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
236 if (unlikely(___r != ___o)) \423 if (unlikely(___r != ___o)) \
237 *___op = ___r; \424 *___op = ___r; \
238 likely(___r == ___o); \425 likely(___r == ___o); \
239})426})
240#endif /* arch_try_cmpxchg64_local */427#endif
428
429#define raw_sync_cmpxchg arch_sync_cmpxchg
430
431/**
432 * raw_atomic_read() - atomic load with relaxed ordering
433 * @v: pointer to atomic_t
434 *
435 * Atomically loads the value of @v with relaxed ordering.
436 *
437 * Safe to use in noinstr code; prefer atomic_read() elsewhere.
438 *
439 * Return: The value loaded from @v.
440 */
441static __always_inline int
442raw_atomic_read(const atomic_t *v)
443{
444 return arch_atomic_read(v);
445}
241 446
242#ifndef arch_atomic_read_acquire447/**
448 * raw_atomic_read_acquire() - atomic load with acquire ordering
449 * @v: pointer to atomic_t
450 *
451 * Atomically loads the value of @v with acquire ordering.
452 *
453 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
454 *
455 * Return: The value loaded from @v.
456 */
243static __always_inline int457static __always_inline int
244arch_atomic_read_acquire(const atomic_t *v)458raw_atomic_read_acquire(const atomic_t *v)
245{459{
460#if defined(arch_atomic_read_acquire)
461 return arch_atomic_read_acquire(v);
462#elif defined(arch_atomic_read)
463 return arch_atomic_read(v);
464#else
246 int ret;465 int ret;
247 466
248 if (__native_word(atomic_t)) {467 if (__native_word(atomic_t)) {
249 ret = smp_load_acquire(&(v)->counter);468 ret = smp_load_acquire(&(v)->counter);
250 } else {469 } else {
251 ret = arch_atomic_read(v);470 ret = raw_atomic_read(v);
252 __atomic_acquire_fence();471 __atomic_acquire_fence();
253 }472 }
254 473
255 return ret;474 return ret;
256}
257#define arch_atomic_read_acquire arch_atomic_read_acquire
258#endif475#endif
476}
477
478/**
479 * raw_atomic_set() - atomic set with relaxed ordering
480 * @v: pointer to atomic_t
481 * @i: int value to assign
482 *
483 * Atomically sets @v to @i with relaxed ordering.
484 *
485 * Safe to use in noinstr code; prefer atomic_set() elsewhere.
486 *
487 * Return: Nothing.
488 */
489static __always_inline void
490raw_atomic_set(atomic_t *v, int i)
491{
492 arch_atomic_set(v, i);
493}
259 494
260#ifndef arch_atomic_set_release495/**
496 * raw_atomic_set_release() - atomic set with release ordering
497 * @v: pointer to atomic_t
498 * @i: int value to assign
499 *
500 * Atomically sets @v to @i with release ordering.
501 *
502 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
503 *
504 * Return: Nothing.
505 */
261static __always_inline void506static __always_inline void
262arch_atomic_set_release(atomic_t *v, int i)507raw_atomic_set_release(atomic_t *v, int i)
263{508{
509#if defined(arch_atomic_set_release)
510 arch_atomic_set_release(v, i);
511#elif defined(arch_atomic_set)
512 arch_atomic_set(v, i);
513#else
264 if (__native_word(atomic_t)) {514 if (__native_word(atomic_t)) {
265 smp_store_release(&(v)->counter, i);515 smp_store_release(&(v)->counter, i);
266 } else {516 } else {
267 __atomic_release_fence();517 __atomic_release_fence();
268 arch_atomic_set(v, i);518 raw_atomic_set(v, i);
269 }519 }
270}
271#define arch_atomic_set_release arch_atomic_set_release
272#endif520#endif
521}
522
523/**
524 * raw_atomic_add() - atomic add with relaxed ordering
525 * @i: int value to add
526 * @v: pointer to atomic_t
527 *
528 * Atomically updates @v to (@v + @i) with relaxed ordering.
529 *
530 * Safe to use in noinstr code; prefer atomic_add() elsewhere.
531 *
532 * Return: Nothing.
533 */
534static __always_inline void
535raw_atomic_add(int i, atomic_t *v)
536{
537 arch_atomic_add(i, v);
538}
273 539
274#ifndef arch_atomic_add_return_relaxed540/**
275#define arch_atomic_add_return_acquire arch_atomic_add_return541 * raw_atomic_add_return() - atomic add with full ordering
276#define arch_atomic_add_return_release arch_atomic_add_return542 * @i: int value to add
277#define arch_atomic_add_return_relaxed arch_atomic_add_return543 * @v: pointer to atomic_t
278#else /* arch_atomic_add_return_relaxed */544 *
545 * Atomically updates @v to (@v + @i) with full ordering.
546 *
547 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
548 *
549 * Return: The updated value of @v.
550 */
551static __always_inline int
552raw_atomic_add_return(int i, atomic_t *v)
553{
554#if defined(arch_atomic_add_return)
555 return arch_atomic_add_return(i, v);
556#elif defined(arch_atomic_add_return_relaxed)
557 int ret;
558 __atomic_pre_full_fence();
559 ret = arch_atomic_add_return_relaxed(i, v);
560 __atomic_post_full_fence();
561 return ret;
562#else
563#error "Unable to define raw_atomic_add_return"
564#endif
565}
279 566
280#ifndef arch_atomic_add_return_acquire567/**
568 * raw_atomic_add_return_acquire() - atomic add with acquire ordering
569 * @i: int value to add
570 * @v: pointer to atomic_t
571 *
572 * Atomically updates @v to (@v + @i) with acquire ordering.
573 *
574 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
575 *
576 * Return: The updated value of @v.
577 */
281static __always_inline int578static __always_inline int
282arch_atomic_add_return_acquire(int i, atomic_t *v)579raw_atomic_add_return_acquire(int i, atomic_t *v)
283{580{
581#if defined(arch_atomic_add_return_acquire)
582 return arch_atomic_add_return_acquire(i, v);
583#elif defined(arch_atomic_add_return_relaxed)
284 int ret = arch_atomic_add_return_relaxed(i, v);584 int ret = arch_atomic_add_return_relaxed(i, v);
285 __atomic_acquire_fence();585 __atomic_acquire_fence();
286 return ret;586 return ret;
287}587#elif defined(arch_atomic_add_return)
288#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire588 return arch_atomic_add_return(i, v);
589#else
590#error "Unable to define raw_atomic_add_return_acquire"
289#endif591#endif
592}
290 593
291#ifndef arch_atomic_add_return_release594/**
595 * raw_atomic_add_return_release() - atomic add with release ordering
596 * @i: int value to add
597 * @v: pointer to atomic_t
598 *
599 * Atomically updates @v to (@v + @i) with release ordering.
600 *
601 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
602 *
603 * Return: The updated value of @v.
604 */
292static __always_inline int605static __always_inline int
293arch_atomic_add_return_release(int i, atomic_t *v)606raw_atomic_add_return_release(int i, atomic_t *v)
294{607{
608#if defined(arch_atomic_add_return_release)
609 return arch_atomic_add_return_release(i, v);
610#elif defined(arch_atomic_add_return_relaxed)
295 __atomic_release_fence();611 __atomic_release_fence();
296 return arch_atomic_add_return_relaxed(i, v);612 return arch_atomic_add_return_relaxed(i, v);
613#elif defined(arch_atomic_add_return)
614 return arch_atomic_add_return(i, v);
615#else
616#error "Unable to define raw_atomic_add_return_release"
617#endif
297}618}
298#define arch_atomic_add_return_release arch_atomic_add_return_release619
620/**
621 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
622 * @i: int value to add
623 * @v: pointer to atomic_t
624 *
625 * Atomically updates @v to (@v + @i) with relaxed ordering.
626 *
627 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
628 *
629 * Return: The updated value of @v.
630 */
631static __always_inline int
632raw_atomic_add_return_relaxed(int i, atomic_t *v)
633{
634#if defined(arch_atomic_add_return_relaxed)
635 return arch_atomic_add_return_relaxed(i, v);
636#elif defined(arch_atomic_add_return)
637 return arch_atomic_add_return(i, v);
638#else
639#error "Unable to define raw_atomic_add_return_relaxed"
299#endif640#endif
641}
300 642
301#ifndef arch_atomic_add_return643/**
644 * raw_atomic_fetch_add() - atomic add with full ordering
645 * @i: int value to add
646 * @v: pointer to atomic_t
647 *
648 * Atomically updates @v to (@v + @i) with full ordering.
649 *
650 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
651 *
652 * Return: The original value of @v.
653 */
302static __always_inline int654static __always_inline int
303arch_atomic_add_return(int i, atomic_t *v)655raw_atomic_fetch_add(int i, atomic_t *v)
304{656{
657#if defined(arch_atomic_fetch_add)
658 return arch_atomic_fetch_add(i, v);
659#elif defined(arch_atomic_fetch_add_relaxed)
305 int ret;660 int ret;
306 __atomic_pre_full_fence();661 __atomic_pre_full_fence();
307 ret = arch_atomic_add_return_relaxed(i, v);662 ret = arch_atomic_fetch_add_relaxed(i, v);
308 __atomic_post_full_fence();663 __atomic_post_full_fence();
309 return ret;664 return ret;
310}665#else
311#define arch_atomic_add_return arch_atomic_add_return666#error "Unable to define raw_atomic_fetch_add"
312#endif667#endif
668}
313 669
314#endif /* arch_atomic_add_return_relaxed */670/**
315 671 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
316#ifndef arch_atomic_fetch_add_relaxed672 * @i: int value to add
317#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add673 * @v: pointer to atomic_t
318#define arch_atomic_fetch_add_release arch_atomic_fetch_add674 *
319#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add675 * Atomically updates @v to (@v + @i) with acquire ordering.
320#else /* arch_atomic_fetch_add_relaxed */676 *
321 677 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
322#ifndef arch_atomic_fetch_add_acquire678 *
679 * Return: The original value of @v.
680 */
323static __always_inline int681static __always_inline int
324arch_atomic_fetch_add_acquire(int i, atomic_t *v)682raw_atomic_fetch_add_acquire(int i, atomic_t *v)
325{683{
684#if defined(arch_atomic_fetch_add_acquire)
685 return arch_atomic_fetch_add_acquire(i, v);
686#elif defined(arch_atomic_fetch_add_relaxed)
326 int ret = arch_atomic_fetch_add_relaxed(i, v);687 int ret = arch_atomic_fetch_add_relaxed(i, v);
327 __atomic_acquire_fence();688 __atomic_acquire_fence();
328 return ret;689 return ret;
329}690#elif defined(arch_atomic_fetch_add)
330#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire691 return arch_atomic_fetch_add(i, v);
692#else
693#error "Unable to define raw_atomic_fetch_add_acquire"
331#endif694#endif
695}
332 696
333#ifndef arch_atomic_fetch_add_release697/**
698 * raw_atomic_fetch_add_release() - atomic add with release ordering
699 * @i: int value to add
700 * @v: pointer to atomic_t
701 *
702 * Atomically updates @v to (@v + @i) with release ordering.
703 *
704 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
705 *
706 * Return: The original value of @v.
707 */
334static __always_inline int708static __always_inline int
335arch_atomic_fetch_add_release(int i, atomic_t *v)709raw_atomic_fetch_add_release(int i, atomic_t *v)
336{710{
711#if defined(arch_atomic_fetch_add_release)
712 return arch_atomic_fetch_add_release(i, v);
713#elif defined(arch_atomic_fetch_add_relaxed)
337 __atomic_release_fence();714 __atomic_release_fence();
338 return arch_atomic_fetch_add_relaxed(i, v);715 return arch_atomic_fetch_add_relaxed(i, v);
716#elif defined(arch_atomic_fetch_add)
717 return arch_atomic_fetch_add(i, v);
718#else
719#error "Unable to define raw_atomic_fetch_add_release"
720#endif
339}721}
340#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release722
723/**
724 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
725 * @i: int value to add
726 * @v: pointer to atomic_t
727 *
728 * Atomically updates @v to (@v + @i) with relaxed ordering.
729 *
730 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
731 *
732 * Return: The original value of @v.
733 */
734static __always_inline int
735raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
736{
737#if defined(arch_atomic_fetch_add_relaxed)
738 return arch_atomic_fetch_add_relaxed(i, v);
739#elif defined(arch_atomic_fetch_add)
740 return arch_atomic_fetch_add(i, v);
741#else
742#error "Unable to define raw_atomic_fetch_add_relaxed"
341#endif743#endif
744}
745
746/**
747 * raw_atomic_sub() - atomic subtract with relaxed ordering
748 * @i: int value to subtract
749 * @v: pointer to atomic_t
750 *
751 * Atomically updates @v to (@v - @i) with relaxed ordering.
752 *
753 * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
754 *
755 * Return: Nothing.
756 */
757static __always_inline void
758raw_atomic_sub(int i, atomic_t *v)
759{
760 arch_atomic_sub(i, v);
761}
342 762
343#ifndef arch_atomic_fetch_add763/**
764 * raw_atomic_sub_return() - atomic subtract with full ordering
765 * @i: int value to subtract
766 * @v: pointer to atomic_t
767 *
768 * Atomically updates @v to (@v - @i) with full ordering.
769 *
770 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
771 *
772 * Return: The updated value of @v.
773 */
344static __always_inline int774static __always_inline int
345arch_atomic_fetch_add(int i, atomic_t *v)775raw_atomic_sub_return(int i, atomic_t *v)
346{776{
777#if defined(arch_atomic_sub_return)
778 return arch_atomic_sub_return(i, v);
779#elif defined(arch_atomic_sub_return_relaxed)
347 int ret;780 int ret;
348 __atomic_pre_full_fence();781 __atomic_pre_full_fence();
349 ret = arch_atomic_fetch_add_relaxed(i, v);782 ret = arch_atomic_sub_return_relaxed(i, v);
350 __atomic_post_full_fence();783 __atomic_post_full_fence();
351 return ret;784 return ret;
352}785#else
353#define arch_atomic_fetch_add arch_atomic_fetch_add786#error "Unable to define raw_atomic_sub_return"
354#endif787#endif
788}
355 789
356#endif /* arch_atomic_fetch_add_relaxed */790/**
357 791 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
358#ifndef arch_atomic_sub_return_relaxed792 * @i: int value to subtract
359#define arch_atomic_sub_return_acquire arch_atomic_sub_return793 * @v: pointer to atomic_t
360#define arch_atomic_sub_return_release arch_atomic_sub_return794 *
361#define arch_atomic_sub_return_relaxed arch_atomic_sub_return795 * Atomically updates @v to (@v - @i) with acquire ordering.
362#else /* arch_atomic_sub_return_relaxed */796 *
363 797 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
364#ifndef arch_atomic_sub_return_acquire798 *
799 * Return: The updated value of @v.
800 */
365static __always_inline int801static __always_inline int
366arch_atomic_sub_return_acquire(int i, atomic_t *v)802raw_atomic_sub_return_acquire(int i, atomic_t *v)
367{803{
804#if defined(arch_atomic_sub_return_acquire)
805 return arch_atomic_sub_return_acquire(i, v);
806#elif defined(arch_atomic_sub_return_relaxed)
368 int ret = arch_atomic_sub_return_relaxed(i, v);807 int ret = arch_atomic_sub_return_relaxed(i, v);
369 __atomic_acquire_fence();808 __atomic_acquire_fence();
370 return ret;809 return ret;
371}810#elif defined(arch_atomic_sub_return)
372#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire811 return arch_atomic_sub_return(i, v);
812#else
813#error "Unable to define raw_atomic_sub_return_acquire"
373#endif814#endif
815}
374 816
375#ifndef arch_atomic_sub_return_release817/**
818 * raw_atomic_sub_return_release() - atomic subtract with release ordering
819 * @i: int value to subtract
820 * @v: pointer to atomic_t
821 *
822 * Atomically updates @v to (@v - @i) with release ordering.
823 *
824 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
825 *
826 * Return: The updated value of @v.
827 */
376static __always_inline int828static __always_inline int
377arch_atomic_sub_return_release(int i, atomic_t *v)829raw_atomic_sub_return_release(int i, atomic_t *v)
378{830{
831#if defined(arch_atomic_sub_return_release)
832 return arch_atomic_sub_return_release(i, v);
833#elif defined(arch_atomic_sub_return_relaxed)
379 __atomic_release_fence();834 __atomic_release_fence();
380 return arch_atomic_sub_return_relaxed(i, v);835 return arch_atomic_sub_return_relaxed(i, v);
836#elif defined(arch_atomic_sub_return)
837 return arch_atomic_sub_return(i, v);
838#else
839#error "Unable to define raw_atomic_sub_return_release"
840#endif
381}841}
382#define arch_atomic_sub_return_release arch_atomic_sub_return_release842
843/**
844 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
845 * @i: int value to subtract
846 * @v: pointer to atomic_t
847 *
848 * Atomically updates @v to (@v - @i) with relaxed ordering.
849 *
850 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
851 *
852 * Return: The updated value of @v.
853 */
854static __always_inline int
855raw_atomic_sub_return_relaxed(int i, atomic_t *v)
856{
857#if defined(arch_atomic_sub_return_relaxed)
858 return arch_atomic_sub_return_relaxed(i, v);
859#elif defined(arch_atomic_sub_return)
860 return arch_atomic_sub_return(i, v);
861#else
862#error "Unable to define raw_atomic_sub_return_relaxed"
383#endif863#endif
864}
384 865
385#ifndef arch_atomic_sub_return866/**
867 * raw_atomic_fetch_sub() - atomic subtract with full ordering
868 * @i: int value to subtract
869 * @v: pointer to atomic_t
870 *
871 * Atomically updates @v to (@v - @i) with full ordering.
872 *
873 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
874 *
875 * Return: The original value of @v.
876 */
386static __always_inline int877static __always_inline int
387arch_atomic_sub_return(int i, atomic_t *v)878raw_atomic_fetch_sub(int i, atomic_t *v)
388{879{
880#if defined(arch_atomic_fetch_sub)
881 return arch_atomic_fetch_sub(i, v);
882#elif defined(arch_atomic_fetch_sub_relaxed)
389 int ret;883 int ret;
390 __atomic_pre_full_fence();884 __atomic_pre_full_fence();
391 ret = arch_atomic_sub_return_relaxed(i, v);885 ret = arch_atomic_fetch_sub_relaxed(i, v);
392 __atomic_post_full_fence();886 __atomic_post_full_fence();
393 return ret;887 return ret;
394}888#else
395#define arch_atomic_sub_return arch_atomic_sub_return889#error "Unable to define raw_atomic_fetch_sub"
396#endif890#endif
891}
397 892
398#endif /* arch_atomic_sub_return_relaxed */893/**
399 894 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
400#ifndef arch_atomic_fetch_sub_relaxed895 * @i: int value to subtract
401#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub896 * @v: pointer to atomic_t
402#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub897 *
403#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub898 * Atomically updates @v to (@v - @i) with acquire ordering.
404#else /* arch_atomic_fetch_sub_relaxed */899 *
405 900 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
406#ifndef arch_atomic_fetch_sub_acquire901 *
902 * Return: The original value of @v.
903 */
407static __always_inline int904static __always_inline int
408arch_atomic_fetch_sub_acquire(int i, atomic_t *v)905raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
409{906{
907#if defined(arch_atomic_fetch_sub_acquire)
908 return arch_atomic_fetch_sub_acquire(i, v);
909#elif defined(arch_atomic_fetch_sub_relaxed)
410 int ret = arch_atomic_fetch_sub_relaxed(i, v);910 int ret = arch_atomic_fetch_sub_relaxed(i, v);
411 __atomic_acquire_fence();911 __atomic_acquire_fence();
412 return ret;912 return ret;
413}913#elif defined(arch_atomic_fetch_sub)
414#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire914 return arch_atomic_fetch_sub(i, v);
915#else
916#error "Unable to define raw_atomic_fetch_sub_acquire"
415#endif917#endif
918}
416 919
417#ifndef arch_atomic_fetch_sub_release920/**
921 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
922 * @i: int value to subtract
923 * @v: pointer to atomic_t
924 *
925 * Atomically updates @v to (@v - @i) with release ordering.
926 *
927 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
928 *
929 * Return: The original value of @v.
930 */
418static __always_inline int931static __always_inline int
419arch_atomic_fetch_sub_release(int i, atomic_t *v)932raw_atomic_fetch_sub_release(int i, atomic_t *v)
420{933{
934#if defined(arch_atomic_fetch_sub_release)
935 return arch_atomic_fetch_sub_release(i, v);
936#elif defined(arch_atomic_fetch_sub_relaxed)
421 __atomic_release_fence();937 __atomic_release_fence();
422 return arch_atomic_fetch_sub_relaxed(i, v);938 return arch_atomic_fetch_sub_relaxed(i, v);
423}939#elif defined(arch_atomic_fetch_sub)
424#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release940 return arch_atomic_fetch_sub(i, v);
941#else
942#error "Unable to define raw_atomic_fetch_sub_release"
425#endif943#endif
944}
426 945
427#ifndef arch_atomic_fetch_sub946/**
947 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
948 * @i: int value to subtract
949 * @v: pointer to atomic_t
950 *
951 * Atomically updates @v to (@v - @i) with relaxed ordering.
952 *
953 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
954 *
955 * Return: The original value of @v.
956 */
428static __always_inline int957static __always_inline int
429arch_atomic_fetch_sub(int i, atomic_t *v)958raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
430{959{
431 int ret;960#if defined(arch_atomic_fetch_sub_relaxed)
432 __atomic_pre_full_fence();961 return arch_atomic_fetch_sub_relaxed(i, v);
433 ret = arch_atomic_fetch_sub_relaxed(i, v);962#elif defined(arch_atomic_fetch_sub)
434 __atomic_post_full_fence();963 return arch_atomic_fetch_sub(i, v);
435 return ret;964#else
436}965#error "Unable to define raw_atomic_fetch_sub_relaxed"
437#define arch_atomic_fetch_sub arch_atomic_fetch_sub
438#endif966#endif
439
440#endif /* arch_atomic_fetch_sub_relaxed */
441
442#ifndef arch_atomic_inc
443static __always_inline void
444arch_atomic_inc(atomic_t *v)
445{
446 arch_atomic_add(1, v);
447}967}
448#define arch_atomic_inc arch_atomic_inc
449#endif
450
451#ifndef arch_atomic_inc_return_relaxed
452#ifdef arch_atomic_inc_return
453#define arch_atomic_inc_return_acquire arch_atomic_inc_return
454#define arch_atomic_inc_return_release arch_atomic_inc_return
455#define arch_atomic_inc_return_relaxed arch_atomic_inc_return
456#endif /* arch_atomic_inc_return */
457 968
458#ifndef arch_atomic_inc_return969/**
459static __always_inline int970 * raw_atomic_inc() - atomic increment with relaxed ordering
460arch_atomic_inc_return(atomic_t *v)971 * @v: pointer to atomic_t
972 *
973 * Atomically updates @v to (@v + 1) with relaxed ordering.
974 *
975 * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
976 *
977 * Return: Nothing.
978 */
979static __always_inline void
980raw_atomic_inc(atomic_t *v)
461{981{
462 return arch_atomic_add_return(1, v);982#if defined(arch_atomic_inc)
463}983 arch_atomic_inc(v);
464#define arch_atomic_inc_return arch_atomic_inc_return984#else
985 raw_atomic_add(1, v);
465#endif986#endif
466
467#ifndef arch_atomic_inc_return_acquire
468static __always_inline int
469arch_atomic_inc_return_acquire(atomic_t *v)
470{
471 return arch_atomic_add_return_acquire(1, v);
472}987}
473#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
474#endif
475 988
476#ifndef arch_atomic_inc_return_release989/**
990 * raw_atomic_inc_return() - atomic increment with full ordering
991 * @v: pointer to atomic_t
992 *
993 * Atomically updates @v to (@v + 1) with full ordering.
994 *
995 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
996 *
997 * Return: The updated value of @v.
998 */
477static __always_inline int999static __always_inline int
478arch_atomic_inc_return_release(atomic_t *v)1000raw_atomic_inc_return(atomic_t *v)
479{1001{
480 return arch_atomic_add_return_release(1, v);1002#if defined(arch_atomic_inc_return)
481}1003 return arch_atomic_inc_return(v);
482#define arch_atomic_inc_return_release arch_atomic_inc_return_release1004#elif defined(arch_atomic_inc_return_relaxed)
1005 int ret;
1006 __atomic_pre_full_fence();
1007 ret = arch_atomic_inc_return_relaxed(v);
1008 __atomic_post_full_fence();
1009 return ret;
1010#else
1011 return raw_atomic_add_return(1, v);
483#endif1012#endif
484
485#ifndef arch_atomic_inc_return_relaxed
486static __always_inline int
487arch_atomic_inc_return_relaxed(atomic_t *v)
488{
489 return arch_atomic_add_return_relaxed(1, v);
490}1013}
491#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
492#endif
493 1014
494#else /* arch_atomic_inc_return_relaxed */1015/**
495 1016 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
496#ifndef arch_atomic_inc_return_acquire1017 * @v: pointer to atomic_t
1018 *
1019 * Atomically updates @v to (@v + 1) with acquire ordering.
1020 *
1021 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
1022 *
1023 * Return: The updated value of @v.
1024 */
497static __always_inline int1025static __always_inline int
498arch_atomic_inc_return_acquire(atomic_t *v)1026raw_atomic_inc_return_acquire(atomic_t *v)
499{1027{
1028#if defined(arch_atomic_inc_return_acquire)
1029 return arch_atomic_inc_return_acquire(v);
1030#elif defined(arch_atomic_inc_return_relaxed)
500 int ret = arch_atomic_inc_return_relaxed(v);1031 int ret = arch_atomic_inc_return_relaxed(v);
501 __atomic_acquire_fence();1032 __atomic_acquire_fence();
502 return ret;1033 return ret;
503}1034#elif defined(arch_atomic_inc_return)
504#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire1035 return arch_atomic_inc_return(v);
1036#else
1037 return raw_atomic_add_return_acquire(1, v);
505#endif1038#endif
1039}
506 1040
507#ifndef arch_atomic_inc_return_release1041/**
1042 * raw_atomic_inc_return_release() - atomic increment with release ordering
1043 * @v: pointer to atomic_t
1044 *
1045 * Atomically updates @v to (@v + 1) with release ordering.
1046 *
1047 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
1048 *
1049 * Return: The updated value of @v.
1050 */
508static __always_inline int1051static __always_inline int
509arch_atomic_inc_return_release(atomic_t *v)1052raw_atomic_inc_return_release(atomic_t *v)
510{1053{
1054#if defined(arch_atomic_inc_return_release)
1055 return arch_atomic_inc_return_release(v);
1056#elif defined(arch_atomic_inc_return_relaxed)
511 __atomic_release_fence();1057 __atomic_release_fence();
512 return arch_atomic_inc_return_relaxed(v);1058 return arch_atomic_inc_return_relaxed(v);
513}1059#elif defined(arch_atomic_inc_return)
514#define arch_atomic_inc_return_release arch_atomic_inc_return_release1060 return arch_atomic_inc_return(v);
1061#else
1062 return raw_atomic_add_return_release(1, v);
515#endif1063#endif
516
517#ifndef arch_atomic_inc_return
518static __always_inline int
519arch_atomic_inc_return(atomic_t *v)
520{
521 int ret;
522 __atomic_pre_full_fence();
523 ret = arch_atomic_inc_return_relaxed(v);
524 __atomic_post_full_fence();
525 return ret;
526}1064}
527#define arch_atomic_inc_return arch_atomic_inc_return
528#endif
529
530#endif /* arch_atomic_inc_return_relaxed */
531 1065
532#ifndef arch_atomic_fetch_inc_relaxed1066/**
533#ifdef arch_atomic_fetch_inc1067 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
534#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc1068 * @v: pointer to atomic_t
535#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc1069 *
536#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc1070 * Atomically updates @v to (@v + 1) with relaxed ordering.
537#endif /* arch_atomic_fetch_inc */1071 *
538 1072 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
539#ifndef arch_atomic_fetch_inc1073 *
1074 * Return: The updated value of @v.
1075 */
540static __always_inline int1076static __always_inline int
541arch_atomic_fetch_inc(atomic_t *v)1077raw_atomic_inc_return_relaxed(atomic_t *v)
542{1078{
543 return arch_atomic_fetch_add(1, v);1079#if defined(arch_atomic_inc_return_relaxed)
544}1080 return arch_atomic_inc_return_relaxed(v);
545#define arch_atomic_fetch_inc arch_atomic_fetch_inc1081#elif defined(arch_atomic_inc_return)
1082 return arch_atomic_inc_return(v);
1083#else
1084 return raw_atomic_add_return_relaxed(1, v);
546#endif1085#endif
547
548#ifndef arch_atomic_fetch_inc_acquire
549static __always_inline int
550arch_atomic_fetch_inc_acquire(atomic_t *v)
551{
552 return arch_atomic_fetch_add_acquire(1, v);
553}1086}
554#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
555#endif
556 1087
557#ifndef arch_atomic_fetch_inc_release1088/**
1089 * raw_atomic_fetch_inc() - atomic increment with full ordering
1090 * @v: pointer to atomic_t
1091 *
1092 * Atomically updates @v to (@v + 1) with full ordering.
1093 *
1094 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
1095 *
1096 * Return: The original value of @v.
1097 */
558static __always_inline int1098static __always_inline int
559arch_atomic_fetch_inc_release(atomic_t *v)1099raw_atomic_fetch_inc(atomic_t *v)
560{1100{
561 return arch_atomic_fetch_add_release(1, v);1101#if defined(arch_atomic_fetch_inc)
562}1102 return arch_atomic_fetch_inc(v);
563#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release1103#elif defined(arch_atomic_fetch_inc_relaxed)
1104 int ret;
1105 __atomic_pre_full_fence();
1106 ret = arch_atomic_fetch_inc_relaxed(v);
1107 __atomic_post_full_fence();
1108 return ret;
1109#else
1110 return raw_atomic_fetch_add(1, v);
564#endif1111#endif
565
566#ifndef arch_atomic_fetch_inc_relaxed
567static __always_inline int
568arch_atomic_fetch_inc_relaxed(atomic_t *v)
569{
570 return arch_atomic_fetch_add_relaxed(1, v);
571}1112}
572#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
573#endif
574 1113
575#else /* arch_atomic_fetch_inc_relaxed */1114/**
576 1115 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
577#ifndef arch_atomic_fetch_inc_acquire1116 * @v: pointer to atomic_t
1117 *
1118 * Atomically updates @v to (@v + 1) with acquire ordering.
1119 *
1120 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
1121 *
1122 * Return: The original value of @v.
1123 */
578static __always_inline int1124static __always_inline int
579arch_atomic_fetch_inc_acquire(atomic_t *v)1125raw_atomic_fetch_inc_acquire(atomic_t *v)
580{1126{
1127#if defined(arch_atomic_fetch_inc_acquire)
1128 return arch_atomic_fetch_inc_acquire(v);
1129#elif defined(arch_atomic_fetch_inc_relaxed)
581 int ret = arch_atomic_fetch_inc_relaxed(v);1130 int ret = arch_atomic_fetch_inc_relaxed(v);
582 __atomic_acquire_fence();1131 __atomic_acquire_fence();
583 return ret;1132 return ret;
584}1133#elif defined(arch_atomic_fetch_inc)
585#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire1134 return arch_atomic_fetch_inc(v);
1135#else
1136 return raw_atomic_fetch_add_acquire(1, v);
586#endif1137#endif
1138}
587 1139
588#ifndef arch_atomic_fetch_inc_release1140/**
1141 * raw_atomic_fetch_inc_release() - atomic increment with release ordering
1142 * @v: pointer to atomic_t
1143 *
1144 * Atomically updates @v to (@v + 1) with release ordering.
1145 *
1146 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
1147 *
1148 * Return: The original value of @v.
1149 */
589static __always_inline int1150static __always_inline int
590arch_atomic_fetch_inc_release(atomic_t *v)1151raw_atomic_fetch_inc_release(atomic_t *v)
591{1152{
1153#if defined(arch_atomic_fetch_inc_release)
1154 return arch_atomic_fetch_inc_release(v);
1155#elif defined(arch_atomic_fetch_inc_relaxed)
592 __atomic_release_fence();1156 __atomic_release_fence();
593 return arch_atomic_fetch_inc_relaxed(v);1157 return arch_atomic_fetch_inc_relaxed(v);
594}1158#elif defined(arch_atomic_fetch_inc)
595#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release1159 return arch_atomic_fetch_inc(v);
1160#else
1161 return raw_atomic_fetch_add_release(1, v);
596#endif1162#endif
1163}
597 1164
598#ifndef arch_atomic_fetch_inc1165/**
1166 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
1167 * @v: pointer to atomic_t
1168 *
1169 * Atomically updates @v to (@v + 1) with relaxed ordering.
1170 *
1171 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
1172 *
1173 * Return: The original value of @v.
1174 */
599static __always_inline int1175static __always_inline int
600arch_atomic_fetch_inc(atomic_t *v)1176raw_atomic_fetch_inc_relaxed(atomic_t *v)
601{1177{
602 int ret;1178#if defined(arch_atomic_fetch_inc_relaxed)
603 __atomic_pre_full_fence();1179 return arch_atomic_fetch_inc_relaxed(v);
604 ret = arch_atomic_fetch_inc_relaxed(v);1180#elif defined(arch_atomic_fetch_inc)
605 __atomic_post_full_fence();1181 return arch_atomic_fetch_inc(v);
606 return ret;1182#else
607}1183 return raw_atomic_fetch_add_relaxed(1, v);
608#define arch_atomic_fetch_inc arch_atomic_fetch_inc
609#endif1184#endif
610
611#endif /* arch_atomic_fetch_inc_relaxed */
612
613#ifndef arch_atomic_dec
614static __always_inline void
615arch_atomic_dec(atomic_t *v)
616{
617 arch_atomic_sub(1, v);
618}1185}
619#define arch_atomic_dec arch_atomic_dec
620#endif
621
622#ifndef arch_atomic_dec_return_relaxed
623#ifdef arch_atomic_dec_return
624#define arch_atomic_dec_return_acquire arch_atomic_dec_return
625#define arch_atomic_dec_return_release arch_atomic_dec_return
626#define arch_atomic_dec_return_relaxed arch_atomic_dec_return
627#endif /* arch_atomic_dec_return */
628 1186
629#ifndef arch_atomic_dec_return1187/**
630static __always_inline int1188 * raw_atomic_dec() - atomic decrement with relaxed ordering
631arch_atomic_dec_return(atomic_t *v)1189 * @v: pointer to atomic_t
1190 *
1191 * Atomically updates @v to (@v - 1) with relaxed ordering.
1192 *
1193 * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
1194 *
1195 * Return: Nothing.
1196 */
1197static __always_inline void
1198raw_atomic_dec(atomic_t *v)
632{1199{
633 return arch_atomic_sub_return(1, v);1200#if defined(arch_atomic_dec)
634}1201 arch_atomic_dec(v);
635#define arch_atomic_dec_return arch_atomic_dec_return1202#else
1203 raw_atomic_sub(1, v);
636#endif1204#endif
637
638#ifndef arch_atomic_dec_return_acquire
639static __always_inline int
640arch_atomic_dec_return_acquire(atomic_t *v)
641{
642 return arch_atomic_sub_return_acquire(1, v);
643}1205}
644#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
645#endif
646 1206
647#ifndef arch_atomic_dec_return_release1207/**
1208 * raw_atomic_dec_return() - atomic decrement with full ordering
1209 * @v: pointer to atomic_t
1210 *
1211 * Atomically updates @v to (@v - 1) with full ordering.
1212 *
1213 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
1214 *
1215 * Return: The updated value of @v.
1216 */
648static __always_inline int1217static __always_inline int
649arch_atomic_dec_return_release(atomic_t *v)1218raw_atomic_dec_return(atomic_t *v)
650{1219{
651 return arch_atomic_sub_return_release(1, v);1220#if defined(arch_atomic_dec_return)
652}1221 return arch_atomic_dec_return(v);
653#define arch_atomic_dec_return_release arch_atomic_dec_return_release1222#elif defined(arch_atomic_dec_return_relaxed)
1223 int ret;
1224 __atomic_pre_full_fence();
1225 ret = arch_atomic_dec_return_relaxed(v);
1226 __atomic_post_full_fence();
1227 return ret;
1228#else
1229 return raw_atomic_sub_return(1, v);
654#endif1230#endif
655
656#ifndef arch_atomic_dec_return_relaxed
657static __always_inline int
658arch_atomic_dec_return_relaxed(atomic_t *v)
659{
660 return arch_atomic_sub_return_relaxed(1, v);
661}1231}
662#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
663#endif
664 1232
665#else /* arch_atomic_dec_return_relaxed */1233/**
666 1234 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
667#ifndef arch_atomic_dec_return_acquire1235 * @v: pointer to atomic_t
1236 *
1237 * Atomically updates @v to (@v - 1) with acquire ordering.
1238 *
1239 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
1240 *
1241 * Return: The updated value of @v.
1242 */
668static __always_inline int1243static __always_inline int
669arch_atomic_dec_return_acquire(atomic_t *v)1244raw_atomic_dec_return_acquire(atomic_t *v)
670{1245{
1246#if defined(arch_atomic_dec_return_acquire)
1247 return arch_atomic_dec_return_acquire(v);
1248#elif defined(arch_atomic_dec_return_relaxed)
671 int ret = arch_atomic_dec_return_relaxed(v);1249 int ret = arch_atomic_dec_return_relaxed(v);
672 __atomic_acquire_fence();1250 __atomic_acquire_fence();
673 return ret;1251 return ret;
674}1252#elif defined(arch_atomic_dec_return)
675#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire1253 return arch_atomic_dec_return(v);
1254#else
1255 return raw_atomic_sub_return_acquire(1, v);
676#endif1256#endif
1257}
677 1258
678#ifndef arch_atomic_dec_return_release1259/**
1260 * raw_atomic_dec_return_release() - atomic decrement with release ordering
1261 * @v: pointer to atomic_t
1262 *
1263 * Atomically updates @v to (@v - 1) with release ordering.
1264 *
1265 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
1266 *
1267 * Return: The updated value of @v.
1268 */
679static __always_inline int1269static __always_inline int
680arch_atomic_dec_return_release(atomic_t *v)1270raw_atomic_dec_return_release(atomic_t *v)
681{1271{
1272#if defined(arch_atomic_dec_return_release)
1273 return arch_atomic_dec_return_release(v);
1274#elif defined(arch_atomic_dec_return_relaxed)
682 __atomic_release_fence();1275 __atomic_release_fence();
683 return arch_atomic_dec_return_relaxed(v);1276 return arch_atomic_dec_return_relaxed(v);
1277#elif defined(arch_atomic_dec_return)
1278 return arch_atomic_dec_return(v);
1279#else
1280 return raw_atomic_sub_return_release(1, v);
1281#endif
684}1282}
685#define arch_atomic_dec_return_release arch_atomic_dec_return_release1283
1284/**
1285 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
1286 * @v: pointer to atomic_t
1287 *
1288 * Atomically updates @v to (@v - 1) with relaxed ordering.
1289 *
1290 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
1291 *
1292 * Return: The updated value of @v.
1293 */
1294static __always_inline int
1295raw_atomic_dec_return_relaxed(atomic_t *v)
1296{
1297#if defined(arch_atomic_dec_return_relaxed)
1298 return arch_atomic_dec_return_relaxed(v);
1299#elif defined(arch_atomic_dec_return)
1300 return arch_atomic_dec_return(v);
1301#else
1302 return raw_atomic_sub_return_relaxed(1, v);
686#endif1303#endif
1304}
687 1305
688#ifndef arch_atomic_dec_return1306/**
1307 * raw_atomic_fetch_dec() - atomic decrement with full ordering
1308 * @v: pointer to atomic_t
1309 *
1310 * Atomically updates @v to (@v - 1) with full ordering.
1311 *
1312 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
1313 *
1314 * Return: The original value of @v.
1315 */
689static __always_inline int1316static __always_inline int
690arch_atomic_dec_return(atomic_t *v)1317raw_atomic_fetch_dec(atomic_t *v)
691{1318{
1319#if defined(arch_atomic_fetch_dec)
1320 return arch_atomic_fetch_dec(v);
1321#elif defined(arch_atomic_fetch_dec_relaxed)
692 int ret;1322 int ret;
693 __atomic_pre_full_fence();1323 __atomic_pre_full_fence();
694 ret = arch_atomic_dec_return_relaxed(v);1324 ret = arch_atomic_fetch_dec_relaxed(v);
695 __atomic_post_full_fence();1325 __atomic_post_full_fence();
696 return ret;1326 return ret;
697}1327#else
698#define arch_atomic_dec_return arch_atomic_dec_return1328 return raw_atomic_fetch_sub(1, v);
699#endif1329#endif
700
701#endif /* arch_atomic_dec_return_relaxed */
702
703#ifndef arch_atomic_fetch_dec_relaxed
704#ifdef arch_atomic_fetch_dec
705#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
706#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
707#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
708#endif /* arch_atomic_fetch_dec */
709
710#ifndef arch_atomic_fetch_dec
711static __always_inline int
712arch_atomic_fetch_dec(atomic_t *v)
713{
714 return arch_atomic_fetch_sub(1, v);
715}1330}
716#define arch_atomic_fetch_dec arch_atomic_fetch_dec
717#endif
718 1331
719#ifndef arch_atomic_fetch_dec_acquire1332/**
1333 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
1334 * @v: pointer to atomic_t
1335 *
1336 * Atomically updates @v to (@v - 1) with acquire ordering.
1337 *
1338 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
1339 *
1340 * Return: The original value of @v.
1341 */
720static __always_inline int1342static __always_inline int
721arch_atomic_fetch_dec_acquire(atomic_t *v)1343raw_atomic_fetch_dec_acquire(atomic_t *v)
722{1344{
723 return arch_atomic_fetch_sub_acquire(1, v);1345#if defined(arch_atomic_fetch_dec_acquire)
724}1346 return arch_atomic_fetch_dec_acquire(v);
725#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire1347#elif defined(arch_atomic_fetch_dec_relaxed)
1348 int ret = arch_atomic_fetch_dec_relaxed(v);
1349 __atomic_acquire_fence();
1350 return ret;
1351#elif defined(arch_atomic_fetch_dec)
1352 return arch_atomic_fetch_dec(v);
1353#else
1354 return raw_atomic_fetch_sub_acquire(1, v);
726#endif1355#endif
727
728#ifndef arch_atomic_fetch_dec_release
729static __always_inline int
730arch_atomic_fetch_dec_release(atomic_t *v)
731{
732 return arch_atomic_fetch_sub_release(1, v);
733}1356}
734#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
735#endif
736 1357
737#ifndef arch_atomic_fetch_dec_relaxed1358/**
1359 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
1360 * @v: pointer to atomic_t
1361 *
1362 * Atomically updates @v to (@v - 1) with release ordering.
1363 *
1364 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
1365 *
1366 * Return: The original value of @v.
1367 */
738static __always_inline int1368static __always_inline int
739arch_atomic_fetch_dec_relaxed(atomic_t *v)1369raw_atomic_fetch_dec_release(atomic_t *v)
740{1370{
741 return arch_atomic_fetch_sub_relaxed(1, v);1371#if defined(arch_atomic_fetch_dec_release)
742}1372 return arch_atomic_fetch_dec_release(v);
743#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed1373#elif defined(arch_atomic_fetch_dec_relaxed)
1374 __atomic_release_fence();
1375 return arch_atomic_fetch_dec_relaxed(v);
1376#elif defined(arch_atomic_fetch_dec)
1377 return arch_atomic_fetch_dec(v);
1378#else
1379 return raw_atomic_fetch_sub_release(1, v);
744#endif1380#endif
1381}
745 1382
746#else /* arch_atomic_fetch_dec_relaxed */1383/**
747 1384 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
748#ifndef arch_atomic_fetch_dec_acquire1385 * @v: pointer to atomic_t
1386 *
1387 * Atomically updates @v to (@v - 1) with relaxed ordering.
1388 *
1389 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
1390 *
1391 * Return: The original value of @v.
1392 */
749static __always_inline int1393static __always_inline int
750arch_atomic_fetch_dec_acquire(atomic_t *v)1394raw_atomic_fetch_dec_relaxed(atomic_t *v)
751{1395{
752 int ret = arch_atomic_fetch_dec_relaxed(v);1396#if defined(arch_atomic_fetch_dec_relaxed)
753 __atomic_acquire_fence();1397 return arch_atomic_fetch_dec_relaxed(v);
754 return ret;1398#elif defined(arch_atomic_fetch_dec)
755}1399 return arch_atomic_fetch_dec(v);
756#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire1400#else
1401 return raw_atomic_fetch_sub_relaxed(1, v);
757#endif1402#endif
1403}
758 1404
759#ifndef arch_atomic_fetch_dec_release1405/**
760static __always_inline int1406 * raw_atomic_and() - atomic bitwise AND with relaxed ordering
761arch_atomic_fetch_dec_release(atomic_t *v)1407 * @i: int value
1408 * @v: pointer to atomic_t
1409 *
1410 * Atomically updates @v to (@v & @i) with relaxed ordering.
1411 *
1412 * Safe to use in noinstr code; prefer atomic_and() elsewhere.
1413 *
1414 * Return: Nothing.
1415 */
1416static __always_inline void
1417raw_atomic_and(int i, atomic_t *v)
762{1418{
763 __atomic_release_fence();1419 arch_atomic_and(i, v);
764 return arch_atomic_fetch_dec_relaxed(v);
765}1420}
766#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
767#endif
768 1421
769#ifndef arch_atomic_fetch_dec1422/**
1423 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
1424 * @i: int value
1425 * @v: pointer to atomic_t
1426 *
1427 * Atomically updates @v to (@v & @i) with full ordering.
1428 *
1429 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
1430 *
1431 * Return: The original value of @v.
1432 */
770static __always_inline int1433static __always_inline int
771arch_atomic_fetch_dec(atomic_t *v)1434raw_atomic_fetch_and(int i, atomic_t *v)
772{1435{
1436#if defined(arch_atomic_fetch_and)
1437 return arch_atomic_fetch_and(i, v);
1438#elif defined(arch_atomic_fetch_and_relaxed)
773 int ret;1439 int ret;
774 __atomic_pre_full_fence();1440 __atomic_pre_full_fence();
775 ret = arch_atomic_fetch_dec_relaxed(v);1441 ret = arch_atomic_fetch_and_relaxed(i, v);
776 __atomic_post_full_fence();1442 __atomic_post_full_fence();
777 return ret;1443 return ret;
778}1444#else
779#define arch_atomic_fetch_dec arch_atomic_fetch_dec1445#error "Unable to define raw_atomic_fetch_and"
780#endif1446#endif
1447}
781 1448
782#endif /* arch_atomic_fetch_dec_relaxed */1449/**
783 1450 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
784#ifndef arch_atomic_fetch_and_relaxed1451 * @i: int value
785#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and1452 * @v: pointer to atomic_t
786#define arch_atomic_fetch_and_release arch_atomic_fetch_and1453 *
787#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and1454 * Atomically updates @v to (@v & @i) with acquire ordering.
788#else /* arch_atomic_fetch_and_relaxed */1455 *
789 1456 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
790#ifndef arch_atomic_fetch_and_acquire1457 *
1458 * Return: The original value of @v.
1459 */
791static __always_inline int1460static __always_inline int
792arch_atomic_fetch_and_acquire(int i, atomic_t *v)1461raw_atomic_fetch_and_acquire(int i, atomic_t *v)
793{1462{
1463#if defined(arch_atomic_fetch_and_acquire)
1464 return arch_atomic_fetch_and_acquire(i, v);
1465#elif defined(arch_atomic_fetch_and_relaxed)
794 int ret = arch_atomic_fetch_and_relaxed(i, v);1466 int ret = arch_atomic_fetch_and_relaxed(i, v);
795 __atomic_acquire_fence();1467 __atomic_acquire_fence();
796 return ret;1468 return ret;
797}1469#elif defined(arch_atomic_fetch_and)
798#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire1470 return arch_atomic_fetch_and(i, v);
1471#else
1472#error "Unable to define raw_atomic_fetch_and_acquire"
799#endif1473#endif
1474}
800 1475
801#ifndef arch_atomic_fetch_and_release1476/**
1477 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
1478 * @i: int value
1479 * @v: pointer to atomic_t
1480 *
1481 * Atomically updates @v to (@v & @i) with release ordering.
1482 *
1483 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
1484 *
1485 * Return: The original value of @v.
1486 */
802static __always_inline int1487static __always_inline int
803arch_atomic_fetch_and_release(int i, atomic_t *v)1488raw_atomic_fetch_and_release(int i, atomic_t *v)
804{1489{
1490#if defined(arch_atomic_fetch_and_release)
1491 return arch_atomic_fetch_and_release(i, v);
1492#elif defined(arch_atomic_fetch_and_relaxed)
805 __atomic_release_fence();1493 __atomic_release_fence();
806 return arch_atomic_fetch_and_relaxed(i, v);1494 return arch_atomic_fetch_and_relaxed(i, v);
807}1495#elif defined(arch_atomic_fetch_and)
808#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release1496 return arch_atomic_fetch_and(i, v);
1497#else
1498#error "Unable to define raw_atomic_fetch_and_release"
809#endif1499#endif
1500}
810 1501
811#ifndef arch_atomic_fetch_and1502/**
1503 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
1504 * @i: int value
1505 * @v: pointer to atomic_t
1506 *
1507 * Atomically updates @v to (@v & @i) with relaxed ordering.
1508 *
1509 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
1510 *
1511 * Return: The original value of @v.
1512 */
812static __always_inline int1513static __always_inline int
813arch_atomic_fetch_and(int i, atomic_t *v)1514raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
814{1515{
815 int ret;1516#if defined(arch_atomic_fetch_and_relaxed)
816 __atomic_pre_full_fence();1517 return arch_atomic_fetch_and_relaxed(i, v);
817 ret = arch_atomic_fetch_and_relaxed(i, v);1518#elif defined(arch_atomic_fetch_and)
818 __atomic_post_full_fence();1519 return arch_atomic_fetch_and(i, v);
819 return ret;1520#else
820}1521#error "Unable to define raw_atomic_fetch_and_relaxed"
821#define arch_atomic_fetch_and arch_atomic_fetch_and
822#endif1522#endif
1523}
823 1524
824#endif /* arch_atomic_fetch_and_relaxed */1525/**
825 1526 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
826#ifndef arch_atomic_andnot1527 * @i: int value
1528 * @v: pointer to atomic_t
1529 *
1530 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1531 *
1532 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
1533 *
1534 * Return: Nothing.
1535 */
827static __always_inline void1536static __always_inline void
828arch_atomic_andnot(int i, atomic_t *v)1537raw_atomic_andnot(int i, atomic_t *v)
829{1538{
830 arch_atomic_and(~i, v);1539#if defined(arch_atomic_andnot)
831}1540 arch_atomic_andnot(i, v);
832#define arch_atomic_andnot arch_atomic_andnot1541#else
1542 raw_atomic_and(~i, v);
833#endif1543#endif
834
835#ifndef arch_atomic_fetch_andnot_relaxed
836#ifdef arch_atomic_fetch_andnot
837#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
838#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
839#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
840#endif /* arch_atomic_fetch_andnot */
841
842#ifndef arch_atomic_fetch_andnot
843static __always_inline int
844arch_atomic_fetch_andnot(int i, atomic_t *v)
845{
846 return arch_atomic_fetch_and(~i, v);
847}1544}
848#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
849#endif
850 1545
851#ifndef arch_atomic_fetch_andnot_acquire1546/**
852static __always_inline int1547 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
853arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)1548 * @i: int value
854{1549 * @v: pointer to atomic_t
855 return arch_atomic_fetch_and_acquire(~i, v);1550 *
856}1551 * Atomically updates @v to (@v & ~@i) with full ordering.
857#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire1552 *
858#endif1553 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
859 1554 *
860#ifndef arch_atomic_fetch_andnot_release1555 * Return: The original value of @v.
1556 */
861static __always_inline int1557static __always_inline int
862arch_atomic_fetch_andnot_release(int i, atomic_t *v)1558raw_atomic_fetch_andnot(int i, atomic_t *v)
863{1559{
864 return arch_atomic_fetch_and_release(~i, v);1560#if defined(arch_atomic_fetch_andnot)
865}1561 return arch_atomic_fetch_andnot(i, v);
866#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release1562#elif defined(arch_atomic_fetch_andnot_relaxed)
1563 int ret;
1564 __atomic_pre_full_fence();
1565 ret = arch_atomic_fetch_andnot_relaxed(i, v);
1566 __atomic_post_full_fence();
1567 return ret;
1568#else
1569 return raw_atomic_fetch_and(~i, v);
867#endif1570#endif
868
869#ifndef arch_atomic_fetch_andnot_relaxed
870static __always_inline int
871arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
872{
873 return arch_atomic_fetch_and_relaxed(~i, v);
874}1571}
875#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
876#endif
877
878#else /* arch_atomic_fetch_andnot_relaxed */
879 1572
880#ifndef arch_atomic_fetch_andnot_acquire1573/**
1574 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
1575 * @i: int value
1576 * @v: pointer to atomic_t
1577 *
1578 * Atomically updates @v to (@v & ~@i) with acquire ordering.
1579 *
1580 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
1581 *
1582 * Return: The original value of @v.
1583 */
881static __always_inline int1584static __always_inline int
882arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)1585raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
883{1586{
1587#if defined(arch_atomic_fetch_andnot_acquire)
1588 return arch_atomic_fetch_andnot_acquire(i, v);
1589#elif defined(arch_atomic_fetch_andnot_relaxed)
884 int ret = arch_atomic_fetch_andnot_relaxed(i, v);1590 int ret = arch_atomic_fetch_andnot_relaxed(i, v);
885 __atomic_acquire_fence();1591 __atomic_acquire_fence();
886 return ret;1592 return ret;
887}1593#elif defined(arch_atomic_fetch_andnot)
888#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire1594 return arch_atomic_fetch_andnot(i, v);
1595#else
1596 return raw_atomic_fetch_and_acquire(~i, v);
889#endif1597#endif
1598}
890 1599
891#ifndef arch_atomic_fetch_andnot_release1600/**
1601 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1602 * @i: int value
1603 * @v: pointer to atomic_t
1604 *
1605 * Atomically updates @v to (@v & ~@i) with release ordering.
1606 *
1607 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
1608 *
1609 * Return: The original value of @v.
1610 */
892static __always_inline int1611static __always_inline int
893arch_atomic_fetch_andnot_release(int i, atomic_t *v)1612raw_atomic_fetch_andnot_release(int i, atomic_t *v)
894{1613{
1614#if defined(arch_atomic_fetch_andnot_release)
1615 return arch_atomic_fetch_andnot_release(i, v);
1616#elif defined(arch_atomic_fetch_andnot_relaxed)
895 __atomic_release_fence();1617 __atomic_release_fence();
896 return arch_atomic_fetch_andnot_relaxed(i, v);1618 return arch_atomic_fetch_andnot_relaxed(i, v);
1619#elif defined(arch_atomic_fetch_andnot)
1620 return arch_atomic_fetch_andnot(i, v);
1621#else
1622 return raw_atomic_fetch_and_release(~i, v);
1623#endif
897}1624}
898#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release1625
1626/**
1627 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1628 * @i: int value
1629 * @v: pointer to atomic_t
1630 *
1631 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1632 *
1633 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
1634 *
1635 * Return: The original value of @v.
1636 */
1637static __always_inline int
1638raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
1639{
1640#if defined(arch_atomic_fetch_andnot_relaxed)
1641 return arch_atomic_fetch_andnot_relaxed(i, v);
1642#elif defined(arch_atomic_fetch_andnot)
1643 return arch_atomic_fetch_andnot(i, v);
1644#else
1645 return raw_atomic_fetch_and_relaxed(~i, v);
899#endif1646#endif
1647}
1648
1649/**
1650 * raw_atomic_or() - atomic bitwise OR with relaxed ordering
1651 * @i: int value
1652 * @v: pointer to atomic_t
1653 *
1654 * Atomically updates @v to (@v | @i) with relaxed ordering.
1655 *
1656 * Safe to use in noinstr code; prefer atomic_or() elsewhere.
1657 *
1658 * Return: Nothing.
1659 */
1660static __always_inline void
1661raw_atomic_or(int i, atomic_t *v)
1662{
1663 arch_atomic_or(i, v);
1664}
900 1665
901#ifndef arch_atomic_fetch_andnot1666/**
1667 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
1668 * @i: int value
1669 * @v: pointer to atomic_t
1670 *
1671 * Atomically updates @v to (@v | @i) with full ordering.
1672 *
1673 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
1674 *
1675 * Return: The original value of @v.
1676 */
902static __always_inline int1677static __always_inline int
903arch_atomic_fetch_andnot(int i, atomic_t *v)1678raw_atomic_fetch_or(int i, atomic_t *v)
904{1679{
1680#if defined(arch_atomic_fetch_or)
1681 return arch_atomic_fetch_or(i, v);
1682#elif defined(arch_atomic_fetch_or_relaxed)
905 int ret;1683 int ret;
906 __atomic_pre_full_fence();1684 __atomic_pre_full_fence();
907 ret = arch_atomic_fetch_andnot_relaxed(i, v);1685 ret = arch_atomic_fetch_or_relaxed(i, v);
908 __atomic_post_full_fence();1686 __atomic_post_full_fence();
909 return ret;1687 return ret;
910}1688#else
911#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot1689#error "Unable to define raw_atomic_fetch_or"
912#endif1690#endif
1691}
913 1692
914#endif /* arch_atomic_fetch_andnot_relaxed */1693/**
915 1694 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
916#ifndef arch_atomic_fetch_or_relaxed1695 * @i: int value
917#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or1696 * @v: pointer to atomic_t
918#define arch_atomic_fetch_or_release arch_atomic_fetch_or1697 *
919#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or1698 * Atomically updates @v to (@v | @i) with acquire ordering.
920#else /* arch_atomic_fetch_or_relaxed */1699 *
921 1700 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
922#ifndef arch_atomic_fetch_or_acquire1701 *
1702 * Return: The original value of @v.
1703 */
923static __always_inline int1704static __always_inline int
924arch_atomic_fetch_or_acquire(int i, atomic_t *v)1705raw_atomic_fetch_or_acquire(int i, atomic_t *v)
925{1706{
1707#if defined(arch_atomic_fetch_or_acquire)
1708 return arch_atomic_fetch_or_acquire(i, v);
1709#elif defined(arch_atomic_fetch_or_relaxed)
926 int ret = arch_atomic_fetch_or_relaxed(i, v);1710 int ret = arch_atomic_fetch_or_relaxed(i, v);
927 __atomic_acquire_fence();1711 __atomic_acquire_fence();
928 return ret;1712 return ret;
929}1713#elif defined(arch_atomic_fetch_or)
930#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire1714 return arch_atomic_fetch_or(i, v);
1715#else
1716#error "Unable to define raw_atomic_fetch_or_acquire"
931#endif1717#endif
1718}
932 1719
933#ifndef arch_atomic_fetch_or_release1720/**
1721 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
1722 * @i: int value
1723 * @v: pointer to atomic_t
1724 *
1725 * Atomically updates @v to (@v | @i) with release ordering.
1726 *
1727 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
1728 *
1729 * Return: The original value of @v.
1730 */
934static __always_inline int1731static __always_inline int
935arch_atomic_fetch_or_release(int i, atomic_t *v)1732raw_atomic_fetch_or_release(int i, atomic_t *v)
936{1733{
1734#if defined(arch_atomic_fetch_or_release)
1735 return arch_atomic_fetch_or_release(i, v);
1736#elif defined(arch_atomic_fetch_or_relaxed)
937 __atomic_release_fence();1737 __atomic_release_fence();
938 return arch_atomic_fetch_or_relaxed(i, v);1738 return arch_atomic_fetch_or_relaxed(i, v);
1739#elif defined(arch_atomic_fetch_or)
1740 return arch_atomic_fetch_or(i, v);
1741#else
1742#error "Unable to define raw_atomic_fetch_or_release"
1743#endif
939}1744}
940#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release1745
1746/**
1747 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1748 * @i: int value
1749 * @v: pointer to atomic_t
1750 *
1751 * Atomically updates @v to (@v | @i) with relaxed ordering.
1752 *
1753 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
1754 *
1755 * Return: The original value of @v.
1756 */
1757static __always_inline int
1758raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
1759{
1760#if defined(arch_atomic_fetch_or_relaxed)
1761 return arch_atomic_fetch_or_relaxed(i, v);
1762#elif defined(arch_atomic_fetch_or)
1763 return arch_atomic_fetch_or(i, v);
1764#else
1765#error "Unable to define raw_atomic_fetch_or_relaxed"
941#endif1766#endif
1767}
942 1768
943#ifndef arch_atomic_fetch_or1769/**
1770 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
1771 * @i: int value
1772 * @v: pointer to atomic_t
1773 *
1774 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1775 *
1776 * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
1777 *
1778 * Return: Nothing.
1779 */
1780static __always_inline void
1781raw_atomic_xor(int i, atomic_t *v)
1782{
1783 arch_atomic_xor(i, v);
1784}
1785
1786/**
1787 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
1788 * @i: int value
1789 * @v: pointer to atomic_t
1790 *
1791 * Atomically updates @v to (@v ^ @i) with full ordering.
1792 *
1793 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
1794 *
1795 * Return: The original value of @v.
1796 */
944static __always_inline int1797static __always_inline int
945arch_atomic_fetch_or(int i, atomic_t *v)1798raw_atomic_fetch_xor(int i, atomic_t *v)
946{1799{
1800#if defined(arch_atomic_fetch_xor)
1801 return arch_atomic_fetch_xor(i, v);
1802#elif defined(arch_atomic_fetch_xor_relaxed)
947 int ret;1803 int ret;
948 __atomic_pre_full_fence();1804 __atomic_pre_full_fence();
949 ret = arch_atomic_fetch_or_relaxed(i, v);1805 ret = arch_atomic_fetch_xor_relaxed(i, v);
950 __atomic_post_full_fence();1806 __atomic_post_full_fence();
951 return ret;1807 return ret;
952}1808#else
953#define arch_atomic_fetch_or arch_atomic_fetch_or1809#error "Unable to define raw_atomic_fetch_xor"
954#endif1810#endif
1811}
955 1812
956#endif /* arch_atomic_fetch_or_relaxed */1813/**
957 1814 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
958#ifndef arch_atomic_fetch_xor_relaxed1815 * @i: int value
959#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor1816 * @v: pointer to atomic_t
960#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor1817 *
961#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor1818 * Atomically updates @v to (@v ^ @i) with acquire ordering.
962#else /* arch_atomic_fetch_xor_relaxed */1819 *
963 1820 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
964#ifndef arch_atomic_fetch_xor_acquire1821 *
1822 * Return: The original value of @v.
1823 */
965static __always_inline int1824static __always_inline int
966arch_atomic_fetch_xor_acquire(int i, atomic_t *v)1825raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
967{1826{
1827#if defined(arch_atomic_fetch_xor_acquire)
1828 return arch_atomic_fetch_xor_acquire(i, v);
1829#elif defined(arch_atomic_fetch_xor_relaxed)
968 int ret = arch_atomic_fetch_xor_relaxed(i, v);1830 int ret = arch_atomic_fetch_xor_relaxed(i, v);
969 __atomic_acquire_fence();1831 __atomic_acquire_fence();
970 return ret;1832 return ret;
971}1833#elif defined(arch_atomic_fetch_xor)
972#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire1834 return arch_atomic_fetch_xor(i, v);
1835#else
1836#error "Unable to define raw_atomic_fetch_xor_acquire"
973#endif1837#endif
1838}
974 1839
975#ifndef arch_atomic_fetch_xor_release1840/**
1841 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1842 * @i: int value
1843 * @v: pointer to atomic_t
1844 *
1845 * Atomically updates @v to (@v ^ @i) with release ordering.
1846 *
1847 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
1848 *
1849 * Return: The original value of @v.
1850 */
976static __always_inline int1851static __always_inline int
977arch_atomic_fetch_xor_release(int i, atomic_t *v)1852raw_atomic_fetch_xor_release(int i, atomic_t *v)
978{1853{
1854#if defined(arch_atomic_fetch_xor_release)
1855 return arch_atomic_fetch_xor_release(i, v);
1856#elif defined(arch_atomic_fetch_xor_relaxed)
979 __atomic_release_fence();1857 __atomic_release_fence();
980 return arch_atomic_fetch_xor_relaxed(i, v);1858 return arch_atomic_fetch_xor_relaxed(i, v);
1859#elif defined(arch_atomic_fetch_xor)
1860 return arch_atomic_fetch_xor(i, v);
1861#else
1862#error "Unable to define raw_atomic_fetch_xor_release"
1863#endif
981}1864}
982#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release1865
1866/**
1867 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1868 * @i: int value
1869 * @v: pointer to atomic_t
1870 *
1871 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1872 *
1873 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
1874 *
1875 * Return: The original value of @v.
1876 */
1877static __always_inline int
1878raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
1879{
1880#if defined(arch_atomic_fetch_xor_relaxed)
1881 return arch_atomic_fetch_xor_relaxed(i, v);
1882#elif defined(arch_atomic_fetch_xor)
1883 return arch_atomic_fetch_xor(i, v);
1884#else
1885#error "Unable to define raw_atomic_fetch_xor_relaxed"
983#endif1886#endif
1887}
984 1888
985#ifndef arch_atomic_fetch_xor1889/**
1890 * raw_atomic_xchg() - atomic exchange with full ordering
1891 * @v: pointer to atomic_t
1892 * @new: int value to assign
1893 *
1894 * Atomically updates @v to @new with full ordering.
1895 *
1896 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
1897 *
1898 * Return: The original value of @v.
1899 */
986static __always_inline int1900static __always_inline int
987arch_atomic_fetch_xor(int i, atomic_t *v)1901raw_atomic_xchg(atomic_t *v, int new)
988{1902{
1903#if defined(arch_atomic_xchg)
1904 return arch_atomic_xchg(v, new);
1905#elif defined(arch_atomic_xchg_relaxed)
989 int ret;1906 int ret;
990 __atomic_pre_full_fence();1907 __atomic_pre_full_fence();
991 ret = arch_atomic_fetch_xor_relaxed(i, v);1908 ret = arch_atomic_xchg_relaxed(v, new);
992 __atomic_post_full_fence();1909 __atomic_post_full_fence();
993 return ret;1910 return ret;
994}1911#else
995#define arch_atomic_fetch_xor arch_atomic_fetch_xor1912 return raw_xchg(&v->counter, new);
996#endif1913#endif
1914}
997 1915
998#endif /* arch_atomic_fetch_xor_relaxed */1916/**
999 1917 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
1000#ifndef arch_atomic_xchg_relaxed1918 * @v: pointer to atomic_t
1001#define arch_atomic_xchg_acquire arch_atomic_xchg1919 * @new: int value to assign
1002#define arch_atomic_xchg_release arch_atomic_xchg1920 *
1003#define arch_atomic_xchg_relaxed arch_atomic_xchg1921 * Atomically updates @v to @new with acquire ordering.
1004#else /* arch_atomic_xchg_relaxed */1922 *
1005 1923 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
1006#ifndef arch_atomic_xchg_acquire1924 *
1925 * Return: The original value of @v.
1926 */
1007static __always_inline int1927static __always_inline int
1008arch_atomic_xchg_acquire(atomic_t *v, int i)1928raw_atomic_xchg_acquire(atomic_t *v, int new)
1009{1929{
1010 int ret = arch_atomic_xchg_relaxed(v, i);1930#if defined(arch_atomic_xchg_acquire)
1931 return arch_atomic_xchg_acquire(v, new);
1932#elif defined(arch_atomic_xchg_relaxed)
1933 int ret = arch_atomic_xchg_relaxed(v, new);
1011 __atomic_acquire_fence();1934 __atomic_acquire_fence();
1012 return ret;1935 return ret;
1013}1936#elif defined(arch_atomic_xchg)
1014#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire1937 return arch_atomic_xchg(v, new);
1938#else
1939 return raw_xchg_acquire(&v->counter, new);
1015#endif1940#endif
1941}
1016 1942
1017#ifndef arch_atomic_xchg_release1943/**
1944 * raw_atomic_xchg_release() - atomic exchange with release ordering
1945 * @v: pointer to atomic_t
1946 * @new: int value to assign
1947 *
1948 * Atomically updates @v to @new with release ordering.
1949 *
1950 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
1951 *
1952 * Return: The original value of @v.
1953 */
1018static __always_inline int1954static __always_inline int
1019arch_atomic_xchg_release(atomic_t *v, int i)1955raw_atomic_xchg_release(atomic_t *v, int new)
1020{1956{
1957#if defined(arch_atomic_xchg_release)
1958 return arch_atomic_xchg_release(v, new);
1959#elif defined(arch_atomic_xchg_relaxed)
1021 __atomic_release_fence();1960 __atomic_release_fence();
1022 return arch_atomic_xchg_relaxed(v, i);1961 return arch_atomic_xchg_relaxed(v, new);
1962#elif defined(arch_atomic_xchg)
1963 return arch_atomic_xchg(v, new);
1964#else
1965 return raw_xchg_release(&v->counter, new);
1966#endif
1023}1967}
1024#define arch_atomic_xchg_release arch_atomic_xchg_release1968
1969/**
1970 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1971 * @v: pointer to atomic_t
1972 * @new: int value to assign
1973 *
1974 * Atomically updates @v to @new with relaxed ordering.
1975 *
1976 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
1977 *
1978 * Return: The original value of @v.
1979 */
1980static __always_inline int
1981raw_atomic_xchg_relaxed(atomic_t *v, int new)
1982{
1983#if defined(arch_atomic_xchg_relaxed)
1984 return arch_atomic_xchg_relaxed(v, new);
1985#elif defined(arch_atomic_xchg)
1986 return arch_atomic_xchg(v, new);
1987#else
1988 return raw_xchg_relaxed(&v->counter, new);
1025#endif1989#endif
1990}
1026 1991
1027#ifndef arch_atomic_xchg1992/**
1993 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
1994 * @v: pointer to atomic_t
1995 * @old: int value to compare with
1996 * @new: int value to assign
1997 *
1998 * If (@v == @old), atomically updates @v to @new with full ordering.
1999 *
2000 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
2001 *
2002 * Return: The original value of @v.
2003 */
1028static __always_inline int2004static __always_inline int
1029arch_atomic_xchg(atomic_t *v, int i)2005raw_atomic_cmpxchg(atomic_t *v, int old, int new)
1030{2006{
2007#if defined(arch_atomic_cmpxchg)
2008 return arch_atomic_cmpxchg(v, old, new);
2009#elif defined(arch_atomic_cmpxchg_relaxed)
1031 int ret;2010 int ret;
1032 __atomic_pre_full_fence();2011 __atomic_pre_full_fence();
1033 ret = arch_atomic_xchg_relaxed(v, i);2012 ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1034 __atomic_post_full_fence();2013 __atomic_post_full_fence();
1035 return ret;2014 return ret;
1036}2015#else
1037#define arch_atomic_xchg arch_atomic_xchg2016 return raw_cmpxchg(&v->counter, old, new);
1038#endif2017#endif
1039 2018}
1040#endif /* arch_atomic_xchg_relaxed */
1041
1042#ifndef arch_atomic_cmpxchg_relaxed
1043#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
1044#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
1045#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
1046#else /* arch_atomic_cmpxchg_relaxed */
1047 2019
1048#ifndef arch_atomic_cmpxchg_acquire2020/**
2021 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2022 * @v: pointer to atomic_t
2023 * @old: int value to compare with
2024 * @new: int value to assign
2025 *
2026 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2027 *
2028 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2029 *
2030 * Return: The original value of @v.
2031 */
1049static __always_inline int2032static __always_inline int
1050arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)2033raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1051{2034{
2035#if defined(arch_atomic_cmpxchg_acquire)
2036 return arch_atomic_cmpxchg_acquire(v, old, new);
2037#elif defined(arch_atomic_cmpxchg_relaxed)
1052 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);2038 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1053 __atomic_acquire_fence();2039 __atomic_acquire_fence();
1054 return ret;2040 return ret;
1055}2041#elif defined(arch_atomic_cmpxchg)
1056#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire2042 return arch_atomic_cmpxchg(v, old, new);
2043#else
2044 return raw_cmpxchg_acquire(&v->counter, old, new);
1057#endif2045#endif
2046}
1058 2047
1059#ifndef arch_atomic_cmpxchg_release2048/**
2049 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
2050 * @v: pointer to atomic_t
2051 * @old: int value to compare with
2052 * @new: int value to assign
2053 *
2054 * If (@v == @old), atomically updates @v to @new with release ordering.
2055 *
2056 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2057 *
2058 * Return: The original value of @v.
2059 */
1060static __always_inline int2060static __always_inline int
1061arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)2061raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
1062{2062{
2063#if defined(arch_atomic_cmpxchg_release)
2064 return arch_atomic_cmpxchg_release(v, old, new);
2065#elif defined(arch_atomic_cmpxchg_relaxed)
1063 __atomic_release_fence();2066 __atomic_release_fence();
1064 return arch_atomic_cmpxchg_relaxed(v, old, new);2067 return arch_atomic_cmpxchg_relaxed(v, old, new);
1065}2068#elif defined(arch_atomic_cmpxchg)
1066#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release2069 return arch_atomic_cmpxchg(v, old, new);
2070#else
2071 return raw_cmpxchg_release(&v->counter, old, new);
1067#endif2072#endif
2073}
1068 2074
1069#ifndef arch_atomic_cmpxchg2075/**
2076 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2077 * @v: pointer to atomic_t
2078 * @old: int value to compare with
2079 * @new: int value to assign
2080 *
2081 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2082 *
2083 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2084 *
2085 * Return: The original value of @v.
2086 */
1070static __always_inline int2087static __always_inline int
1071arch_atomic_cmpxchg(atomic_t *v, int old, int new)2088raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
1072{2089{
1073 int ret;2090#if defined(arch_atomic_cmpxchg_relaxed)
1074 __atomic_pre_full_fence();2091 return arch_atomic_cmpxchg_relaxed(v, old, new);
1075 ret = arch_atomic_cmpxchg_relaxed(v, old, new);2092#elif defined(arch_atomic_cmpxchg)
1076 __atomic_post_full_fence();2093 return arch_atomic_cmpxchg(v, old, new);
1077 return ret;2094#else
1078}2095 return raw_cmpxchg_relaxed(&v->counter, old, new);
1079#define arch_atomic_cmpxchg arch_atomic_cmpxchg
1080#endif2096#endif
2097}
1081 2098
1082#endif /* arch_atomic_cmpxchg_relaxed */2099/**
1083 2100 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
1084#ifndef arch_atomic_try_cmpxchg_relaxed2101 * @v: pointer to atomic_t
1085#ifdef arch_atomic_try_cmpxchg2102 * @old: pointer to int value to compare with
1086#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg2103 * @new: int value to assign
1087#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg2104 *
1088#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg2105 * If (@v == @old), atomically updates @v to @new with full ordering.
1089#endif /* arch_atomic_try_cmpxchg */2106 * Otherwise, updates @old to the current value of @v.
1090 2107 *
1091#ifndef arch_atomic_try_cmpxchg2108 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2109 *
2110 * Return: @true if the exchange occured, @false otherwise.
2111 */
1092static __always_inline bool2112static __always_inline bool
1093arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)2113raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1094{2114{
2115#if defined(arch_atomic_try_cmpxchg)
2116 return arch_atomic_try_cmpxchg(v, old, new);
2117#elif defined(arch_atomic_try_cmpxchg_relaxed)
2118 bool ret;
2119 __atomic_pre_full_fence();
2120 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2121 __atomic_post_full_fence();
2122 return ret;
2123#else
1095 int r, o = *old;2124 int r, o = *old;
1096 r = arch_atomic_cmpxchg(v, o, new);2125 r = raw_atomic_cmpxchg(v, o, new);
1097 if (unlikely(r != o))2126 if (unlikely(r != o))
1098 *old = r;2127 *old = r;
1099 return likely(r == o);2128 return likely(r == o);
1100}
1101#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1102#endif2129#endif
2130}
1103 2131
1104#ifndef arch_atomic_try_cmpxchg_acquire2132/**
2133 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2134 * @v: pointer to atomic_t
2135 * @old: pointer to int value to compare with
2136 * @new: int value to assign
2137 *
2138 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2139 * Otherwise, updates @old to the current value of @v.
2140 *
2141 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2142 *
2143 * Return: @true if the exchange occured, @false otherwise.
2144 */
1105static __always_inline bool2145static __always_inline bool
1106arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)2146raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1107{2147{
2148#if defined(arch_atomic_try_cmpxchg_acquire)
2149 return arch_atomic_try_cmpxchg_acquire(v, old, new);
2150#elif defined(arch_atomic_try_cmpxchg_relaxed)
2151 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2152 __atomic_acquire_fence();
2153 return ret;
2154#elif defined(arch_atomic_try_cmpxchg)
2155 return arch_atomic_try_cmpxchg(v, old, new);
2156#else
1108 int r, o = *old;2157 int r, o = *old;
1109 r = arch_atomic_cmpxchg_acquire(v, o, new);2158 r = raw_atomic_cmpxchg_acquire(v, o, new);
1110 if (unlikely(r != o))2159 if (unlikely(r != o))
1111 *old = r;2160 *old = r;
1112 return likely(r == o);2161 return likely(r == o);
1113}
1114#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1115#endif2162#endif
2163}
1116 2164
1117#ifndef arch_atomic_try_cmpxchg_release2165/**
2166 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
2167 * @v: pointer to atomic_t
2168 * @old: pointer to int value to compare with
2169 * @new: int value to assign
2170 *
2171 * If (@v == @old), atomically updates @v to @new with release ordering.
2172 * Otherwise, updates @old to the current value of @v.
2173 *
2174 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2175 *
2176 * Return: @true if the exchange occured, @false otherwise.
2177 */
1118static __always_inline bool2178static __always_inline bool
1119arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)2179raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1120{2180{
2181#if defined(arch_atomic_try_cmpxchg_release)
2182 return arch_atomic_try_cmpxchg_release(v, old, new);
2183#elif defined(arch_atomic_try_cmpxchg_relaxed)
2184 __atomic_release_fence();
2185 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2186#elif defined(arch_atomic_try_cmpxchg)
2187 return arch_atomic_try_cmpxchg(v, old, new);
2188#else
1121 int r, o = *old;2189 int r, o = *old;
1122 r = arch_atomic_cmpxchg_release(v, o, new);2190 r = raw_atomic_cmpxchg_release(v, o, new);
1123 if (unlikely(r != o))2191 if (unlikely(r != o))
1124 *old = r;2192 *old = r;
1125 return likely(r == o);2193 return likely(r == o);
1126}
1127#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1128#endif2194#endif
2195}
1129 2196
1130#ifndef arch_atomic_try_cmpxchg_relaxed2197/**
2198 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2199 * @v: pointer to atomic_t
2200 * @old: pointer to int value to compare with
2201 * @new: int value to assign
2202 *
2203 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2204 * Otherwise, updates @old to the current value of @v.
2205 *
2206 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2207 *
2208 * Return: @true if the exchange occured, @false otherwise.
2209 */
1131static __always_inline bool2210static __always_inline bool
1132arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)2211raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1133{2212{
2213#if defined(arch_atomic_try_cmpxchg_relaxed)
2214 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2215#elif defined(arch_atomic_try_cmpxchg)
2216 return arch_atomic_try_cmpxchg(v, old, new);
2217#else
1134 int r, o = *old;2218 int r, o = *old;
1135 r = arch_atomic_cmpxchg_relaxed(v, o, new);2219 r = raw_atomic_cmpxchg_relaxed(v, o, new);
1136 if (unlikely(r != o))2220 if (unlikely(r != o))
1137 *old = r;2221 *old = r;
1138 return likely(r == o);2222 return likely(r == o);
1139}
1140#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1141#endif
1142
1143#else /* arch_atomic_try_cmpxchg_relaxed */
1144
1145#ifndef arch_atomic_try_cmpxchg_acquire
1146static __always_inline bool
1147arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1148{
1149 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1150 __atomic_acquire_fence();
1151 return ret;
1152}
1153#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1154#endif
1155
1156#ifndef arch_atomic_try_cmpxchg_release
1157static __always_inline bool
1158arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1159{
1160 __atomic_release_fence();
1161 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1162}
1163#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1164#endif2223#endif
1165
1166#ifndef arch_atomic_try_cmpxchg
1167static __always_inline bool
1168arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1169{
1170 bool ret;
1171 __atomic_pre_full_fence();
1172 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1173 __atomic_post_full_fence();
1174 return ret;
1175}2224}
1176#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1177#endif
1178 2225
1179#endif /* arch_atomic_try_cmpxchg_relaxed */
1180
1181#ifndef arch_atomic_sub_and_test
1182/**2226/**
1183 * arch_atomic_sub_and_test - subtract value from variable and test result2227 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
1184 * @i: integer value to subtract2228 * @i: int value to add
1185 * @v: pointer of type atomic_t2229 * @v: pointer to atomic_t
2230 *
2231 * Atomically updates @v to (@v - @i) with full ordering.
1186 *2232 *
1187 * Atomically subtracts @i from @v and returns2233 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
1188 * true if the result is zero, or false for all2234 *
1189 * other cases.2235 * Return: @true if the resulting value of @v is zero, @false otherwise.
1190 */2236 */
1191static __always_inline bool2237static __always_inline bool
1192arch_atomic_sub_and_test(int i, atomic_t *v)2238raw_atomic_sub_and_test(int i, atomic_t *v)
1193{2239{
1194 return arch_atomic_sub_return(i, v) == 0;2240#if defined(arch_atomic_sub_and_test)
1195}2241 return arch_atomic_sub_and_test(i, v);
1196#define arch_atomic_sub_and_test arch_atomic_sub_and_test2242#else
2243 return raw_atomic_sub_return(i, v) == 0;
1197#endif2244#endif
2245}
1198 2246
1199#ifndef arch_atomic_dec_and_test
1200/**2247/**
1201 * arch_atomic_dec_and_test - decrement and test2248 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
1202 * @v: pointer of type atomic_t2249 * @v: pointer to atomic_t
2250 *
2251 * Atomically updates @v to (@v - 1) with full ordering.
1203 *2252 *
1204 * Atomically decrements @v by 1 and2253 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
1205 * returns true if the result is 0, or false for all other2254 *
1206 * cases.2255 * Return: @true if the resulting value of @v is zero, @false otherwise.
1207 */2256 */
1208static __always_inline bool2257static __always_inline bool
1209arch_atomic_dec_and_test(atomic_t *v)2258raw_atomic_dec_and_test(atomic_t *v)
1210{2259{
1211 return arch_atomic_dec_return(v) == 0;2260#if defined(arch_atomic_dec_and_test)
1212}2261 return arch_atomic_dec_and_test(v);
1213#define arch_atomic_dec_and_test arch_atomic_dec_and_test2262#else
2263 return raw_atomic_dec_return(v) == 0;
1214#endif2264#endif
2265}
1215 2266
1216#ifndef arch_atomic_inc_and_test
1217/**2267/**
1218 * arch_atomic_inc_and_test - increment and test2268 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
1219 * @v: pointer of type atomic_t2269 * @v: pointer to atomic_t
2270 *
2271 * Atomically updates @v to (@v + 1) with full ordering.
1220 *2272 *
1221 * Atomically increments @v by 12273 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
1222 * and returns true if the result is zero, or false for all2274 *
1223 * other cases.2275 * Return: @true if the resulting value of @v is zero, @false otherwise.
1224 */2276 */
1225static __always_inline bool2277static __always_inline bool
1226arch_atomic_inc_and_test(atomic_t *v)2278raw_atomic_inc_and_test(atomic_t *v)
1227{2279{
1228 return arch_atomic_inc_return(v) == 0;2280#if defined(arch_atomic_inc_and_test)
1229}2281 return arch_atomic_inc_and_test(v);
1230#define arch_atomic_inc_and_test arch_atomic_inc_and_test2282#else
2283 return raw_atomic_inc_return(v) == 0;
1231#endif2284#endif
2285}
1232 2286
1233#ifndef arch_atomic_add_negative_relaxed
1234#ifdef arch_atomic_add_negative
1235#define arch_atomic_add_negative_acquire arch_atomic_add_negative
1236#define arch_atomic_add_negative_release arch_atomic_add_negative
1237#define arch_atomic_add_negative_relaxed arch_atomic_add_negative
1238#endif /* arch_atomic_add_negative */
1239
1240#ifndef arch_atomic_add_negative
1241/**2287/**
1242 * arch_atomic_add_negative - Add and test if negative2288 * raw_atomic_add_negative() - atomic add and test if negative with full ordering
1243 * @i: integer value to add2289 * @i: int value to add
1244 * @v: pointer of type atomic_t2290 * @v: pointer to atomic_t
1245 *2291 *
1246 * Atomically adds @i to @v and returns true if the result is negative,2292 * Atomically updates @v to (@v + @i) with full ordering.
1247 * or false when the result is greater than or equal to zero.2293 *
2294 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
2295 *
2296 * Return: @true if the resulting value of @v is negative, @false otherwise.
1248 */2297 */
1249static __always_inline bool2298static __always_inline bool
1250arch_atomic_add_negative(int i, atomic_t *v)2299raw_atomic_add_negative(int i, atomic_t *v)
1251{2300{
1252 return arch_atomic_add_return(i, v) < 0;2301#if defined(arch_atomic_add_negative)
1253}2302 return arch_atomic_add_negative(i, v);
1254#define arch_atomic_add_negative arch_atomic_add_negative2303#elif defined(arch_atomic_add_negative_relaxed)
2304 bool ret;
2305 __atomic_pre_full_fence();
2306 ret = arch_atomic_add_negative_relaxed(i, v);
2307 __atomic_post_full_fence();
2308 return ret;
2309#else
2310 return raw_atomic_add_return(i, v) < 0;
1255#endif2311#endif
2312}
1256 2313
1257#ifndef arch_atomic_add_negative_acquire
1258/**2314/**
1259 * arch_atomic_add_negative_acquire - Add and test if negative2315 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
1260 * @i: integer value to add2316 * @i: int value to add
1261 * @v: pointer of type atomic_t2317 * @v: pointer to atomic_t
2318 *
2319 * Atomically updates @v to (@v + @i) with acquire ordering.
1262 *2320 *
1263 * Atomically adds @i to @v and returns true if the result is negative,2321 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
1264 * or false when the result is greater than or equal to zero.2322 *
2323 * Return: @true if the resulting value of @v is negative, @false otherwise.
1265 */2324 */
1266static __always_inline bool2325static __always_inline bool
1267arch_atomic_add_negative_acquire(int i, atomic_t *v)2326raw_atomic_add_negative_acquire(int i, atomic_t *v)
1268{2327{
1269 return arch_atomic_add_return_acquire(i, v) < 0;2328#if defined(arch_atomic_add_negative_acquire)
1270}2329 return arch_atomic_add_negative_acquire(i, v);
1271#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire2330#elif defined(arch_atomic_add_negative_relaxed)
2331 bool ret = arch_atomic_add_negative_relaxed(i, v);
2332 __atomic_acquire_fence();
2333 return ret;
2334#elif defined(arch_atomic_add_negative)
2335 return arch_atomic_add_negative(i, v);
2336#else
2337 return raw_atomic_add_return_acquire(i, v) < 0;
1272#endif2338#endif
2339}
1273 2340
1274#ifndef arch_atomic_add_negative_release
1275/**2341/**
1276 * arch_atomic_add_negative_release - Add and test if negative2342 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
1277 * @i: integer value to add2343 * @i: int value to add
1278 * @v: pointer of type atomic_t2344 * @v: pointer to atomic_t
1279 *2345 *
1280 * Atomically adds @i to @v and returns true if the result is negative,2346 * Atomically updates @v to (@v + @i) with release ordering.
1281 * or false when the result is greater than or equal to zero.2347 *
2348 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
2349 *
2350 * Return: @true if the resulting value of @v is negative, @false otherwise.
1282 */2351 */
1283static __always_inline bool2352static __always_inline bool
1284arch_atomic_add_negative_release(int i, atomic_t *v)2353raw_atomic_add_negative_release(int i, atomic_t *v)
1285{2354{
1286 return arch_atomic_add_return_release(i, v) < 0;2355#if defined(arch_atomic_add_negative_release)
1287}2356 return arch_atomic_add_negative_release(i, v);
1288#define arch_atomic_add_negative_release arch_atomic_add_negative_release2357#elif defined(arch_atomic_add_negative_relaxed)
2358 __atomic_release_fence();
2359 return arch_atomic_add_negative_relaxed(i, v);
2360#elif defined(arch_atomic_add_negative)
2361 return arch_atomic_add_negative(i, v);
2362#else
2363 return raw_atomic_add_return_release(i, v) < 0;
1289#endif2364#endif
2365}
1290 2366
1291#ifndef arch_atomic_add_negative_relaxed
1292/**2367/**
1293 * arch_atomic_add_negative_relaxed - Add and test if negative2368 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
1294 * @i: integer value to add2369 * @i: int value to add
1295 * @v: pointer of type atomic_t2370 * @v: pointer to atomic_t
2371 *
2372 * Atomically updates @v to (@v + @i) with relaxed ordering.
2373 *
2374 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
1296 *2375 *
1297 * Atomically adds @i to @v and returns true if the result is negative,2376 * Return: @true if the resulting value of @v is negative, @false otherwise.
1298 * or false when the result is greater than or equal to zero.
1299 */2377 */
1300static __always_inline bool2378static __always_inline bool
1301arch_atomic_add_negative_relaxed(int i, atomic_t *v)2379raw_atomic_add_negative_relaxed(int i, atomic_t *v)
1302{
1303 return arch_atomic_add_return_relaxed(i, v) < 0;
1304}
1305#define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed
1306#endif
1307
1308#else /* arch_atomic_add_negative_relaxed */
1309
1310#ifndef arch_atomic_add_negative_acquire
1311static __always_inline bool
1312arch_atomic_add_negative_acquire(int i, atomic_t *v)
1313{
1314 bool ret = arch_atomic_add_negative_relaxed(i, v);
1315 __atomic_acquire_fence();
1316 return ret;
1317}
1318#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
1319#endif
1320
1321#ifndef arch_atomic_add_negative_release
1322static __always_inline bool
1323arch_atomic_add_negative_release(int i, atomic_t *v)
1324{2380{
1325 __atomic_release_fence();2381#if defined(arch_atomic_add_negative_relaxed)
1326 return arch_atomic_add_negative_relaxed(i, v);2382 return arch_atomic_add_negative_relaxed(i, v);
1327}2383#elif defined(arch_atomic_add_negative)
1328#define arch_atomic_add_negative_release arch_atomic_add_negative_release2384 return arch_atomic_add_negative(i, v);
2385#else
2386 return raw_atomic_add_return_relaxed(i, v) < 0;
1329#endif2387#endif
1330
1331#ifndef arch_atomic_add_negative
1332static __always_inline bool
1333arch_atomic_add_negative(int i, atomic_t *v)
1334{
1335 bool ret;
1336 __atomic_pre_full_fence();
1337 ret = arch_atomic_add_negative_relaxed(i, v);
1338 __atomic_post_full_fence();
1339 return ret;
1340}2388}
1341#define arch_atomic_add_negative arch_atomic_add_negative
1342#endif
1343
1344#endif /* arch_atomic_add_negative_relaxed */
1345 2389
1346#ifndef arch_atomic_fetch_add_unless
1347/**2390/**
1348 * arch_atomic_fetch_add_unless - add unless the number is already a given value2391 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
1349 * @v: pointer of type atomic_t2392 * @v: pointer to atomic_t
1350 * @a: the amount to add to v...2393 * @a: int value to add
1351 * @u: ...unless v is equal to u.2394 * @u: int value to compare with
2395 *
2396 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2397 *
2398 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
1352 *2399 *
1353 * Atomically adds @a to @v, so long as @v was not already @u.2400 * Return: The original value of @v.
1354 * Returns original value of @v
1355 */2401 */
1356static __always_inline int2402static __always_inline int
1357arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)2403raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1358{2404{
1359 int c = arch_atomic_read(v);2405#if defined(arch_atomic_fetch_add_unless)
2406 return arch_atomic_fetch_add_unless(v, a, u);
2407#else
2408 int c = raw_atomic_read(v);
1360 2409
1361 do {2410 do {
1362 if (unlikely(c == u))2411 if (unlikely(c == u))
1363 break;2412 break;
1364 } while (!arch_atomic_try_cmpxchg(v, &c, c + a));2413 } while (!raw_atomic_try_cmpxchg(v, &c, c + a));
1365 2414
1366 return c;2415 return c;
1367}
1368#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1369#endif2416#endif
2417}
1370 2418
1371#ifndef arch_atomic_add_unless
1372/**2419/**
1373 * arch_atomic_add_unless - add unless the number is already a given value2420 * raw_atomic_add_unless() - atomic add unless value with full ordering
1374 * @v: pointer of type atomic_t2421 * @v: pointer to atomic_t
1375 * @a: the amount to add to v...2422 * @a: int value to add
1376 * @u: ...unless v is equal to u.2423 * @u: int value to compare with
2424 *
2425 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2426 *
2427 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
1377 *2428 *
1378 * Atomically adds @a to @v, if @v was not already @u.2429 * Return: @true if @v was updated, @false otherwise.
1379 * Returns true if the addition was done.
1380 */2430 */
1381static __always_inline bool2431static __always_inline bool
1382arch_atomic_add_unless(atomic_t *v, int a, int u)2432raw_atomic_add_unless(atomic_t *v, int a, int u)
1383{2433{
1384 return arch_atomic_fetch_add_unless(v, a, u) != u;2434#if defined(arch_atomic_add_unless)
1385}2435 return arch_atomic_add_unless(v, a, u);
1386#define arch_atomic_add_unless arch_atomic_add_unless2436#else
2437 return raw_atomic_fetch_add_unless(v, a, u) != u;
1387#endif2438#endif
2439}
1388 2440
1389#ifndef arch_atomic_inc_not_zero
1390/**2441/**
1391 * arch_atomic_inc_not_zero - increment unless the number is zero2442 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
1392 * @v: pointer of type atomic_t2443 * @v: pointer to atomic_t
2444 *
2445 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
2446 *
2447 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
1393 *2448 *
1394 * Atomically increments @v by 1, if @v is non-zero.2449 * Return: @true if @v was updated, @false otherwise.
1395 * Returns true if the increment was done.
1396 */2450 */
1397static __always_inline bool2451static __always_inline bool
1398arch_atomic_inc_not_zero(atomic_t *v)2452raw_atomic_inc_not_zero(atomic_t *v)
1399{2453{
1400 return arch_atomic_add_unless(v, 1, 0);2454#if defined(arch_atomic_inc_not_zero)
1401}2455 return arch_atomic_inc_not_zero(v);
1402#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero2456#else
2457 return raw_atomic_add_unless(v, 1, 0);
1403#endif2458#endif
2459}
1404 2460
1405#ifndef arch_atomic_inc_unless_negative2461/**
2462 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
2463 * @v: pointer to atomic_t
2464 *
2465 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
2466 *
2467 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2468 *
2469 * Return: @true if @v was updated, @false otherwise.
2470 */
1406static __always_inline bool2471static __always_inline bool
1407arch_atomic_inc_unless_negative(atomic_t *v)2472raw_atomic_inc_unless_negative(atomic_t *v)
1408{2473{
1409 int c = arch_atomic_read(v);2474#if defined(arch_atomic_inc_unless_negative)
2475 return arch_atomic_inc_unless_negative(v);
2476#else
2477 int c = raw_atomic_read(v);
1410 2478
1411 do {2479 do {
1412 if (unlikely(c < 0))2480 if (unlikely(c < 0))
1413 return false;2481 return false;
1414 } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));2482 } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
1415 2483
1416 return true;2484 return true;
1417}
1418#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1419#endif2485#endif
2486}
1420 2487
1421#ifndef arch_atomic_dec_unless_positive2488/**
2489 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
2490 * @v: pointer to atomic_t
2491 *
2492 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
2493 *
2494 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2495 *
2496 * Return: @true if @v was updated, @false otherwise.
2497 */
1422static __always_inline bool2498static __always_inline bool
1423arch_atomic_dec_unless_positive(atomic_t *v)2499raw_atomic_dec_unless_positive(atomic_t *v)
1424{2500{
1425 int c = arch_atomic_read(v);2501#if defined(arch_atomic_dec_unless_positive)
2502 return arch_atomic_dec_unless_positive(v);
2503#else
2504 int c = raw_atomic_read(v);
1426 2505
1427 do {2506 do {
1428 if (unlikely(c > 0))2507 if (unlikely(c > 0))
1429 return false;2508 return false;
1430 } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));2509 } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
1431 2510
1432 return true;2511 return true;
1433}
1434#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1435#endif2512#endif
2513}
1436 2514
1437#ifndef arch_atomic_dec_if_positive2515/**
2516 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
2517 * @v: pointer to atomic_t
2518 *
2519 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
2520 *
2521 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2522 *
2523 * Return: The old value of (@v - 1), regardless of whether @v was updated.
2524 */
1438static __always_inline int2525static __always_inline int
1439arch_atomic_dec_if_positive(atomic_t *v)2526raw_atomic_dec_if_positive(atomic_t *v)
1440{2527{
1441 int dec, c = arch_atomic_read(v);2528#if defined(arch_atomic_dec_if_positive)
2529 return arch_atomic_dec_if_positive(v);
2530#else
2531 int dec, c = raw_atomic_read(v);
1442 2532
1443 do {2533 do {
1444 dec = c - 1;2534 dec = c - 1;
1445 if (unlikely(dec < 0))2535 if (unlikely(dec < 0))
1446 break;2536 break;
1447 } while (!arch_atomic_try_cmpxchg(v, &c, dec));2537 } while (!raw_atomic_try_cmpxchg(v, &c, dec));
1448 2538
1449 return dec;2539 return dec;
1450}
1451#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1452#endif2540#endif
2541}
1453 2542
1454#ifdef CONFIG_GENERIC_ATOMIC642543#ifdef CONFIG_GENERIC_ATOMIC64
1455#include <asm-generic/atomic64.h>2544#include <asm-generic/atomic64.h>
1456#endif2545#endif
1457 2546
1458#ifndef arch_atomic64_read_acquire2547/**
2548 * raw_atomic64_read() - atomic load with relaxed ordering
2549 * @v: pointer to atomic64_t
2550 *
2551 * Atomically loads the value of @v with relaxed ordering.
2552 *
2553 * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
2554 *
2555 * Return: The value loaded from @v.
2556 */
2557static __always_inline s64
2558raw_atomic64_read(const atomic64_t *v)
2559{
2560 return arch_atomic64_read(v);
2561}
2562
2563/**
2564 * raw_atomic64_read_acquire() - atomic load with acquire ordering
2565 * @v: pointer to atomic64_t
2566 *
2567 * Atomically loads the value of @v with acquire ordering.
2568 *
2569 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
2570 *
2571 * Return: The value loaded from @v.
2572 */
1459static __always_inline s642573static __always_inline s64
1460arch_atomic64_read_acquire(const atomic64_t *v)2574raw_atomic64_read_acquire(const atomic64_t *v)
1461{2575{
2576#if defined(arch_atomic64_read_acquire)
2577 return arch_atomic64_read_acquire(v);
2578#elif defined(arch_atomic64_read)
2579 return arch_atomic64_read(v);
2580#else
1462 s64 ret;2581 s64 ret;
1463 2582
1464 if (__native_word(atomic64_t)) {2583 if (__native_word(atomic64_t)) {
1465 ret = smp_load_acquire(&(v)->counter);2584 ret = smp_load_acquire(&(v)->counter);
1466 } else {2585 } else {
1467 ret = arch_atomic64_read(v);2586 ret = raw_atomic64_read(v);
1468 __atomic_acquire_fence();2587 __atomic_acquire_fence();
1469 }2588 }
1470 2589
1471 return ret;2590 return ret;
1472}
1473#define arch_atomic64_read_acquire arch_atomic64_read_acquire
1474#endif2591#endif
2592}
2593
2594/**
2595 * raw_atomic64_set() - atomic set with relaxed ordering
2596 * @v: pointer to atomic64_t
2597 * @i: s64 value to assign
2598 *
2599 * Atomically sets @v to @i with relaxed ordering.
2600 *
2601 * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
2602 *
2603 * Return: Nothing.
2604 */
2605static __always_inline void
2606raw_atomic64_set(atomic64_t *v, s64 i)
2607{
2608 arch_atomic64_set(v, i);
2609}
1475 2610
1476#ifndef arch_atomic64_set_release2611/**
2612 * raw_atomic64_set_release() - atomic set with release ordering
2613 * @v: pointer to atomic64_t
2614 * @i: s64 value to assign
2615 *
2616 * Atomically sets @v to @i with release ordering.
2617 *
2618 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
2619 *
2620 * Return: Nothing.
2621 */
1477static __always_inline void2622static __always_inline void
1478arch_atomic64_set_release(atomic64_t *v, s64 i)2623raw_atomic64_set_release(atomic64_t *v, s64 i)
1479{2624{
2625#if defined(arch_atomic64_set_release)
2626 arch_atomic64_set_release(v, i);
2627#elif defined(arch_atomic64_set)
2628 arch_atomic64_set(v, i);
2629#else
1480 if (__native_word(atomic64_t)) {2630 if (__native_word(atomic64_t)) {
1481 smp_store_release(&(v)->counter, i);2631 smp_store_release(&(v)->counter, i);
1482 } else {2632 } else {
1483 __atomic_release_fence();2633 __atomic_release_fence();
1484 arch_atomic64_set(v, i);2634 raw_atomic64_set(v, i);
1485 }2635 }
1486}
1487#define arch_atomic64_set_release arch_atomic64_set_release
1488#endif2636#endif
2637}
2638
2639/**
2640 * raw_atomic64_add() - atomic add with relaxed ordering
2641 * @i: s64 value to add
2642 * @v: pointer to atomic64_t
2643 *
2644 * Atomically updates @v to (@v + @i) with relaxed ordering.
2645 *
2646 * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
2647 *
2648 * Return: Nothing.
2649 */
2650static __always_inline void
2651raw_atomic64_add(s64 i, atomic64_t *v)
2652{
2653 arch_atomic64_add(i, v);
2654}
1489 2655
1490#ifndef arch_atomic64_add_return_relaxed2656/**
1491#define arch_atomic64_add_return_acquire arch_atomic64_add_return2657 * raw_atomic64_add_return() - atomic add with full ordering
1492#define arch_atomic64_add_return_release arch_atomic64_add_return2658 * @i: s64 value to add
1493#define arch_atomic64_add_return_relaxed arch_atomic64_add_return2659 * @v: pointer to atomic64_t
1494#else /* arch_atomic64_add_return_relaxed */2660 *
2661 * Atomically updates @v to (@v + @i) with full ordering.
2662 *
2663 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
2664 *
2665 * Return: The updated value of @v.
2666 */
2667static __always_inline s64
2668raw_atomic64_add_return(s64 i, atomic64_t *v)
2669{
2670#if defined(arch_atomic64_add_return)
2671 return arch_atomic64_add_return(i, v);
2672#elif defined(arch_atomic64_add_return_relaxed)
2673 s64 ret;
2674 __atomic_pre_full_fence();
2675 ret = arch_atomic64_add_return_relaxed(i, v);
2676 __atomic_post_full_fence();
2677 return ret;
2678#else
2679#error "Unable to define raw_atomic64_add_return"
2680#endif
2681}
1495 2682
1496#ifndef arch_atomic64_add_return_acquire2683/**
2684 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
2685 * @i: s64 value to add
2686 * @v: pointer to atomic64_t
2687 *
2688 * Atomically updates @v to (@v + @i) with acquire ordering.
2689 *
2690 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
2691 *
2692 * Return: The updated value of @v.
2693 */
1497static __always_inline s642694static __always_inline s64
1498arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)2695raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1499{2696{
2697#if defined(arch_atomic64_add_return_acquire)
2698 return arch_atomic64_add_return_acquire(i, v);
2699#elif defined(arch_atomic64_add_return_relaxed)
1500 s64 ret = arch_atomic64_add_return_relaxed(i, v);2700 s64 ret = arch_atomic64_add_return_relaxed(i, v);
1501 __atomic_acquire_fence();2701 __atomic_acquire_fence();
1502 return ret;2702 return ret;
1503}2703#elif defined(arch_atomic64_add_return)
1504#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire2704 return arch_atomic64_add_return(i, v);
2705#else
2706#error "Unable to define raw_atomic64_add_return_acquire"
1505#endif2707#endif
2708}
1506 2709
1507#ifndef arch_atomic64_add_return_release2710/**
2711 * raw_atomic64_add_return_release() - atomic add with release ordering
2712 * @i: s64 value to add
2713 * @v: pointer to atomic64_t
2714 *
2715 * Atomically updates @v to (@v + @i) with release ordering.
2716 *
2717 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
2718 *
2719 * Return: The updated value of @v.
2720 */
1508static __always_inline s642721static __always_inline s64
1509arch_atomic64_add_return_release(s64 i, atomic64_t *v)2722raw_atomic64_add_return_release(s64 i, atomic64_t *v)
1510{2723{
2724#if defined(arch_atomic64_add_return_release)
2725 return arch_atomic64_add_return_release(i, v);
2726#elif defined(arch_atomic64_add_return_relaxed)
1511 __atomic_release_fence();2727 __atomic_release_fence();
1512 return arch_atomic64_add_return_relaxed(i, v);2728 return arch_atomic64_add_return_relaxed(i, v);
2729#elif defined(arch_atomic64_add_return)
2730 return arch_atomic64_add_return(i, v);
2731#else
2732#error "Unable to define raw_atomic64_add_return_release"
2733#endif
1513}2734}
1514#define arch_atomic64_add_return_release arch_atomic64_add_return_release2735
2736/**
2737 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
2738 * @i: s64 value to add
2739 * @v: pointer to atomic64_t
2740 *
2741 * Atomically updates @v to (@v + @i) with relaxed ordering.
2742 *
2743 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
2744 *
2745 * Return: The updated value of @v.
2746 */
2747static __always_inline s64
2748raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
2749{
2750#if defined(arch_atomic64_add_return_relaxed)
2751 return arch_atomic64_add_return_relaxed(i, v);
2752#elif defined(arch_atomic64_add_return)
2753 return arch_atomic64_add_return(i, v);
2754#else
2755#error "Unable to define raw_atomic64_add_return_relaxed"
1515#endif2756#endif
2757}
1516 2758
1517#ifndef arch_atomic64_add_return2759/**
2760 * raw_atomic64_fetch_add() - atomic add with full ordering
2761 * @i: s64 value to add
2762 * @v: pointer to atomic64_t
2763 *
2764 * Atomically updates @v to (@v + @i) with full ordering.
2765 *
2766 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
2767 *
2768 * Return: The original value of @v.
2769 */
1518static __always_inline s642770static __always_inline s64
1519arch_atomic64_add_return(s64 i, atomic64_t *v)2771raw_atomic64_fetch_add(s64 i, atomic64_t *v)
1520{2772{
2773#if defined(arch_atomic64_fetch_add)
2774 return arch_atomic64_fetch_add(i, v);
2775#elif defined(arch_atomic64_fetch_add_relaxed)
1521 s64 ret;2776 s64 ret;
1522 __atomic_pre_full_fence();2777 __atomic_pre_full_fence();
1523 ret = arch_atomic64_add_return_relaxed(i, v);2778 ret = arch_atomic64_fetch_add_relaxed(i, v);
1524 __atomic_post_full_fence();2779 __atomic_post_full_fence();
1525 return ret;2780 return ret;
1526}2781#else
1527#define arch_atomic64_add_return arch_atomic64_add_return2782#error "Unable to define raw_atomic64_fetch_add"
1528#endif2783#endif
2784}
1529 2785
1530#endif /* arch_atomic64_add_return_relaxed */2786/**
1531 2787 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
1532#ifndef arch_atomic64_fetch_add_relaxed2788 * @i: s64 value to add
1533#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add2789 * @v: pointer to atomic64_t
1534#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add2790 *
1535#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add2791 * Atomically updates @v to (@v + @i) with acquire ordering.
1536#else /* arch_atomic64_fetch_add_relaxed */2792 *
1537 2793 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
1538#ifndef arch_atomic64_fetch_add_acquire2794 *
2795 * Return: The original value of @v.
2796 */
1539static __always_inline s642797static __always_inline s64
1540arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)2798raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1541{2799{
2800#if defined(arch_atomic64_fetch_add_acquire)
2801 return arch_atomic64_fetch_add_acquire(i, v);
2802#elif defined(arch_atomic64_fetch_add_relaxed)
1542 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);2803 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1543 __atomic_acquire_fence();2804 __atomic_acquire_fence();
1544 return ret;2805 return ret;
1545}2806#elif defined(arch_atomic64_fetch_add)
1546#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire2807 return arch_atomic64_fetch_add(i, v);
2808#else
2809#error "Unable to define raw_atomic64_fetch_add_acquire"
1547#endif2810#endif
2811}
1548 2812
1549#ifndef arch_atomic64_fetch_add_release2813/**
2814 * raw_atomic64_fetch_add_release() - atomic add with release ordering
2815 * @i: s64 value to add
2816 * @v: pointer to atomic64_t
2817 *
2818 * Atomically updates @v to (@v + @i) with release ordering.
2819 *
2820 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
2821 *
2822 * Return: The original value of @v.
2823 */
1550static __always_inline s642824static __always_inline s64
1551arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)2825raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1552{2826{
2827#if defined(arch_atomic64_fetch_add_release)
2828 return arch_atomic64_fetch_add_release(i, v);
2829#elif defined(arch_atomic64_fetch_add_relaxed)
1553 __atomic_release_fence();2830 __atomic_release_fence();
1554 return arch_atomic64_fetch_add_relaxed(i, v);2831 return arch_atomic64_fetch_add_relaxed(i, v);
2832#elif defined(arch_atomic64_fetch_add)
2833 return arch_atomic64_fetch_add(i, v);
2834#else
2835#error "Unable to define raw_atomic64_fetch_add_release"
2836#endif
1555}2837}
1556#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release2838
2839/**
2840 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
2841 * @i: s64 value to add
2842 * @v: pointer to atomic64_t
2843 *
2844 * Atomically updates @v to (@v + @i) with relaxed ordering.
2845 *
2846 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
2847 *
2848 * Return: The original value of @v.
2849 */
2850static __always_inline s64
2851raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
2852{
2853#if defined(arch_atomic64_fetch_add_relaxed)
2854 return arch_atomic64_fetch_add_relaxed(i, v);
2855#elif defined(arch_atomic64_fetch_add)
2856 return arch_atomic64_fetch_add(i, v);
2857#else
2858#error "Unable to define raw_atomic64_fetch_add_relaxed"
1557#endif2859#endif
2860}
2861
2862/**
2863 * raw_atomic64_sub() - atomic subtract with relaxed ordering
2864 * @i: s64 value to subtract
2865 * @v: pointer to atomic64_t
2866 *
2867 * Atomically updates @v to (@v - @i) with relaxed ordering.
2868 *
2869 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
2870 *
2871 * Return: Nothing.
2872 */
2873static __always_inline void
2874raw_atomic64_sub(s64 i, atomic64_t *v)
2875{
2876 arch_atomic64_sub(i, v);
2877}
1558 2878
1559#ifndef arch_atomic64_fetch_add2879/**
2880 * raw_atomic64_sub_return() - atomic subtract with full ordering
2881 * @i: s64 value to subtract
2882 * @v: pointer to atomic64_t
2883 *
2884 * Atomically updates @v to (@v - @i) with full ordering.
2885 *
2886 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
2887 *
2888 * Return: The updated value of @v.
2889 */
1560static __always_inline s642890static __always_inline s64
1561arch_atomic64_fetch_add(s64 i, atomic64_t *v)2891raw_atomic64_sub_return(s64 i, atomic64_t *v)
1562{2892{
2893#if defined(arch_atomic64_sub_return)
2894 return arch_atomic64_sub_return(i, v);
2895#elif defined(arch_atomic64_sub_return_relaxed)
1563 s64 ret;2896 s64 ret;
1564 __atomic_pre_full_fence();2897 __atomic_pre_full_fence();
1565 ret = arch_atomic64_fetch_add_relaxed(i, v);2898 ret = arch_atomic64_sub_return_relaxed(i, v);
1566 __atomic_post_full_fence();2899 __atomic_post_full_fence();
1567 return ret;2900 return ret;
1568}2901#else
1569#define arch_atomic64_fetch_add arch_atomic64_fetch_add2902#error "Unable to define raw_atomic64_sub_return"
1570#endif2903#endif
2904}
1571 2905
1572#endif /* arch_atomic64_fetch_add_relaxed */2906/**
1573 2907 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
1574#ifndef arch_atomic64_sub_return_relaxed2908 * @i: s64 value to subtract
1575#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return2909 * @v: pointer to atomic64_t
1576#define arch_atomic64_sub_return_release arch_atomic64_sub_return2910 *
1577#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return2911 * Atomically updates @v to (@v - @i) with acquire ordering.
1578#else /* arch_atomic64_sub_return_relaxed */2912 *
1579 2913 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
1580#ifndef arch_atomic64_sub_return_acquire2914 *
2915 * Return: The updated value of @v.
2916 */
1581static __always_inline s642917static __always_inline s64
1582arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)2918raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1583{2919{
2920#if defined(arch_atomic64_sub_return_acquire)
2921 return arch_atomic64_sub_return_acquire(i, v);
2922#elif defined(arch_atomic64_sub_return_relaxed)
1584 s64 ret = arch_atomic64_sub_return_relaxed(i, v);2923 s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1585 __atomic_acquire_fence();2924 __atomic_acquire_fence();
1586 return ret;2925 return ret;
1587}2926#elif defined(arch_atomic64_sub_return)
1588#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire2927 return arch_atomic64_sub_return(i, v);
2928#else
2929#error "Unable to define raw_atomic64_sub_return_acquire"
1589#endif2930#endif
2931}
1590 2932
1591#ifndef arch_atomic64_sub_return_release2933/**
2934 * raw_atomic64_sub_return_release() - atomic subtract with release ordering
2935 * @i: s64 value to subtract
2936 * @v: pointer to atomic64_t
2937 *
2938 * Atomically updates @v to (@v - @i) with release ordering.
2939 *
2940 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
2941 *
2942 * Return: The updated value of @v.
2943 */
1592static __always_inline s642944static __always_inline s64
1593arch_atomic64_sub_return_release(s64 i, atomic64_t *v)2945raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
1594{2946{
2947#if defined(arch_atomic64_sub_return_release)
2948 return arch_atomic64_sub_return_release(i, v);
2949#elif defined(arch_atomic64_sub_return_relaxed)
1595 __atomic_release_fence();2950 __atomic_release_fence();
1596 return arch_atomic64_sub_return_relaxed(i, v);2951 return arch_atomic64_sub_return_relaxed(i, v);
2952#elif defined(arch_atomic64_sub_return)
2953 return arch_atomic64_sub_return(i, v);
2954#else
2955#error "Unable to define raw_atomic64_sub_return_release"
2956#endif
1597}2957}
1598#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release2958
2959/**
2960 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
2961 * @i: s64 value to subtract
2962 * @v: pointer to atomic64_t
2963 *
2964 * Atomically updates @v to (@v - @i) with relaxed ordering.
2965 *
2966 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
2967 *
2968 * Return: The updated value of @v.
2969 */
2970static __always_inline s64
2971raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
2972{
2973#if defined(arch_atomic64_sub_return_relaxed)
2974 return arch_atomic64_sub_return_relaxed(i, v);
2975#elif defined(arch_atomic64_sub_return)
2976 return arch_atomic64_sub_return(i, v);
2977#else
2978#error "Unable to define raw_atomic64_sub_return_relaxed"
1599#endif2979#endif
2980}
1600 2981
1601#ifndef arch_atomic64_sub_return2982/**
2983 * raw_atomic64_fetch_sub() - atomic subtract with full ordering
2984 * @i: s64 value to subtract
2985 * @v: pointer to atomic64_t
2986 *
2987 * Atomically updates @v to (@v - @i) with full ordering.
2988 *
2989 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
2990 *
2991 * Return: The original value of @v.
2992 */
1602static __always_inline s642993static __always_inline s64
1603arch_atomic64_sub_return(s64 i, atomic64_t *v)2994raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
1604{2995{
2996#if defined(arch_atomic64_fetch_sub)
2997 return arch_atomic64_fetch_sub(i, v);
2998#elif defined(arch_atomic64_fetch_sub_relaxed)
1605 s64 ret;2999 s64 ret;
1606 __atomic_pre_full_fence();3000 __atomic_pre_full_fence();
1607 ret = arch_atomic64_sub_return_relaxed(i, v);3001 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1608 __atomic_post_full_fence();3002 __atomic_post_full_fence();
1609 return ret;3003 return ret;
1610}3004#else
1611#define arch_atomic64_sub_return arch_atomic64_sub_return3005#error "Unable to define raw_atomic64_fetch_sub"
1612#endif3006#endif
3007}
1613 3008
1614#endif /* arch_atomic64_sub_return_relaxed */3009/**
1615 3010 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
1616#ifndef arch_atomic64_fetch_sub_relaxed3011 * @i: s64 value to subtract
1617#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub3012 * @v: pointer to atomic64_t
1618#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub3013 *
1619#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub3014 * Atomically updates @v to (@v - @i) with acquire ordering.
1620#else /* arch_atomic64_fetch_sub_relaxed */3015 *
1621 3016 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
1622#ifndef arch_atomic64_fetch_sub_acquire3017 *
3018 * Return: The original value of @v.
3019 */
1623static __always_inline s643020static __always_inline s64
1624arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)3021raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1625{3022{
3023#if defined(arch_atomic64_fetch_sub_acquire)
3024 return arch_atomic64_fetch_sub_acquire(i, v);
3025#elif defined(arch_atomic64_fetch_sub_relaxed)
1626 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);3026 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1627 __atomic_acquire_fence();3027 __atomic_acquire_fence();
1628 return ret;3028 return ret;
1629}3029#elif defined(arch_atomic64_fetch_sub)
1630#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire3030 return arch_atomic64_fetch_sub(i, v);
3031#else
3032#error "Unable to define raw_atomic64_fetch_sub_acquire"
1631#endif3033#endif
3034}
1632 3035
1633#ifndef arch_atomic64_fetch_sub_release3036/**
3037 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
3038 * @i: s64 value to subtract
3039 * @v: pointer to atomic64_t
3040 *
3041 * Atomically updates @v to (@v - @i) with release ordering.
3042 *
3043 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
3044 *
3045 * Return: The original value of @v.
3046 */
1634static __always_inline s643047static __always_inline s64
1635arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)3048raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1636{3049{
3050#if defined(arch_atomic64_fetch_sub_release)
3051 return arch_atomic64_fetch_sub_release(i, v);
3052#elif defined(arch_atomic64_fetch_sub_relaxed)
1637 __atomic_release_fence();3053 __atomic_release_fence();
1638 return arch_atomic64_fetch_sub_relaxed(i, v);3054 return arch_atomic64_fetch_sub_relaxed(i, v);
1639}3055#elif defined(arch_atomic64_fetch_sub)
1640#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release3056 return arch_atomic64_fetch_sub(i, v);
3057#else
3058#error "Unable to define raw_atomic64_fetch_sub_release"
1641#endif3059#endif
3060}
1642 3061
1643#ifndef arch_atomic64_fetch_sub3062/**
3063 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3064 * @i: s64 value to subtract
3065 * @v: pointer to atomic64_t
3066 *
3067 * Atomically updates @v to (@v - @i) with relaxed ordering.
3068 *
3069 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
3070 *
3071 * Return: The original value of @v.
3072 */
1644static __always_inline s643073static __always_inline s64
1645arch_atomic64_fetch_sub(s64 i, atomic64_t *v)3074raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1646{3075{
1647 s64 ret;3076#if defined(arch_atomic64_fetch_sub_relaxed)
1648 __atomic_pre_full_fence();3077 return arch_atomic64_fetch_sub_relaxed(i, v);
1649 ret = arch_atomic64_fetch_sub_relaxed(i, v);3078#elif defined(arch_atomic64_fetch_sub)
1650 __atomic_post_full_fence();3079 return arch_atomic64_fetch_sub(i, v);
1651 return ret;3080#else
1652}3081#error "Unable to define raw_atomic64_fetch_sub_relaxed"
1653#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1654#endif3082#endif
1655
1656#endif /* arch_atomic64_fetch_sub_relaxed */
1657
1658#ifndef arch_atomic64_inc
1659static __always_inline void
1660arch_atomic64_inc(atomic64_t *v)
1661{
1662 arch_atomic64_add(1, v);
1663}3083}
1664#define arch_atomic64_inc arch_atomic64_inc
1665#endif
1666 3084
1667#ifndef arch_atomic64_inc_return_relaxed3085/**
1668#ifdef arch_atomic64_inc_return3086 * raw_atomic64_inc() - atomic increment with relaxed ordering
1669#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return3087 * @v: pointer to atomic64_t
1670#define arch_atomic64_inc_return_release arch_atomic64_inc_return3088 *
1671#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return3089 * Atomically updates @v to (@v + 1) with relaxed ordering.
1672#endif /* arch_atomic64_inc_return */3090 *
1673 3091 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
1674#ifndef arch_atomic64_inc_return3092 *
1675static __always_inline s643093 * Return: Nothing.
1676arch_atomic64_inc_return(atomic64_t *v)3094 */
3095static __always_inline void
3096raw_atomic64_inc(atomic64_t *v)
1677{3097{
1678 return arch_atomic64_add_return(1, v);3098#if defined(arch_atomic64_inc)
1679}3099 arch_atomic64_inc(v);
1680#define arch_atomic64_inc_return arch_atomic64_inc_return3100#else
3101 raw_atomic64_add(1, v);
1681#endif3102#endif
1682
1683#ifndef arch_atomic64_inc_return_acquire
1684static __always_inline s64
1685arch_atomic64_inc_return_acquire(atomic64_t *v)
1686{
1687 return arch_atomic64_add_return_acquire(1, v);
1688}3103}
1689#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1690#endif
1691 3104
1692#ifndef arch_atomic64_inc_return_release3105/**
3106 * raw_atomic64_inc_return() - atomic increment with full ordering
3107 * @v: pointer to atomic64_t
3108 *
3109 * Atomically updates @v to (@v + 1) with full ordering.
3110 *
3111 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
3112 *
3113 * Return: The updated value of @v.
3114 */
1693static __always_inline s643115static __always_inline s64
1694arch_atomic64_inc_return_release(atomic64_t *v)3116raw_atomic64_inc_return(atomic64_t *v)
1695{3117{
1696 return arch_atomic64_add_return_release(1, v);3118#if defined(arch_atomic64_inc_return)
1697}3119 return arch_atomic64_inc_return(v);
1698#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release3120#elif defined(arch_atomic64_inc_return_relaxed)
3121 s64 ret;
3122 __atomic_pre_full_fence();
3123 ret = arch_atomic64_inc_return_relaxed(v);
3124 __atomic_post_full_fence();
3125 return ret;
3126#else
3127 return raw_atomic64_add_return(1, v);
1699#endif3128#endif
1700
1701#ifndef arch_atomic64_inc_return_relaxed
1702static __always_inline s64
1703arch_atomic64_inc_return_relaxed(atomic64_t *v)
1704{
1705 return arch_atomic64_add_return_relaxed(1, v);
1706}3129}
1707#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1708#endif
1709 3130
1710#else /* arch_atomic64_inc_return_relaxed */3131/**
1711 3132 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
1712#ifndef arch_atomic64_inc_return_acquire3133 * @v: pointer to atomic64_t
3134 *
3135 * Atomically updates @v to (@v + 1) with acquire ordering.
3136 *
3137 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
3138 *
3139 * Return: The updated value of @v.
3140 */
1713static __always_inline s643141static __always_inline s64
1714arch_atomic64_inc_return_acquire(atomic64_t *v)3142raw_atomic64_inc_return_acquire(atomic64_t *v)
1715{3143{
3144#if defined(arch_atomic64_inc_return_acquire)
3145 return arch_atomic64_inc_return_acquire(v);
3146#elif defined(arch_atomic64_inc_return_relaxed)
1716 s64 ret = arch_atomic64_inc_return_relaxed(v);3147 s64 ret = arch_atomic64_inc_return_relaxed(v);
1717 __atomic_acquire_fence();3148 __atomic_acquire_fence();
1718 return ret;3149 return ret;
1719}3150#elif defined(arch_atomic64_inc_return)
1720#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire3151 return arch_atomic64_inc_return(v);
3152#else
3153 return raw_atomic64_add_return_acquire(1, v);
1721#endif3154#endif
3155}
1722 3156
1723#ifndef arch_atomic64_inc_return_release3157/**
3158 * raw_atomic64_inc_return_release() - atomic increment with release ordering
3159 * @v: pointer to atomic64_t
3160 *
3161 * Atomically updates @v to (@v + 1) with release ordering.
3162 *
3163 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
3164 *
3165 * Return: The updated value of @v.
3166 */
1724static __always_inline s643167static __always_inline s64
1725arch_atomic64_inc_return_release(atomic64_t *v)3168raw_atomic64_inc_return_release(atomic64_t *v)
1726{3169{
3170#if defined(arch_atomic64_inc_return_release)
3171 return arch_atomic64_inc_return_release(v);
3172#elif defined(arch_atomic64_inc_return_relaxed)
1727 __atomic_release_fence();3173 __atomic_release_fence();
1728 return arch_atomic64_inc_return_relaxed(v);3174 return arch_atomic64_inc_return_relaxed(v);
1729}3175#elif defined(arch_atomic64_inc_return)
1730#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release3176 return arch_atomic64_inc_return(v);
3177#else
3178 return raw_atomic64_add_return_release(1, v);
1731#endif3179#endif
1732
1733#ifndef arch_atomic64_inc_return
1734static __always_inline s64
1735arch_atomic64_inc_return(atomic64_t *v)
1736{
1737 s64 ret;
1738 __atomic_pre_full_fence();
1739 ret = arch_atomic64_inc_return_relaxed(v);
1740 __atomic_post_full_fence();
1741 return ret;
1742}3180}
1743#define arch_atomic64_inc_return arch_atomic64_inc_return
1744#endif
1745
1746#endif /* arch_atomic64_inc_return_relaxed */
1747 3181
1748#ifndef arch_atomic64_fetch_inc_relaxed3182/**
1749#ifdef arch_atomic64_fetch_inc3183 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
1750#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc3184 * @v: pointer to atomic64_t
1751#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc3185 *
1752#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc3186 * Atomically updates @v to (@v + 1) with relaxed ordering.
1753#endif /* arch_atomic64_fetch_inc */3187 *
1754 3188 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
1755#ifndef arch_atomic64_fetch_inc3189 *
3190 * Return: The updated value of @v.
3191 */
1756static __always_inline s643192static __always_inline s64
1757arch_atomic64_fetch_inc(atomic64_t *v)3193raw_atomic64_inc_return_relaxed(atomic64_t *v)
1758{3194{
1759 return arch_atomic64_fetch_add(1, v);3195#if defined(arch_atomic64_inc_return_relaxed)
1760}3196 return arch_atomic64_inc_return_relaxed(v);
1761#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc3197#elif defined(arch_atomic64_inc_return)
3198 return arch_atomic64_inc_return(v);
3199#else
3200 return raw_atomic64_add_return_relaxed(1, v);
1762#endif3201#endif
1763
1764#ifndef arch_atomic64_fetch_inc_acquire
1765static __always_inline s64
1766arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1767{
1768 return arch_atomic64_fetch_add_acquire(1, v);
1769}3202}
1770#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1771#endif
1772 3203
1773#ifndef arch_atomic64_fetch_inc_release3204/**
3205 * raw_atomic64_fetch_inc() - atomic increment with full ordering
3206 * @v: pointer to atomic64_t
3207 *
3208 * Atomically updates @v to (@v + 1) with full ordering.
3209 *
3210 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
3211 *
3212 * Return: The original value of @v.
3213 */
1774static __always_inline s643214static __always_inline s64
1775arch_atomic64_fetch_inc_release(atomic64_t *v)3215raw_atomic64_fetch_inc(atomic64_t *v)
1776{3216{
1777 return arch_atomic64_fetch_add_release(1, v);3217#if defined(arch_atomic64_fetch_inc)
1778}3218 return arch_atomic64_fetch_inc(v);
1779#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release3219#elif defined(arch_atomic64_fetch_inc_relaxed)
3220 s64 ret;
3221 __atomic_pre_full_fence();
3222 ret = arch_atomic64_fetch_inc_relaxed(v);
3223 __atomic_post_full_fence();
3224 return ret;
3225#else
3226 return raw_atomic64_fetch_add(1, v);
1780#endif3227#endif
1781
1782#ifndef arch_atomic64_fetch_inc_relaxed
1783static __always_inline s64
1784arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1785{
1786 return arch_atomic64_fetch_add_relaxed(1, v);
1787}3228}
1788#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1789#endif
1790
1791#else /* arch_atomic64_fetch_inc_relaxed */
1792 3229
1793#ifndef arch_atomic64_fetch_inc_acquire3230/**
3231 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
3232 * @v: pointer to atomic64_t
3233 *
3234 * Atomically updates @v to (@v + 1) with acquire ordering.
3235 *
3236 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
3237 *
3238 * Return: The original value of @v.
3239 */
1794static __always_inline s643240static __always_inline s64
1795arch_atomic64_fetch_inc_acquire(atomic64_t *v)3241raw_atomic64_fetch_inc_acquire(atomic64_t *v)
1796{3242{
3243#if defined(arch_atomic64_fetch_inc_acquire)
3244 return arch_atomic64_fetch_inc_acquire(v);
3245#elif defined(arch_atomic64_fetch_inc_relaxed)
1797 s64 ret = arch_atomic64_fetch_inc_relaxed(v);3246 s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1798 __atomic_acquire_fence();3247 __atomic_acquire_fence();
1799 return ret;3248 return ret;
1800}3249#elif defined(arch_atomic64_fetch_inc)
1801#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire3250 return arch_atomic64_fetch_inc(v);
3251#else
3252 return raw_atomic64_fetch_add_acquire(1, v);
1802#endif3253#endif
3254}
1803 3255
1804#ifndef arch_atomic64_fetch_inc_release3256/**
3257 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
3258 * @v: pointer to atomic64_t
3259 *
3260 * Atomically updates @v to (@v + 1) with release ordering.
3261 *
3262 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
3263 *
3264 * Return: The original value of @v.
3265 */
1805static __always_inline s643266static __always_inline s64
1806arch_atomic64_fetch_inc_release(atomic64_t *v)3267raw_atomic64_fetch_inc_release(atomic64_t *v)
1807{3268{
3269#if defined(arch_atomic64_fetch_inc_release)
3270 return arch_atomic64_fetch_inc_release(v);
3271#elif defined(arch_atomic64_fetch_inc_relaxed)
1808 __atomic_release_fence();3272 __atomic_release_fence();
1809 return arch_atomic64_fetch_inc_relaxed(v);3273 return arch_atomic64_fetch_inc_relaxed(v);
1810}3274#elif defined(arch_atomic64_fetch_inc)
1811#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release3275 return arch_atomic64_fetch_inc(v);
1812#endif3276#else
1813 3277 return raw_atomic64_fetch_add_release(1, v);
1814#ifndef arch_atomic64_fetch_inc
1815static __always_inline s64
1816arch_atomic64_fetch_inc(atomic64_t *v)
1817{
1818 s64 ret;
1819 __atomic_pre_full_fence();
1820 ret = arch_atomic64_fetch_inc_relaxed(v);
1821 __atomic_post_full_fence();
1822 return ret;
1823}
1824#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1825#endif3278#endif
1826
1827#endif /* arch_atomic64_fetch_inc_relaxed */
1828
1829#ifndef arch_atomic64_dec
1830static __always_inline void
1831arch_atomic64_dec(atomic64_t *v)
1832{
1833 arch_atomic64_sub(1, v);
1834}3279}
1835#define arch_atomic64_dec arch_atomic64_dec
1836#endif
1837 3280
1838#ifndef arch_atomic64_dec_return_relaxed3281/**
1839#ifdef arch_atomic64_dec_return3282 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
1840#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return3283 * @v: pointer to atomic64_t
1841#define arch_atomic64_dec_return_release arch_atomic64_dec_return3284 *
1842#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return3285 * Atomically updates @v to (@v + 1) with relaxed ordering.
1843#endif /* arch_atomic64_dec_return */3286 *
1844 3287 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
1845#ifndef arch_atomic64_dec_return3288 *
3289 * Return: The original value of @v.
3290 */
1846static __always_inline s643291static __always_inline s64
1847arch_atomic64_dec_return(atomic64_t *v)3292raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
1848{3293{
1849 return arch_atomic64_sub_return(1, v);3294#if defined(arch_atomic64_fetch_inc_relaxed)
1850}3295 return arch_atomic64_fetch_inc_relaxed(v);
1851#define arch_atomic64_dec_return arch_atomic64_dec_return3296#elif defined(arch_atomic64_fetch_inc)
3297 return arch_atomic64_fetch_inc(v);
3298#else
3299 return raw_atomic64_fetch_add_relaxed(1, v);
1852#endif3300#endif
3301}
1853 3302
1854#ifndef arch_atomic64_dec_return_acquire3303/**
1855static __always_inline s643304 * raw_atomic64_dec() - atomic decrement with relaxed ordering
1856arch_atomic64_dec_return_acquire(atomic64_t *v)3305 * @v: pointer to atomic64_t
3306 *
3307 * Atomically updates @v to (@v - 1) with relaxed ordering.
3308 *
3309 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
3310 *
3311 * Return: Nothing.
3312 */
3313static __always_inline void
3314raw_atomic64_dec(atomic64_t *v)
1857{3315{
1858 return arch_atomic64_sub_return_acquire(1, v);3316#if defined(arch_atomic64_dec)
1859}3317 arch_atomic64_dec(v);
1860#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire3318#else
3319 raw_atomic64_sub(1, v);
1861#endif3320#endif
1862
1863#ifndef arch_atomic64_dec_return_release
1864static __always_inline s64
1865arch_atomic64_dec_return_release(atomic64_t *v)
1866{
1867 return arch_atomic64_sub_return_release(1, v);
1868}3321}
1869#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1870#endif
1871 3322
1872#ifndef arch_atomic64_dec_return_relaxed3323/**
3324 * raw_atomic64_dec_return() - atomic decrement with full ordering
3325 * @v: pointer to atomic64_t
3326 *
3327 * Atomically updates @v to (@v - 1) with full ordering.
3328 *
3329 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
3330 *
3331 * Return: The updated value of @v.
3332 */
1873static __always_inline s643333static __always_inline s64
1874arch_atomic64_dec_return_relaxed(atomic64_t *v)3334raw_atomic64_dec_return(atomic64_t *v)
1875{3335{
1876 return arch_atomic64_sub_return_relaxed(1, v);3336#if defined(arch_atomic64_dec_return)
1877}3337 return arch_atomic64_dec_return(v);
1878#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed3338#elif defined(arch_atomic64_dec_return_relaxed)
3339 s64 ret;
3340 __atomic_pre_full_fence();
3341 ret = arch_atomic64_dec_return_relaxed(v);
3342 __atomic_post_full_fence();
3343 return ret;
3344#else
3345 return raw_atomic64_sub_return(1, v);
1879#endif3346#endif
3347}
1880 3348
1881#else /* arch_atomic64_dec_return_relaxed */3349/**
1882 3350 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
1883#ifndef arch_atomic64_dec_return_acquire3351 * @v: pointer to atomic64_t
3352 *
3353 * Atomically updates @v to (@v - 1) with acquire ordering.
3354 *
3355 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
3356 *
3357 * Return: The updated value of @v.
3358 */
1884static __always_inline s643359static __always_inline s64
1885arch_atomic64_dec_return_acquire(atomic64_t *v)3360raw_atomic64_dec_return_acquire(atomic64_t *v)
1886{3361{
3362#if defined(arch_atomic64_dec_return_acquire)
3363 return arch_atomic64_dec_return_acquire(v);
3364#elif defined(arch_atomic64_dec_return_relaxed)
1887 s64 ret = arch_atomic64_dec_return_relaxed(v);3365 s64 ret = arch_atomic64_dec_return_relaxed(v);
1888 __atomic_acquire_fence();3366 __atomic_acquire_fence();
1889 return ret;3367 return ret;
1890}3368#elif defined(arch_atomic64_dec_return)
1891#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire3369 return arch_atomic64_dec_return(v);
3370#else
3371 return raw_atomic64_sub_return_acquire(1, v);
1892#endif3372#endif
3373}
1893 3374
1894#ifndef arch_atomic64_dec_return_release3375/**
3376 * raw_atomic64_dec_return_release() - atomic decrement with release ordering
3377 * @v: pointer to atomic64_t
3378 *
3379 * Atomically updates @v to (@v - 1) with release ordering.
3380 *
3381 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
3382 *
3383 * Return: The updated value of @v.
3384 */
1895static __always_inline s643385static __always_inline s64
1896arch_atomic64_dec_return_release(atomic64_t *v)3386raw_atomic64_dec_return_release(atomic64_t *v)
1897{3387{
3388#if defined(arch_atomic64_dec_return_release)
3389 return arch_atomic64_dec_return_release(v);
3390#elif defined(arch_atomic64_dec_return_relaxed)
1898 __atomic_release_fence();3391 __atomic_release_fence();
1899 return arch_atomic64_dec_return_relaxed(v);3392 return arch_atomic64_dec_return_relaxed(v);
3393#elif defined(arch_atomic64_dec_return)
3394 return arch_atomic64_dec_return(v);
3395#else
3396 return raw_atomic64_sub_return_release(1, v);
3397#endif
1900}3398}
1901#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release3399
3400/**
3401 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
3402 * @v: pointer to atomic64_t
3403 *
3404 * Atomically updates @v to (@v - 1) with relaxed ordering.
3405 *
3406 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
3407 *
3408 * Return: The updated value of @v.
3409 */
3410static __always_inline s64
3411raw_atomic64_dec_return_relaxed(atomic64_t *v)
3412{
3413#if defined(arch_atomic64_dec_return_relaxed)
3414 return arch_atomic64_dec_return_relaxed(v);
3415#elif defined(arch_atomic64_dec_return)
3416 return arch_atomic64_dec_return(v);
3417#else
3418 return raw_atomic64_sub_return_relaxed(1, v);
1902#endif3419#endif
3420}
1903 3421
1904#ifndef arch_atomic64_dec_return3422/**
3423 * raw_atomic64_fetch_dec() - atomic decrement with full ordering
3424 * @v: pointer to atomic64_t
3425 *
3426 * Atomically updates @v to (@v - 1) with full ordering.
3427 *
3428 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
3429 *
3430 * Return: The original value of @v.
3431 */
1905static __always_inline s643432static __always_inline s64
1906arch_atomic64_dec_return(atomic64_t *v)3433raw_atomic64_fetch_dec(atomic64_t *v)
1907{3434{
3435#if defined(arch_atomic64_fetch_dec)
3436 return arch_atomic64_fetch_dec(v);
3437#elif defined(arch_atomic64_fetch_dec_relaxed)
1908 s64 ret;3438 s64 ret;
1909 __atomic_pre_full_fence();3439 __atomic_pre_full_fence();
1910 ret = arch_atomic64_dec_return_relaxed(v);3440 ret = arch_atomic64_fetch_dec_relaxed(v);
1911 __atomic_post_full_fence();3441 __atomic_post_full_fence();
1912 return ret;3442 return ret;
1913}3443#else
1914#define arch_atomic64_dec_return arch_atomic64_dec_return3444 return raw_atomic64_fetch_sub(1, v);
1915#endif3445#endif
1916
1917#endif /* arch_atomic64_dec_return_relaxed */
1918
1919#ifndef arch_atomic64_fetch_dec_relaxed
1920#ifdef arch_atomic64_fetch_dec
1921#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1922#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1923#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1924#endif /* arch_atomic64_fetch_dec */
1925
1926#ifndef arch_atomic64_fetch_dec
1927static __always_inline s64
1928arch_atomic64_fetch_dec(atomic64_t *v)
1929{
1930 return arch_atomic64_fetch_sub(1, v);
1931}3446}
1932#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1933#endif
1934 3447
1935#ifndef arch_atomic64_fetch_dec_acquire3448/**
3449 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
3450 * @v: pointer to atomic64_t
3451 *
3452 * Atomically updates @v to (@v - 1) with acquire ordering.
3453 *
3454 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
3455 *
3456 * Return: The original value of @v.
3457 */
1936static __always_inline s643458static __always_inline s64
1937arch_atomic64_fetch_dec_acquire(atomic64_t *v)3459raw_atomic64_fetch_dec_acquire(atomic64_t *v)
1938{3460{
1939 return arch_atomic64_fetch_sub_acquire(1, v);3461#if defined(arch_atomic64_fetch_dec_acquire)
1940}3462 return arch_atomic64_fetch_dec_acquire(v);
1941#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire3463#elif defined(arch_atomic64_fetch_dec_relaxed)
3464 s64 ret = arch_atomic64_fetch_dec_relaxed(v);
3465 __atomic_acquire_fence();
3466 return ret;
3467#elif defined(arch_atomic64_fetch_dec)
3468 return arch_atomic64_fetch_dec(v);
3469#else
3470 return raw_atomic64_fetch_sub_acquire(1, v);
1942#endif3471#endif
1943
1944#ifndef arch_atomic64_fetch_dec_release
1945static __always_inline s64
1946arch_atomic64_fetch_dec_release(atomic64_t *v)
1947{
1948 return arch_atomic64_fetch_sub_release(1, v);
1949}3472}
1950#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1951#endif
1952 3473
1953#ifndef arch_atomic64_fetch_dec_relaxed3474/**
3475 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
3476 * @v: pointer to atomic64_t
3477 *
3478 * Atomically updates @v to (@v - 1) with release ordering.
3479 *
3480 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
3481 *
3482 * Return: The original value of @v.
3483 */
1954static __always_inline s643484static __always_inline s64
1955arch_atomic64_fetch_dec_relaxed(atomic64_t *v)3485raw_atomic64_fetch_dec_release(atomic64_t *v)
1956{3486{
1957 return arch_atomic64_fetch_sub_relaxed(1, v);3487#if defined(arch_atomic64_fetch_dec_release)
1958}3488 return arch_atomic64_fetch_dec_release(v);
1959#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed3489#elif defined(arch_atomic64_fetch_dec_relaxed)
3490 __atomic_release_fence();
3491 return arch_atomic64_fetch_dec_relaxed(v);
3492#elif defined(arch_atomic64_fetch_dec)
3493 return arch_atomic64_fetch_dec(v);
3494#else
3495 return raw_atomic64_fetch_sub_release(1, v);
1960#endif3496#endif
3497}
1961 3498
1962#else /* arch_atomic64_fetch_dec_relaxed */3499/**
1963 3500 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
1964#ifndef arch_atomic64_fetch_dec_acquire3501 * @v: pointer to atomic64_t
3502 *
3503 * Atomically updates @v to (@v - 1) with relaxed ordering.
3504 *
3505 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
3506 *
3507 * Return: The original value of @v.
3508 */
1965static __always_inline s643509static __always_inline s64
1966arch_atomic64_fetch_dec_acquire(atomic64_t *v)3510raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
1967{3511{
1968 s64 ret = arch_atomic64_fetch_dec_relaxed(v);3512#if defined(arch_atomic64_fetch_dec_relaxed)
1969 __atomic_acquire_fence();3513 return arch_atomic64_fetch_dec_relaxed(v);
1970 return ret;3514#elif defined(arch_atomic64_fetch_dec)
1971}3515 return arch_atomic64_fetch_dec(v);
1972#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire3516#else
3517 return raw_atomic64_fetch_sub_relaxed(1, v);
1973#endif3518#endif
3519}
1974 3520
1975#ifndef arch_atomic64_fetch_dec_release3521/**
1976static __always_inline s643522 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
1977arch_atomic64_fetch_dec_release(atomic64_t *v)3523 * @i: s64 value
3524 * @v: pointer to atomic64_t
3525 *
3526 * Atomically updates @v to (@v & @i) with relaxed ordering.
3527 *
3528 * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
3529 *
3530 * Return: Nothing.
3531 */
3532static __always_inline void
3533raw_atomic64_and(s64 i, atomic64_t *v)
1978{3534{
1979 __atomic_release_fence();3535 arch_atomic64_and(i, v);
1980 return arch_atomic64_fetch_dec_relaxed(v);
1981}3536}
1982#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1983#endif
1984 3537
1985#ifndef arch_atomic64_fetch_dec3538/**
3539 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
3540 * @i: s64 value
3541 * @v: pointer to atomic64_t
3542 *
3543 * Atomically updates @v to (@v & @i) with full ordering.
3544 *
3545 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
3546 *
3547 * Return: The original value of @v.
3548 */
1986static __always_inline s643549static __always_inline s64
1987arch_atomic64_fetch_dec(atomic64_t *v)3550raw_atomic64_fetch_and(s64 i, atomic64_t *v)
1988{3551{
3552#if defined(arch_atomic64_fetch_and)
3553 return arch_atomic64_fetch_and(i, v);
3554#elif defined(arch_atomic64_fetch_and_relaxed)
1989 s64 ret;3555 s64 ret;
1990 __atomic_pre_full_fence();3556 __atomic_pre_full_fence();
1991 ret = arch_atomic64_fetch_dec_relaxed(v);3557 ret = arch_atomic64_fetch_and_relaxed(i, v);
1992 __atomic_post_full_fence();3558 __atomic_post_full_fence();
1993 return ret;3559 return ret;
1994}3560#else
1995#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec3561#error "Unable to define raw_atomic64_fetch_and"
1996#endif3562#endif
3563}
1997 3564
1998#endif /* arch_atomic64_fetch_dec_relaxed */3565/**
1999 3566 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
2000#ifndef arch_atomic64_fetch_and_relaxed3567 * @i: s64 value
2001#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and3568 * @v: pointer to atomic64_t
2002#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and3569 *
2003#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and3570 * Atomically updates @v to (@v & @i) with acquire ordering.
2004#else /* arch_atomic64_fetch_and_relaxed */3571 *
2005 3572 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
2006#ifndef arch_atomic64_fetch_and_acquire3573 *
3574 * Return: The original value of @v.
3575 */
2007static __always_inline s643576static __always_inline s64
2008arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)3577raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2009{3578{
3579#if defined(arch_atomic64_fetch_and_acquire)
3580 return arch_atomic64_fetch_and_acquire(i, v);
3581#elif defined(arch_atomic64_fetch_and_relaxed)
2010 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);3582 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
2011 __atomic_acquire_fence();3583 __atomic_acquire_fence();
2012 return ret;3584 return ret;
2013}3585#elif defined(arch_atomic64_fetch_and)
2014#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire3586 return arch_atomic64_fetch_and(i, v);
3587#else
3588#error "Unable to define raw_atomic64_fetch_and_acquire"
2015#endif3589#endif
3590}
2016 3591
2017#ifndef arch_atomic64_fetch_and_release3592/**
3593 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
3594 * @i: s64 value
3595 * @v: pointer to atomic64_t
3596 *
3597 * Atomically updates @v to (@v & @i) with release ordering.
3598 *
3599 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
3600 *
3601 * Return: The original value of @v.
3602 */
2018static __always_inline s643603static __always_inline s64
2019arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)3604raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
2020{3605{
3606#if defined(arch_atomic64_fetch_and_release)
3607 return arch_atomic64_fetch_and_release(i, v);
3608#elif defined(arch_atomic64_fetch_and_relaxed)
2021 __atomic_release_fence();3609 __atomic_release_fence();
2022 return arch_atomic64_fetch_and_relaxed(i, v);3610 return arch_atomic64_fetch_and_relaxed(i, v);
2023}3611#elif defined(arch_atomic64_fetch_and)
2024#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release3612 return arch_atomic64_fetch_and(i, v);
3613#else
3614#error "Unable to define raw_atomic64_fetch_and_release"
2025#endif3615#endif
3616}
2026 3617
2027#ifndef arch_atomic64_fetch_and3618/**
3619 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3620 * @i: s64 value
3621 * @v: pointer to atomic64_t
3622 *
3623 * Atomically updates @v to (@v & @i) with relaxed ordering.
3624 *
3625 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
3626 *
3627 * Return: The original value of @v.
3628 */
2028static __always_inline s643629static __always_inline s64
2029arch_atomic64_fetch_and(s64 i, atomic64_t *v)3630raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
2030{3631{
2031 s64 ret;3632#if defined(arch_atomic64_fetch_and_relaxed)
2032 __atomic_pre_full_fence();3633 return arch_atomic64_fetch_and_relaxed(i, v);
2033 ret = arch_atomic64_fetch_and_relaxed(i, v);3634#elif defined(arch_atomic64_fetch_and)
2034 __atomic_post_full_fence();3635 return arch_atomic64_fetch_and(i, v);
2035 return ret;3636#else
2036}3637#error "Unable to define raw_atomic64_fetch_and_relaxed"
2037#define arch_atomic64_fetch_and arch_atomic64_fetch_and
2038#endif3638#endif
2039
2040#endif /* arch_atomic64_fetch_and_relaxed */
2041
2042#ifndef arch_atomic64_andnot
2043static __always_inline void
2044arch_atomic64_andnot(s64 i, atomic64_t *v)
2045{
2046 arch_atomic64_and(~i, v);
2047}3639}
2048#define arch_atomic64_andnot arch_atomic64_andnot
2049#endif
2050
2051#ifndef arch_atomic64_fetch_andnot_relaxed
2052#ifdef arch_atomic64_fetch_andnot
2053#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
2054#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
2055#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
2056#endif /* arch_atomic64_fetch_andnot */
2057 3640
2058#ifndef arch_atomic64_fetch_andnot3641/**
2059static __always_inline s643642 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
2060arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)3643 * @i: s64 value
3644 * @v: pointer to atomic64_t
3645 *
3646 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3647 *
3648 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
3649 *
3650 * Return: Nothing.
3651 */
3652static __always_inline void
3653raw_atomic64_andnot(s64 i, atomic64_t *v)
2061{3654{
2062 return arch_atomic64_fetch_and(~i, v);3655#if defined(arch_atomic64_andnot)
2063}3656 arch_atomic64_andnot(i, v);
2064#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot3657#else
3658 raw_atomic64_and(~i, v);
2065#endif3659#endif
2066
2067#ifndef arch_atomic64_fetch_andnot_acquire
2068static __always_inline s64
2069arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2070{
2071 return arch_atomic64_fetch_and_acquire(~i, v);
2072}3660}
2073#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
2074#endif
2075 3661
2076#ifndef arch_atomic64_fetch_andnot_release3662/**
3663 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
3664 * @i: s64 value
3665 * @v: pointer to atomic64_t
3666 *
3667 * Atomically updates @v to (@v & ~@i) with full ordering.
3668 *
3669 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
3670 *
3671 * Return: The original value of @v.
3672 */
2077static __always_inline s643673static __always_inline s64
2078arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)3674raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
2079{3675{
2080 return arch_atomic64_fetch_and_release(~i, v);3676#if defined(arch_atomic64_fetch_andnot)
2081}3677 return arch_atomic64_fetch_andnot(i, v);
2082#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release3678#elif defined(arch_atomic64_fetch_andnot_relaxed)
3679 s64 ret;
3680 __atomic_pre_full_fence();
3681 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3682 __atomic_post_full_fence();
3683 return ret;
3684#else
3685 return raw_atomic64_fetch_and(~i, v);
2083#endif3686#endif
2084
2085#ifndef arch_atomic64_fetch_andnot_relaxed
2086static __always_inline s64
2087arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2088{
2089 return arch_atomic64_fetch_and_relaxed(~i, v);
2090}3687}
2091#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
2092#endif
2093
2094#else /* arch_atomic64_fetch_andnot_relaxed */
2095 3688
2096#ifndef arch_atomic64_fetch_andnot_acquire3689/**
3690 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3691 * @i: s64 value
3692 * @v: pointer to atomic64_t
3693 *
3694 * Atomically updates @v to (@v & ~@i) with acquire ordering.
3695 *
3696 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
3697 *
3698 * Return: The original value of @v.
3699 */
2097static __always_inline s643700static __always_inline s64
2098arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)3701raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2099{3702{
3703#if defined(arch_atomic64_fetch_andnot_acquire)
3704 return arch_atomic64_fetch_andnot_acquire(i, v);
3705#elif defined(arch_atomic64_fetch_andnot_relaxed)
2100 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);3706 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
2101 __atomic_acquire_fence();3707 __atomic_acquire_fence();
2102 return ret;3708 return ret;
2103}3709#elif defined(arch_atomic64_fetch_andnot)
2104#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire3710 return arch_atomic64_fetch_andnot(i, v);
3711#else
3712 return raw_atomic64_fetch_and_acquire(~i, v);
2105#endif3713#endif
3714}
2106 3715
2107#ifndef arch_atomic64_fetch_andnot_release3716/**
3717 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
3718 * @i: s64 value
3719 * @v: pointer to atomic64_t
3720 *
3721 * Atomically updates @v to (@v & ~@i) with release ordering.
3722 *
3723 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
3724 *
3725 * Return: The original value of @v.
3726 */
2108static __always_inline s643727static __always_inline s64
2109arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)3728raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2110{3729{
3730#if defined(arch_atomic64_fetch_andnot_release)
3731 return arch_atomic64_fetch_andnot_release(i, v);
3732#elif defined(arch_atomic64_fetch_andnot_relaxed)
2111 __atomic_release_fence();3733 __atomic_release_fence();
2112 return arch_atomic64_fetch_andnot_relaxed(i, v);3734 return arch_atomic64_fetch_andnot_relaxed(i, v);
3735#elif defined(arch_atomic64_fetch_andnot)
3736 return arch_atomic64_fetch_andnot(i, v);
3737#else
3738 return raw_atomic64_fetch_and_release(~i, v);
3739#endif
2113}3740}
2114#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release3741
3742/**
3743 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering

1241 more lines aren’t shown here.

include/linux/atomic/atomic-instrumented.h+3194−361
@@ -4,15 +4,10 @@ · 3 unmodified lines
4// DO NOT MODIFY THIS FILE DIRECTLY4// DO NOT MODIFY THIS FILE DIRECTLY
5 5
6/*6/*
7 * This file provides wrappers with KASAN instrumentation for atomic operations.7 * This file provoides atomic operations with explicit instrumentation (e.g.
8 * To use this functionality an arch's atomic.h file needs to define all8 * KASAN, KCSAN), which should be used unless it is necessary to avoid
9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include9 * instrumentation. Where it is necessary to aovid instrumenation, the
10 * this file at the end. This file provides atomic_read() that forwards to10 * raw_atomic*() operations should be used.
11 * arch_atomic_read() for actual atomic operation.
12 * Note: if an arch atomic operation is implemented by means of other atomic
13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 * double instrumentation.
16 */11 */
17#ifndef _LINUX_ATOMIC_INSTRUMENTED_H12#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18#define _LINUX_ATOMIC_INSTRUMENTED_H13#define _LINUX_ATOMIC_INSTRUMENTED_H
@@ -21,1927 +16,4696 @@ · 2 unmodified lines
21#include <linux/compiler.h>16#include <linux/compiler.h>
22#include <linux/instrumented.h>17#include <linux/instrumented.h>
23 18
19/**
20 * atomic_read() - atomic load with relaxed ordering
21 * @v: pointer to atomic_t
22 *
23 * Atomically loads the value of @v with relaxed ordering.
24 *
25 * Unsafe to use in noinstr code; use raw_atomic_read() there.
26 *
27 * Return: The value loaded from @v.
28 */
24static __always_inline int29static __always_inline int
25atomic_read(const atomic_t *v)30atomic_read(const atomic_t *v)
26{31{
27 instrument_atomic_read(v, sizeof(*v));32 instrument_atomic_read(v, sizeof(*v));
28 return arch_atomic_read(v);33 return raw_atomic_read(v);
29}34}
30 35
36/**
37 * atomic_read_acquire() - atomic load with acquire ordering
38 * @v: pointer to atomic_t
39 *
40 * Atomically loads the value of @v with acquire ordering.
41 *
42 * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
43 *
44 * Return: The value loaded from @v.
45 */
31static __always_inline int46static __always_inline int
32atomic_read_acquire(const atomic_t *v)47atomic_read_acquire(const atomic_t *v)
33{48{
34 instrument_atomic_read(v, sizeof(*v));49 instrument_atomic_read(v, sizeof(*v));
35 return arch_atomic_read_acquire(v);50 return raw_atomic_read_acquire(v);
36}51}
37 52
53/**
54 * atomic_set() - atomic set with relaxed ordering
55 * @v: pointer to atomic_t
56 * @i: int value to assign
57 *
58 * Atomically sets @v to @i with relaxed ordering.
59 *
60 * Unsafe to use in noinstr code; use raw_atomic_set() there.
61 *
62 * Return: Nothing.
63 */
38static __always_inline void64static __always_inline void
39atomic_set(atomic_t *v, int i)65atomic_set(atomic_t *v, int i)
40{66{
41 instrument_atomic_write(v, sizeof(*v));67 instrument_atomic_write(v, sizeof(*v));
42 arch_atomic_set(v, i);68 raw_atomic_set(v, i);
43}69}
44 70
71/**
72 * atomic_set_release() - atomic set with release ordering
73 * @v: pointer to atomic_t
74 * @i: int value to assign
75 *
76 * Atomically sets @v to @i with release ordering.
77 *
78 * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
79 *
80 * Return: Nothing.
81 */
45static __always_inline void82static __always_inline void
46atomic_set_release(atomic_t *v, int i)83atomic_set_release(atomic_t *v, int i)
47{84{
48 kcsan_release();85 kcsan_release();
49 instrument_atomic_write(v, sizeof(*v));86 instrument_atomic_write(v, sizeof(*v));
50 arch_atomic_set_release(v, i);87 raw_atomic_set_release(v, i);
51}88}
52 89
90/**
91 * atomic_add() - atomic add with relaxed ordering
92 * @i: int value to add
93 * @v: pointer to atomic_t
94 *
95 * Atomically updates @v to (@v + @i) with relaxed ordering.
96 *
97 * Unsafe to use in noinstr code; use raw_atomic_add() there.
98 *
99 * Return: Nothing.
100 */
53static __always_inline void101static __always_inline void
54atomic_add(int i, atomic_t *v)102atomic_add(int i, atomic_t *v)
55{103{
56 instrument_atomic_read_write(v, sizeof(*v));104 instrument_atomic_read_write(v, sizeof(*v));
57 arch_atomic_add(i, v);105 raw_atomic_add(i, v);
58}106}
59 107
108/**
109 * atomic_add_return() - atomic add with full ordering
110 * @i: int value to add
111 * @v: pointer to atomic_t
112 *
113 * Atomically updates @v to (@v + @i) with full ordering.
114 *
115 * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
116 *
117 * Return: The updated value of @v.
118 */
60static __always_inline int119static __always_inline int
61atomic_add_return(int i, atomic_t *v)120atomic_add_return(int i, atomic_t *v)
62{121{
63 kcsan_mb();122 kcsan_mb();
64 instrument_atomic_read_write(v, sizeof(*v));123 instrument_atomic_read_write(v, sizeof(*v));
65 return arch_atomic_add_return(i, v);124 return raw_atomic_add_return(i, v);
66}125}
67 126
127/**
128 * atomic_add_return_acquire() - atomic add with acquire ordering
129 * @i: int value to add
130 * @v: pointer to atomic_t
131 *
132 * Atomically updates @v to (@v + @i) with acquire ordering.
133 *
134 * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
135 *
136 * Return: The updated value of @v.
137 */
68static __always_inline int138static __always_inline int
69atomic_add_return_acquire(int i, atomic_t *v)139atomic_add_return_acquire(int i, atomic_t *v)
70{140{
71 instrument_atomic_read_write(v, sizeof(*v));141 instrument_atomic_read_write(v, sizeof(*v));
72 return arch_atomic_add_return_acquire(i, v);142 return raw_atomic_add_return_acquire(i, v);
73}143}
74 144
145/**
146 * atomic_add_return_release() - atomic add with release ordering
147 * @i: int value to add
148 * @v: pointer to atomic_t
149 *
150 * Atomically updates @v to (@v + @i) with release ordering.
151 *
152 * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
153 *
154 * Return: The updated value of @v.
155 */
75static __always_inline int156static __always_inline int
76atomic_add_return_release(int i, atomic_t *v)157atomic_add_return_release(int i, atomic_t *v)
77{158{
78 kcsan_release();159 kcsan_release();
79 instrument_atomic_read_write(v, sizeof(*v));160 instrument_atomic_read_write(v, sizeof(*v));
80 return arch_atomic_add_return_release(i, v);161 return raw_atomic_add_return_release(i, v);
81}162}
82 163
164/**
165 * atomic_add_return_relaxed() - atomic add with relaxed ordering
166 * @i: int value to add
167 * @v: pointer to atomic_t
168 *
169 * Atomically updates @v to (@v + @i) with relaxed ordering.
170 *
171 * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
172 *
173 * Return: The updated value of @v.
174 */
83static __always_inline int175static __always_inline int
84atomic_add_return_relaxed(int i, atomic_t *v)176atomic_add_return_relaxed(int i, atomic_t *v)
85{177{
86 instrument_atomic_read_write(v, sizeof(*v));178 instrument_atomic_read_write(v, sizeof(*v));
87 return arch_atomic_add_return_relaxed(i, v);179 return raw_atomic_add_return_relaxed(i, v);
88}180}
89 181
182/**
183 * atomic_fetch_add() - atomic add with full ordering
184 * @i: int value to add
185 * @v: pointer to atomic_t
186 *
187 * Atomically updates @v to (@v + @i) with full ordering.
188 *
189 * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
190 *
191 * Return: The original value of @v.
192 */
90static __always_inline int193static __always_inline int
91atomic_fetch_add(int i, atomic_t *v)194atomic_fetch_add(int i, atomic_t *v)
92{195{
93 kcsan_mb();196 kcsan_mb();
94 instrument_atomic_read_write(v, sizeof(*v));197 instrument_atomic_read_write(v, sizeof(*v));
95 return arch_atomic_fetch_add(i, v);198 return raw_atomic_fetch_add(i, v);
96}199}
97 200
201/**
202 * atomic_fetch_add_acquire() - atomic add with acquire ordering
203 * @i: int value to add
204 * @v: pointer to atomic_t
205 *
206 * Atomically updates @v to (@v + @i) with acquire ordering.
207 *
208 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
209 *
210 * Return: The original value of @v.
211 */
98static __always_inline int212static __always_inline int
99atomic_fetch_add_acquire(int i, atomic_t *v)213atomic_fetch_add_acquire(int i, atomic_t *v)
100{214{
101 instrument_atomic_read_write(v, sizeof(*v));215 instrument_atomic_read_write(v, sizeof(*v));
102 return arch_atomic_fetch_add_acquire(i, v);216 return raw_atomic_fetch_add_acquire(i, v);
103}217}
104 218
219/**
220 * atomic_fetch_add_release() - atomic add with release ordering
221 * @i: int value to add
222 * @v: pointer to atomic_t
223 *
224 * Atomically updates @v to (@v + @i) with release ordering.
225 *
226 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
227 *
228 * Return: The original value of @v.
229 */
105static __always_inline int230static __always_inline int
106atomic_fetch_add_release(int i, atomic_t *v)231atomic_fetch_add_release(int i, atomic_t *v)
107{232{
108 kcsan_release();233 kcsan_release();
109 instrument_atomic_read_write(v, sizeof(*v));234 instrument_atomic_read_write(v, sizeof(*v));
110 return arch_atomic_fetch_add_release(i, v);235 return raw_atomic_fetch_add_release(i, v);
111}236}
112 237
238/**
239 * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
240 * @i: int value to add
241 * @v: pointer to atomic_t
242 *
243 * Atomically updates @v to (@v + @i) with relaxed ordering.
244 *
245 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
246 *
247 * Return: The original value of @v.
248 */
113static __always_inline int249static __always_inline int
114atomic_fetch_add_relaxed(int i, atomic_t *v)250atomic_fetch_add_relaxed(int i, atomic_t *v)
115{251{
116 instrument_atomic_read_write(v, sizeof(*v));252 instrument_atomic_read_write(v, sizeof(*v));
117 return arch_atomic_fetch_add_relaxed(i, v);253 return raw_atomic_fetch_add_relaxed(i, v);
118}254}
119 255
256/**
257 * atomic_sub() - atomic subtract with relaxed ordering
258 * @i: int value to subtract
259 * @v: pointer to atomic_t
260 *
261 * Atomically updates @v to (@v - @i) with relaxed ordering.
262 *
263 * Unsafe to use in noinstr code; use raw_atomic_sub() there.
264 *
265 * Return: Nothing.
266 */
120static __always_inline void267static __always_inline void
121atomic_sub(int i, atomic_t *v)268atomic_sub(int i, atomic_t *v)
122{269{
123 instrument_atomic_read_write(v, sizeof(*v));270 instrument_atomic_read_write(v, sizeof(*v));
124 arch_atomic_sub(i, v);271 raw_atomic_sub(i, v);
125}272}
126 273
274/**
275 * atomic_sub_return() - atomic subtract with full ordering
276 * @i: int value to subtract
277 * @v: pointer to atomic_t
278 *
279 * Atomically updates @v to (@v - @i) with full ordering.
280 *
281 * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
282 *
283 * Return: The updated value of @v.
284 */
127static __always_inline int285static __always_inline int
128atomic_sub_return(int i, atomic_t *v)286atomic_sub_return(int i, atomic_t *v)
129{287{
130 kcsan_mb();288 kcsan_mb();
131 instrument_atomic_read_write(v, sizeof(*v));289 instrument_atomic_read_write(v, sizeof(*v));
132 return arch_atomic_sub_return(i, v);290 return raw_atomic_sub_return(i, v);
133}291}
134 292
293/**
294 * atomic_sub_return_acquire() - atomic subtract with acquire ordering
295 * @i: int value to subtract
296 * @v: pointer to atomic_t
297 *
298 * Atomically updates @v to (@v - @i) with acquire ordering.
299 *
300 * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
301 *
302 * Return: The updated value of @v.
303 */
135static __always_inline int304static __always_inline int
136atomic_sub_return_acquire(int i, atomic_t *v)305atomic_sub_return_acquire(int i, atomic_t *v)
137{306{
138 instrument_atomic_read_write(v, sizeof(*v));307 instrument_atomic_read_write(v, sizeof(*v));
139 return arch_atomic_sub_return_acquire(i, v);308 return raw_atomic_sub_return_acquire(i, v);
140}309}
141 310
311/**
312 * atomic_sub_return_release() - atomic subtract with release ordering
313 * @i: int value to subtract
314 * @v: pointer to atomic_t
315 *
316 * Atomically updates @v to (@v - @i) with release ordering.
317 *
318 * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
319 *
320 * Return: The updated value of @v.
321 */
142static __always_inline int322static __always_inline int
143atomic_sub_return_release(int i, atomic_t *v)323atomic_sub_return_release(int i, atomic_t *v)
144{324{
145 kcsan_release();325 kcsan_release();
146 instrument_atomic_read_write(v, sizeof(*v));326 instrument_atomic_read_write(v, sizeof(*v));
147 return arch_atomic_sub_return_release(i, v);327 return raw_atomic_sub_return_release(i, v);
148}328}
149 329
330/**
331 * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
332 * @i: int value to subtract
333 * @v: pointer to atomic_t
334 *
335 * Atomically updates @v to (@v - @i) with relaxed ordering.
336 *
337 * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
338 *
339 * Return: The updated value of @v.
340 */
150static __always_inline int341static __always_inline int
151atomic_sub_return_relaxed(int i, atomic_t *v)342atomic_sub_return_relaxed(int i, atomic_t *v)
152{343{
153 instrument_atomic_read_write(v, sizeof(*v));344 instrument_atomic_read_write(v, sizeof(*v));
154 return arch_atomic_sub_return_relaxed(i, v);345 return raw_atomic_sub_return_relaxed(i, v);
155}346}
156 347
348/**
349 * atomic_fetch_sub() - atomic subtract with full ordering
350 * @i: int value to subtract
351 * @v: pointer to atomic_t
352 *
353 * Atomically updates @v to (@v - @i) with full ordering.
354 *
355 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
356 *
357 * Return: The original value of @v.
358 */
157static __always_inline int359static __always_inline int
158atomic_fetch_sub(int i, atomic_t *v)360atomic_fetch_sub(int i, atomic_t *v)
159{361{
160 kcsan_mb();362 kcsan_mb();
161 instrument_atomic_read_write(v, sizeof(*v));363 instrument_atomic_read_write(v, sizeof(*v));
162 return arch_atomic_fetch_sub(i, v);364 return raw_atomic_fetch_sub(i, v);
163}365}
164 366
367/**
368 * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
369 * @i: int value to subtract
370 * @v: pointer to atomic_t
371 *
372 * Atomically updates @v to (@v - @i) with acquire ordering.
373 *
374 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
375 *
376 * Return: The original value of @v.
377 */
165static __always_inline int378static __always_inline int
166atomic_fetch_sub_acquire(int i, atomic_t *v)379atomic_fetch_sub_acquire(int i, atomic_t *v)
167{380{
168 instrument_atomic_read_write(v, sizeof(*v));381 instrument_atomic_read_write(v, sizeof(*v));
169 return arch_atomic_fetch_sub_acquire(i, v);382 return raw_atomic_fetch_sub_acquire(i, v);
170}383}
171 384
385/**
386 * atomic_fetch_sub_release() - atomic subtract with release ordering
387 * @i: int value to subtract
388 * @v: pointer to atomic_t
389 *
390 * Atomically updates @v to (@v - @i) with release ordering.
391 *
392 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
393 *
394 * Return: The original value of @v.
395 */
172static __always_inline int396static __always_inline int
173atomic_fetch_sub_release(int i, atomic_t *v)397atomic_fetch_sub_release(int i, atomic_t *v)
174{398{
175 kcsan_release();399 kcsan_release();
176 instrument_atomic_read_write(v, sizeof(*v));400 instrument_atomic_read_write(v, sizeof(*v));
177 return arch_atomic_fetch_sub_release(i, v);401 return raw_atomic_fetch_sub_release(i, v);
178}402}
179 403
404/**
405 * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
406 * @i: int value to subtract
407 * @v: pointer to atomic_t
408 *
409 * Atomically updates @v to (@v - @i) with relaxed ordering.
410 *
411 * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
412 *
413 * Return: The original value of @v.
414 */
180static __always_inline int415static __always_inline int
181atomic_fetch_sub_relaxed(int i, atomic_t *v)416atomic_fetch_sub_relaxed(int i, atomic_t *v)
182{417{
183 instrument_atomic_read_write(v, sizeof(*v));418 instrument_atomic_read_write(v, sizeof(*v));
184 return arch_atomic_fetch_sub_relaxed(i, v);419 return raw_atomic_fetch_sub_relaxed(i, v);
185}420}
186 421
422/**
423 * atomic_inc() - atomic increment with relaxed ordering
424 * @v: pointer to atomic_t
425 *
426 * Atomically updates @v to (@v + 1) with relaxed ordering.
427 *
428 * Unsafe to use in noinstr code; use raw_atomic_inc() there.
429 *
430 * Return: Nothing.
431 */
187static __always_inline void432static __always_inline void
188atomic_inc(atomic_t *v)433atomic_inc(atomic_t *v)
189{434{
190 instrument_atomic_read_write(v, sizeof(*v));435 instrument_atomic_read_write(v, sizeof(*v));
191 arch_atomic_inc(v);436 raw_atomic_inc(v);
192}437}
193 438
439/**
440 * atomic_inc_return() - atomic increment with full ordering
441 * @v: pointer to atomic_t
442 *
443 * Atomically updates @v to (@v + 1) with full ordering.
444 *
445 * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
446 *
447 * Return: The updated value of @v.
448 */
194static __always_inline int449static __always_inline int
195atomic_inc_return(atomic_t *v)450atomic_inc_return(atomic_t *v)
196{451{
197 kcsan_mb();452 kcsan_mb();
198 instrument_atomic_read_write(v, sizeof(*v));453 instrument_atomic_read_write(v, sizeof(*v));
199 return arch_atomic_inc_return(v);454 return raw_atomic_inc_return(v);
200}455}
201 456
457/**
458 * atomic_inc_return_acquire() - atomic increment with acquire ordering
459 * @v: pointer to atomic_t
460 *
461 * Atomically updates @v to (@v + 1) with acquire ordering.
462 *
463 * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
464 *
465 * Return: The updated value of @v.
466 */
202static __always_inline int467static __always_inline int
203atomic_inc_return_acquire(atomic_t *v)468atomic_inc_return_acquire(atomic_t *v)
204{469{
205 instrument_atomic_read_write(v, sizeof(*v));470 instrument_atomic_read_write(v, sizeof(*v));
206 return arch_atomic_inc_return_acquire(v);471 return raw_atomic_inc_return_acquire(v);
207}472}
208 473
474/**
475 * atomic_inc_return_release() - atomic increment with release ordering
476 * @v: pointer to atomic_t
477 *
478 * Atomically updates @v to (@v + 1) with release ordering.
479 *
480 * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
481 *
482 * Return: The updated value of @v.
483 */
209static __always_inline int484static __always_inline int
210atomic_inc_return_release(atomic_t *v)485atomic_inc_return_release(atomic_t *v)
211{486{
212 kcsan_release();487 kcsan_release();
213 instrument_atomic_read_write(v, sizeof(*v));488 instrument_atomic_read_write(v, sizeof(*v));
214 return arch_atomic_inc_return_release(v);489 return raw_atomic_inc_return_release(v);
215}490}
216 491
492/**
493 * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
494 * @v: pointer to atomic_t
495 *
496 * Atomically updates @v to (@v + 1) with relaxed ordering.
497 *
498 * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
499 *
500 * Return: The updated value of @v.
501 */
217static __always_inline int502static __always_inline int
218atomic_inc_return_relaxed(atomic_t *v)503atomic_inc_return_relaxed(atomic_t *v)
219{504{
220 instrument_atomic_read_write(v, sizeof(*v));505 instrument_atomic_read_write(v, sizeof(*v));
221 return arch_atomic_inc_return_relaxed(v);506 return raw_atomic_inc_return_relaxed(v);
222}507}
223 508
509/**
510 * atomic_fetch_inc() - atomic increment with full ordering
511 * @v: pointer to atomic_t
512 *
513 * Atomically updates @v to (@v + 1) with full ordering.
514 *
515 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
516 *
517 * Return: The original value of @v.
518 */
224static __always_inline int519static __always_inline int
225atomic_fetch_inc(atomic_t *v)520atomic_fetch_inc(atomic_t *v)
226{521{
227 kcsan_mb();522 kcsan_mb();
228 instrument_atomic_read_write(v, sizeof(*v));523 instrument_atomic_read_write(v, sizeof(*v));
229 return arch_atomic_fetch_inc(v);524 return raw_atomic_fetch_inc(v);
230}525}
231 526
527/**
528 * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
529 * @v: pointer to atomic_t
530 *
531 * Atomically updates @v to (@v + 1) with acquire ordering.
532 *
533 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
534 *
535 * Return: The original value of @v.
536 */
232static __always_inline int537static __always_inline int
233atomic_fetch_inc_acquire(atomic_t *v)538atomic_fetch_inc_acquire(atomic_t *v)
234{539{
235 instrument_atomic_read_write(v, sizeof(*v));540 instrument_atomic_read_write(v, sizeof(*v));
236 return arch_atomic_fetch_inc_acquire(v);541 return raw_atomic_fetch_inc_acquire(v);
237}542}
238 543
544/**
545 * atomic_fetch_inc_release() - atomic increment with release ordering
546 * @v: pointer to atomic_t
547 *
548 * Atomically updates @v to (@v + 1) with release ordering.
549 *
550 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
551 *
552 * Return: The original value of @v.
553 */
239static __always_inline int554static __always_inline int
240atomic_fetch_inc_release(atomic_t *v)555atomic_fetch_inc_release(atomic_t *v)
241{556{
242 kcsan_release();557 kcsan_release();
243 instrument_atomic_read_write(v, sizeof(*v));558 instrument_atomic_read_write(v, sizeof(*v));
244 return arch_atomic_fetch_inc_release(v);559 return raw_atomic_fetch_inc_release(v);
245}560}
246 561
562/**
563 * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
564 * @v: pointer to atomic_t
565 *
566 * Atomically updates @v to (@v + 1) with relaxed ordering.
567 *
568 * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
569 *
570 * Return: The original value of @v.
571 */
247static __always_inline int572static __always_inline int
248atomic_fetch_inc_relaxed(atomic_t *v)573atomic_fetch_inc_relaxed(atomic_t *v)
249{574{
250 instrument_atomic_read_write(v, sizeof(*v));575 instrument_atomic_read_write(v, sizeof(*v));
251 return arch_atomic_fetch_inc_relaxed(v);576 return raw_atomic_fetch_inc_relaxed(v);
252}577}
253 578
579/**
580 * atomic_dec() - atomic decrement with relaxed ordering
581 * @v: pointer to atomic_t
582 *
583 * Atomically updates @v to (@v - 1) with relaxed ordering.
584 *
585 * Unsafe to use in noinstr code; use raw_atomic_dec() there.
586 *
587 * Return: Nothing.
588 */
254static __always_inline void589static __always_inline void
255atomic_dec(atomic_t *v)590atomic_dec(atomic_t *v)
256{591{
257 instrument_atomic_read_write(v, sizeof(*v));592 instrument_atomic_read_write(v, sizeof(*v));
258 arch_atomic_dec(v);593 raw_atomic_dec(v);
259}594}
260 595
596/**
597 * atomic_dec_return() - atomic decrement with full ordering
598 * @v: pointer to atomic_t
599 *
600 * Atomically updates @v to (@v - 1) with full ordering.
601 *
602 * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
603 *
604 * Return: The updated value of @v.
605 */
261static __always_inline int606static __always_inline int
262atomic_dec_return(atomic_t *v)607atomic_dec_return(atomic_t *v)
263{608{
264 kcsan_mb();609 kcsan_mb();
265 instrument_atomic_read_write(v, sizeof(*v));610 instrument_atomic_read_write(v, sizeof(*v));
266 return arch_atomic_dec_return(v);611 return raw_atomic_dec_return(v);
267}612}
268 613
614/**
615 * atomic_dec_return_acquire() - atomic decrement with acquire ordering
616 * @v: pointer to atomic_t
617 *
618 * Atomically updates @v to (@v - 1) with acquire ordering.
619 *
620 * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
621 *
622 * Return: The updated value of @v.
623 */
269static __always_inline int624static __always_inline int
270atomic_dec_return_acquire(atomic_t *v)625atomic_dec_return_acquire(atomic_t *v)
271{626{
272 instrument_atomic_read_write(v, sizeof(*v));627 instrument_atomic_read_write(v, sizeof(*v));
273 return arch_atomic_dec_return_acquire(v);628 return raw_atomic_dec_return_acquire(v);
274}629}
275 630
631/**
632 * atomic_dec_return_release() - atomic decrement with release ordering
633 * @v: pointer to atomic_t
634 *
635 * Atomically updates @v to (@v - 1) with release ordering.
636 *
637 * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
638 *
639 * Return: The updated value of @v.
640 */
276static __always_inline int641static __always_inline int
277atomic_dec_return_release(atomic_t *v)642atomic_dec_return_release(atomic_t *v)
278{643{
279 kcsan_release();644 kcsan_release();
280 instrument_atomic_read_write(v, sizeof(*v));645 instrument_atomic_read_write(v, sizeof(*v));
281 return arch_atomic_dec_return_release(v);646 return raw_atomic_dec_return_release(v);
282}647}
283 648
649/**
650 * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
651 * @v: pointer to atomic_t
652 *
653 * Atomically updates @v to (@v - 1) with relaxed ordering.
654 *
655 * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
656 *
657 * Return: The updated value of @v.
658 */
284static __always_inline int659static __always_inline int
285atomic_dec_return_relaxed(atomic_t *v)660atomic_dec_return_relaxed(atomic_t *v)
286{661{
287 instrument_atomic_read_write(v, sizeof(*v));662 instrument_atomic_read_write(v, sizeof(*v));
288 return arch_atomic_dec_return_relaxed(v);663 return raw_atomic_dec_return_relaxed(v);
289}664}
290 665
666/**
667 * atomic_fetch_dec() - atomic decrement with full ordering
668 * @v: pointer to atomic_t
669 *
670 * Atomically updates @v to (@v - 1) with full ordering.
671 *
672 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
673 *
674 * Return: The original value of @v.
675 */
291static __always_inline int676static __always_inline int
292atomic_fetch_dec(atomic_t *v)677atomic_fetch_dec(atomic_t *v)
293{678{
294 kcsan_mb();679 kcsan_mb();
295 instrument_atomic_read_write(v, sizeof(*v));680 instrument_atomic_read_write(v, sizeof(*v));
296 return arch_atomic_fetch_dec(v);681 return raw_atomic_fetch_dec(v);
297}682}
298 683
684/**
685 * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
686 * @v: pointer to atomic_t
687 *
688 * Atomically updates @v to (@v - 1) with acquire ordering.
689 *
690 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
691 *
692 * Return: The original value of @v.
693 */
299static __always_inline int694static __always_inline int
300atomic_fetch_dec_acquire(atomic_t *v)695atomic_fetch_dec_acquire(atomic_t *v)
301{696{
302 instrument_atomic_read_write(v, sizeof(*v));697 instrument_atomic_read_write(v, sizeof(*v));
303 return arch_atomic_fetch_dec_acquire(v);698 return raw_atomic_fetch_dec_acquire(v);
304}699}
305 700
701/**
702 * atomic_fetch_dec_release() - atomic decrement with release ordering
703 * @v: pointer to atomic_t
704 *
705 * Atomically updates @v to (@v - 1) with release ordering.
706 *
707 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
708 *
709 * Return: The original value of @v.
710 */
306static __always_inline int711static __always_inline int
307atomic_fetch_dec_release(atomic_t *v)712atomic_fetch_dec_release(atomic_t *v)
308{713{
309 kcsan_release();714 kcsan_release();
310 instrument_atomic_read_write(v, sizeof(*v));715 instrument_atomic_read_write(v, sizeof(*v));
311 return arch_atomic_fetch_dec_release(v);716 return raw_atomic_fetch_dec_release(v);
312}717}
313 718
719/**
720 * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
721 * @v: pointer to atomic_t
722 *
723 * Atomically updates @v to (@v - 1) with relaxed ordering.
724 *
725 * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
726 *
727 * Return: The original value of @v.
728 */
314static __always_inline int729static __always_inline int
315atomic_fetch_dec_relaxed(atomic_t *v)730atomic_fetch_dec_relaxed(atomic_t *v)
316{731{
317 instrument_atomic_read_write(v, sizeof(*v));732 instrument_atomic_read_write(v, sizeof(*v));
318 return arch_atomic_fetch_dec_relaxed(v);733 return raw_atomic_fetch_dec_relaxed(v);
319}734}
320 735
736/**
737 * atomic_and() - atomic bitwise AND with relaxed ordering
738 * @i: int value
739 * @v: pointer to atomic_t
740 *
741 * Atomically updates @v to (@v & @i) with relaxed ordering.
742 *
743 * Unsafe to use in noinstr code; use raw_atomic_and() there.
744 *
745 * Return: Nothing.
746 */
321static __always_inline void747static __always_inline void
322atomic_and(int i, atomic_t *v)748atomic_and(int i, atomic_t *v)
323{749{
324 instrument_atomic_read_write(v, sizeof(*v));750 instrument_atomic_read_write(v, sizeof(*v));
325 arch_atomic_and(i, v);751 raw_atomic_and(i, v);
326}752}
327 753
754/**
755 * atomic_fetch_and() - atomic bitwise AND with full ordering
756 * @i: int value
757 * @v: pointer to atomic_t
758 *
759 * Atomically updates @v to (@v & @i) with full ordering.
760 *
761 * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
762 *
763 * Return: The original value of @v.
764 */
328static __always_inline int765static __always_inline int
329atomic_fetch_and(int i, atomic_t *v)766atomic_fetch_and(int i, atomic_t *v)
330{767{
331 kcsan_mb();768 kcsan_mb();
332 instrument_atomic_read_write(v, sizeof(*v));769 instrument_atomic_read_write(v, sizeof(*v));
333 return arch_atomic_fetch_and(i, v);770 return raw_atomic_fetch_and(i, v);
334}771}
335 772
773/**
774 * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
775 * @i: int value
776 * @v: pointer to atomic_t
777 *
778 * Atomically updates @v to (@v & @i) with acquire ordering.
779 *
780 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
781 *
782 * Return: The original value of @v.
783 */
336static __always_inline int784static __always_inline int
337atomic_fetch_and_acquire(int i, atomic_t *v)785atomic_fetch_and_acquire(int i, atomic_t *v)
338{786{
339 instrument_atomic_read_write(v, sizeof(*v));787 instrument_atomic_read_write(v, sizeof(*v));
340 return arch_atomic_fetch_and_acquire(i, v);788 return raw_atomic_fetch_and_acquire(i, v);
341}789}
342 790
791/**
792 * atomic_fetch_and_release() - atomic bitwise AND with release ordering
793 * @i: int value
794 * @v: pointer to atomic_t
795 *
796 * Atomically updates @v to (@v & @i) with release ordering.
797 *
798 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
799 *
800 * Return: The original value of @v.
801 */
343static __always_inline int802static __always_inline int
344atomic_fetch_and_release(int i, atomic_t *v)803atomic_fetch_and_release(int i, atomic_t *v)
345{804{
346 kcsan_release();805 kcsan_release();
347 instrument_atomic_read_write(v, sizeof(*v));806 instrument_atomic_read_write(v, sizeof(*v));
348 return arch_atomic_fetch_and_release(i, v);807 return raw_atomic_fetch_and_release(i, v);
349}808}
350 809
810/**
811 * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
812 * @i: int value
813 * @v: pointer to atomic_t
814 *
815 * Atomically updates @v to (@v & @i) with relaxed ordering.
816 *
817 * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
818 *
819 * Return: The original value of @v.
820 */
351static __always_inline int821static __always_inline int
352atomic_fetch_and_relaxed(int i, atomic_t *v)822atomic_fetch_and_relaxed(int i, atomic_t *v)
353{823{
354 instrument_atomic_read_write(v, sizeof(*v));824 instrument_atomic_read_write(v, sizeof(*v));
355 return arch_atomic_fetch_and_relaxed(i, v);825 return raw_atomic_fetch_and_relaxed(i, v);
356}826}
357 827
828/**
829 * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
830 * @i: int value
831 * @v: pointer to atomic_t
832 *
833 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
834 *
835 * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
836 *
837 * Return: Nothing.
838 */
358static __always_inline void839static __always_inline void
359atomic_andnot(int i, atomic_t *v)840atomic_andnot(int i, atomic_t *v)
360{841{
361 instrument_atomic_read_write(v, sizeof(*v));842 instrument_atomic_read_write(v, sizeof(*v));
362 arch_atomic_andnot(i, v);843 raw_atomic_andnot(i, v);
363}844}
364 845
846/**
847 * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
848 * @i: int value
849 * @v: pointer to atomic_t
850 *
851 * Atomically updates @v to (@v & ~@i) with full ordering.
852 *
853 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
854 *
855 * Return: The original value of @v.
856 */
365static __always_inline int857static __always_inline int
366atomic_fetch_andnot(int i, atomic_t *v)858atomic_fetch_andnot(int i, atomic_t *v)
367{859{
368 kcsan_mb();860 kcsan_mb();
369 instrument_atomic_read_write(v, sizeof(*v));861 instrument_atomic_read_write(v, sizeof(*v));
370 return arch_atomic_fetch_andnot(i, v);862 return raw_atomic_fetch_andnot(i, v);
371}863}
372 864
865/**
866 * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
867 * @i: int value
868 * @v: pointer to atomic_t
869 *
870 * Atomically updates @v to (@v & ~@i) with acquire ordering.
871 *
872 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
873 *
874 * Return: The original value of @v.
875 */
373static __always_inline int876static __always_inline int
374atomic_fetch_andnot_acquire(int i, atomic_t *v)877atomic_fetch_andnot_acquire(int i, atomic_t *v)
375{878{
376 instrument_atomic_read_write(v, sizeof(*v));879 instrument_atomic_read_write(v, sizeof(*v));
377 return arch_atomic_fetch_andnot_acquire(i, v);880 return raw_atomic_fetch_andnot_acquire(i, v);
378}881}
379 882
883/**
884 * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
885 * @i: int value
886 * @v: pointer to atomic_t
887 *
888 * Atomically updates @v to (@v & ~@i) with release ordering.
889 *
890 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
891 *
892 * Return: The original value of @v.
893 */
380static __always_inline int894static __always_inline int
381atomic_fetch_andnot_release(int i, atomic_t *v)895atomic_fetch_andnot_release(int i, atomic_t *v)
382{896{
383 kcsan_release();897 kcsan_release();
384 instrument_atomic_read_write(v, sizeof(*v));898 instrument_atomic_read_write(v, sizeof(*v));
385 return arch_atomic_fetch_andnot_release(i, v);899 return raw_atomic_fetch_andnot_release(i, v);
386}900}
387 901
902/**
903 * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
904 * @i: int value
905 * @v: pointer to atomic_t
906 *
907 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
908 *
909 * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
910 *
911 * Return: The original value of @v.
912 */
388static __always_inline int913static __always_inline int
389atomic_fetch_andnot_relaxed(int i, atomic_t *v)914atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390{915{
391 instrument_atomic_read_write(v, sizeof(*v));916 instrument_atomic_read_write(v, sizeof(*v));
392 return arch_atomic_fetch_andnot_relaxed(i, v);917 return raw_atomic_fetch_andnot_relaxed(i, v);
393}918}
394 919
920/**
921 * atomic_or() - atomic bitwise OR with relaxed ordering
922 * @i: int value
923 * @v: pointer to atomic_t
924 *
925 * Atomically updates @v to (@v | @i) with relaxed ordering.
926 *
927 * Unsafe to use in noinstr code; use raw_atomic_or() there.
928 *
929 * Return: Nothing.
930 */
395static __always_inline void931static __always_inline void
396atomic_or(int i, atomic_t *v)932atomic_or(int i, atomic_t *v)
397{933{
398 instrument_atomic_read_write(v, sizeof(*v));934 instrument_atomic_read_write(v, sizeof(*v));
399 arch_atomic_or(i, v);935 raw_atomic_or(i, v);
400}936}
401 937
938/**
939 * atomic_fetch_or() - atomic bitwise OR with full ordering
940 * @i: int value
941 * @v: pointer to atomic_t
942 *
943 * Atomically updates @v to (@v | @i) with full ordering.
944 *
945 * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
946 *
947 * Return: The original value of @v.
948 */
402static __always_inline int949static __always_inline int
403atomic_fetch_or(int i, atomic_t *v)950atomic_fetch_or(int i, atomic_t *v)
404{951{
405 kcsan_mb();952 kcsan_mb();
406 instrument_atomic_read_write(v, sizeof(*v));953 instrument_atomic_read_write(v, sizeof(*v));
407 return arch_atomic_fetch_or(i, v);954 return raw_atomic_fetch_or(i, v);
408}955}
409 956
957/**
958 * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
959 * @i: int value
960 * @v: pointer to atomic_t
961 *
962 * Atomically updates @v to (@v | @i) with acquire ordering.
963 *
964 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
965 *
966 * Return: The original value of @v.
967 */
410static __always_inline int968static __always_inline int
411atomic_fetch_or_acquire(int i, atomic_t *v)969atomic_fetch_or_acquire(int i, atomic_t *v)
412{970{
413 instrument_atomic_read_write(v, sizeof(*v));971 instrument_atomic_read_write(v, sizeof(*v));
414 return arch_atomic_fetch_or_acquire(i, v);972 return raw_atomic_fetch_or_acquire(i, v);
415}973}
416 974
975/**
976 * atomic_fetch_or_release() - atomic bitwise OR with release ordering
977 * @i: int value
978 * @v: pointer to atomic_t
979 *
980 * Atomically updates @v to (@v | @i) with release ordering.
981 *
982 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
983 *
984 * Return: The original value of @v.
985 */
417static __always_inline int986static __always_inline int
418atomic_fetch_or_release(int i, atomic_t *v)987atomic_fetch_or_release(int i, atomic_t *v)
419{988{
420 kcsan_release();989 kcsan_release();
421 instrument_atomic_read_write(v, sizeof(*v));990 instrument_atomic_read_write(v, sizeof(*v));
422 return arch_atomic_fetch_or_release(i, v);991 return raw_atomic_fetch_or_release(i, v);
423}992}
424 993
994/**
995 * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
996 * @i: int value
997 * @v: pointer to atomic_t
998 *
999 * Atomically updates @v to (@v | @i) with relaxed ordering.
1000 *
1001 * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
1002 *
1003 * Return: The original value of @v.
1004 */
425static __always_inline int1005static __always_inline int
426atomic_fetch_or_relaxed(int i, atomic_t *v)1006atomic_fetch_or_relaxed(int i, atomic_t *v)
427{1007{
428 instrument_atomic_read_write(v, sizeof(*v));1008 instrument_atomic_read_write(v, sizeof(*v));
429 return arch_atomic_fetch_or_relaxed(i, v);1009 return raw_atomic_fetch_or_relaxed(i, v);
430}1010}
431 1011
1012/**
1013 * atomic_xor() - atomic bitwise XOR with relaxed ordering
1014 * @i: int value
1015 * @v: pointer to atomic_t
1016 *
1017 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1018 *
1019 * Unsafe to use in noinstr code; use raw_atomic_xor() there.
1020 *
1021 * Return: Nothing.
1022 */
432static __always_inline void1023static __always_inline void
433atomic_xor(int i, atomic_t *v)1024atomic_xor(int i, atomic_t *v)
434{1025{
435 instrument_atomic_read_write(v, sizeof(*v));1026 instrument_atomic_read_write(v, sizeof(*v));
436 arch_atomic_xor(i, v);1027 raw_atomic_xor(i, v);
437}1028}
438 1029
1030/**
1031 * atomic_fetch_xor() - atomic bitwise XOR with full ordering
1032 * @i: int value
1033 * @v: pointer to atomic_t
1034 *
1035 * Atomically updates @v to (@v ^ @i) with full ordering.
1036 *
1037 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
1038 *
1039 * Return: The original value of @v.
1040 */
439static __always_inline int1041static __always_inline int
440atomic_fetch_xor(int i, atomic_t *v)1042atomic_fetch_xor(int i, atomic_t *v)
441{1043{
442 kcsan_mb();1044 kcsan_mb();
443 instrument_atomic_read_write(v, sizeof(*v));1045 instrument_atomic_read_write(v, sizeof(*v));
444 return arch_atomic_fetch_xor(i, v);1046 return raw_atomic_fetch_xor(i, v);
445}1047}
446 1048
1049/**
1050 * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1051 * @i: int value
1052 * @v: pointer to atomic_t
1053 *
1054 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1055 *
1056 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
1057 *
1058 * Return: The original value of @v.
1059 */
447static __always_inline int1060static __always_inline int
448atomic_fetch_xor_acquire(int i, atomic_t *v)1061atomic_fetch_xor_acquire(int i, atomic_t *v)
449{1062{
450 instrument_atomic_read_write(v, sizeof(*v));1063 instrument_atomic_read_write(v, sizeof(*v));
451 return arch_atomic_fetch_xor_acquire(i, v);1064 return raw_atomic_fetch_xor_acquire(i, v);
452}1065}
453 1066
1067/**
1068 * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1069 * @i: int value
1070 * @v: pointer to atomic_t
1071 *
1072 * Atomically updates @v to (@v ^ @i) with release ordering.
1073 *
1074 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
1075 *
1076 * Return: The original value of @v.
1077 */
454static __always_inline int1078static __always_inline int
455atomic_fetch_xor_release(int i, atomic_t *v)1079atomic_fetch_xor_release(int i, atomic_t *v)
456{1080{
457 kcsan_release();1081 kcsan_release();
458 instrument_atomic_read_write(v, sizeof(*v));1082 instrument_atomic_read_write(v, sizeof(*v));
459 return arch_atomic_fetch_xor_release(i, v);1083 return raw_atomic_fetch_xor_release(i, v);
460}1084}
461 1085
1086/**
1087 * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1088 * @i: int value
1089 * @v: pointer to atomic_t
1090 *
1091 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1092 *
1093 * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
1094 *
1095 * Return: The original value of @v.
1096 */
462static __always_inline int1097static __always_inline int
463atomic_fetch_xor_relaxed(int i, atomic_t *v)1098atomic_fetch_xor_relaxed(int i, atomic_t *v)
464{1099{
465 instrument_atomic_read_write(v, sizeof(*v));1100 instrument_atomic_read_write(v, sizeof(*v));
466 return arch_atomic_fetch_xor_relaxed(i, v);1101 return raw_atomic_fetch_xor_relaxed(i, v);
467}1102}
468 1103
1104/**
1105 * atomic_xchg() - atomic exchange with full ordering
1106 * @v: pointer to atomic_t
1107 * @new: int value to assign
1108 *
1109 * Atomically updates @v to @new with full ordering.
1110 *
1111 * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
1112 *
1113 * Return: The original value of @v.
1114 */
469static __always_inline int1115static __always_inline int
470atomic_xchg(atomic_t *v, int i)1116atomic_xchg(atomic_t *v, int new)
471{1117{
472 kcsan_mb();1118 kcsan_mb();
473 instrument_atomic_read_write(v, sizeof(*v));1119 instrument_atomic_read_write(v, sizeof(*v));
474 return arch_atomic_xchg(v, i);1120 return raw_atomic_xchg(v, new);
475}1121}
476 1122
1123/**
1124 * atomic_xchg_acquire() - atomic exchange with acquire ordering
1125 * @v: pointer to atomic_t
1126 * @new: int value to assign
1127 *
1128 * Atomically updates @v to @new with acquire ordering.
1129 *
1130 * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
1131 *
1132 * Return: The original value of @v.
1133 */
477static __always_inline int1134static __always_inline int
478atomic_xchg_acquire(atomic_t *v, int i)1135atomic_xchg_acquire(atomic_t *v, int new)
479{1136{
480 instrument_atomic_read_write(v, sizeof(*v));1137 instrument_atomic_read_write(v, sizeof(*v));
481 return arch_atomic_xchg_acquire(v, i);1138 return raw_atomic_xchg_acquire(v, new);
482}1139}
483 1140
1141/**
1142 * atomic_xchg_release() - atomic exchange with release ordering
1143 * @v: pointer to atomic_t
1144 * @new: int value to assign
1145 *
1146 * Atomically updates @v to @new with release ordering.
1147 *
1148 * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
1149 *
1150 * Return: The original value of @v.
1151 */
484static __always_inline int1152static __always_inline int
485atomic_xchg_release(atomic_t *v, int i)1153atomic_xchg_release(atomic_t *v, int new)
486{1154{
487 kcsan_release();1155 kcsan_release();
488 instrument_atomic_read_write(v, sizeof(*v));1156 instrument_atomic_read_write(v, sizeof(*v));
489 return arch_atomic_xchg_release(v, i);1157 return raw_atomic_xchg_release(v, new);
490}1158}
491 1159
1160/**
1161 * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1162 * @v: pointer to atomic_t
1163 * @new: int value to assign
1164 *
1165 * Atomically updates @v to @new with relaxed ordering.
1166 *
1167 * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
1168 *
1169 * Return: The original value of @v.
1170 */
492static __always_inline int1171static __always_inline int
493atomic_xchg_relaxed(atomic_t *v, int i)1172atomic_xchg_relaxed(atomic_t *v, int new)
494{1173{
495 instrument_atomic_read_write(v, sizeof(*v));1174 instrument_atomic_read_write(v, sizeof(*v));
496 return arch_atomic_xchg_relaxed(v, i);1175 return raw_atomic_xchg_relaxed(v, new);
497}1176}
498 1177
1178/**
1179 * atomic_cmpxchg() - atomic compare and exchange with full ordering
1180 * @v: pointer to atomic_t
1181 * @old: int value to compare with
1182 * @new: int value to assign
1183 *
1184 * If (@v == @old), atomically updates @v to @new with full ordering.
1185 *
1186 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
1187 *
1188 * Return: The original value of @v.
1189 */
499static __always_inline int1190static __always_inline int
500atomic_cmpxchg(atomic_t *v, int old, int new)1191atomic_cmpxchg(atomic_t *v, int old, int new)
501{1192{
502 kcsan_mb();1193 kcsan_mb();
503 instrument_atomic_read_write(v, sizeof(*v));1194 instrument_atomic_read_write(v, sizeof(*v));
504 return arch_atomic_cmpxchg(v, old, new);1195 return raw_atomic_cmpxchg(v, old, new);
505}1196}
506 1197
1198/**
1199 * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1200 * @v: pointer to atomic_t
1201 * @old: int value to compare with
1202 * @new: int value to assign
1203 *
1204 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1205 *
1206 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
1207 *
1208 * Return: The original value of @v.
1209 */
507static __always_inline int1210static __always_inline int
508atomic_cmpxchg_acquire(atomic_t *v, int old, int new)1211atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509{1212{
510 instrument_atomic_read_write(v, sizeof(*v));1213 instrument_atomic_read_write(v, sizeof(*v));
511 return arch_atomic_cmpxchg_acquire(v, old, new);1214 return raw_atomic_cmpxchg_acquire(v, old, new);
512}1215}
513 1216
1217/**
1218 * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
1219 * @v: pointer to atomic_t
1220 * @old: int value to compare with
1221 * @new: int value to assign
1222 *
1223 * If (@v == @old), atomically updates @v to @new with release ordering.
1224 *
1225 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
1226 *
1227 * Return: The original value of @v.
1228 */
514static __always_inline int1229static __always_inline int
515atomic_cmpxchg_release(atomic_t *v, int old, int new)1230atomic_cmpxchg_release(atomic_t *v, int old, int new)
516{1231{
517 kcsan_release();1232 kcsan_release();
518 instrument_atomic_read_write(v, sizeof(*v));1233 instrument_atomic_read_write(v, sizeof(*v));
519 return arch_atomic_cmpxchg_release(v, old, new);1234 return raw_atomic_cmpxchg_release(v, old, new);
520}1235}
521 1236
1237/**
1238 * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1239 * @v: pointer to atomic_t
1240 * @old: int value to compare with
1241 * @new: int value to assign
1242 *
1243 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1244 *
1245 * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
1246 *
1247 * Return: The original value of @v.
1248 */
522static __always_inline int1249static __always_inline int
523atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)1250atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524{1251{
525 instrument_atomic_read_write(v, sizeof(*v));1252 instrument_atomic_read_write(v, sizeof(*v));
526 return arch_atomic_cmpxchg_relaxed(v, old, new);1253 return raw_atomic_cmpxchg_relaxed(v, old, new);
527}1254}
528 1255
1256/**
1257 * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
1258 * @v: pointer to atomic_t
1259 * @old: pointer to int value to compare with
1260 * @new: int value to assign
1261 *
1262 * If (@v == @old), atomically updates @v to @new with full ordering.
1263 * Otherwise, updates @old to the current value of @v.
1264 *
1265 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
1266 *
1267 * Return: @true if the exchange occured, @false otherwise.
1268 */
529static __always_inline bool1269static __always_inline bool
530atomic_try_cmpxchg(atomic_t *v, int *old, int new)1270atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531{1271{
532 kcsan_mb();1272 kcsan_mb();
533 instrument_atomic_read_write(v, sizeof(*v));1273 instrument_atomic_read_write(v, sizeof(*v));
534 instrument_atomic_read_write(old, sizeof(*old));1274 instrument_atomic_read_write(old, sizeof(*old));
535 return arch_atomic_try_cmpxchg(v, old, new);1275 return raw_atomic_try_cmpxchg(v, old, new);
536}1276}
537 1277
1278/**
1279 * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1280 * @v: pointer to atomic_t
1281 * @old: pointer to int value to compare with
1282 * @new: int value to assign
1283 *
1284 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1285 * Otherwise, updates @old to the current value of @v.
1286 *
1287 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
1288 *
1289 * Return: @true if the exchange occured, @false otherwise.
1290 */
538static __always_inline bool1291static __always_inline bool
539atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)1292atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540{1293{
541 instrument_atomic_read_write(v, sizeof(*v));1294 instrument_atomic_read_write(v, sizeof(*v));
542 instrument_atomic_read_write(old, sizeof(*old));1295 instrument_atomic_read_write(old, sizeof(*old));
543 return arch_atomic_try_cmpxchg_acquire(v, old, new);1296 return raw_atomic_try_cmpxchg_acquire(v, old, new);
544}1297}
545 1298
1299/**
1300 * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
1301 * @v: pointer to atomic_t
1302 * @old: pointer to int value to compare with
1303 * @new: int value to assign
1304 *
1305 * If (@v == @old), atomically updates @v to @new with release ordering.
1306 * Otherwise, updates @old to the current value of @v.
1307 *
1308 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
1309 *
1310 * Return: @true if the exchange occured, @false otherwise.
1311 */
546static __always_inline bool1312static __always_inline bool
547atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)1313atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548{1314{
549 kcsan_release();1315 kcsan_release();
550 instrument_atomic_read_write(v, sizeof(*v));1316 instrument_atomic_read_write(v, sizeof(*v));
551 instrument_atomic_read_write(old, sizeof(*old));1317 instrument_atomic_read_write(old, sizeof(*old));
552 return arch_atomic_try_cmpxchg_release(v, old, new);1318 return raw_atomic_try_cmpxchg_release(v, old, new);
553}1319}
554 1320
1321/**
1322 * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1323 * @v: pointer to atomic_t
1324 * @old: pointer to int value to compare with
1325 * @new: int value to assign
1326 *
1327 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1328 * Otherwise, updates @old to the current value of @v.
1329 *
1330 * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
1331 *
1332 * Return: @true if the exchange occured, @false otherwise.
1333 */
555static __always_inline bool1334static __always_inline bool
556atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)1335atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557{1336{
558 instrument_atomic_read_write(v, sizeof(*v));1337 instrument_atomic_read_write(v, sizeof(*v));
559 instrument_atomic_read_write(old, sizeof(*old));1338 instrument_atomic_read_write(old, sizeof(*old));
560 return arch_atomic_try_cmpxchg_relaxed(v, old, new);1339 return raw_atomic_try_cmpxchg_relaxed(v, old, new);
561}1340}
562 1341
1342/**
1343 * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
1344 * @i: int value to add
1345 * @v: pointer to atomic_t
1346 *
1347 * Atomically updates @v to (@v - @i) with full ordering.
1348 *
1349 * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
1350 *
1351 * Return: @true if the resulting value of @v is zero, @false otherwise.
1352 */
563static __always_inline bool1353static __always_inline bool
564atomic_sub_and_test(int i, atomic_t *v)1354atomic_sub_and_test(int i, atomic_t *v)
565{1355{
566 kcsan_mb();1356 kcsan_mb();
567 instrument_atomic_read_write(v, sizeof(*v));1357 instrument_atomic_read_write(v, sizeof(*v));
568 return arch_atomic_sub_and_test(i, v);1358 return raw_atomic_sub_and_test(i, v);
569}1359}
570 1360
1361/**
1362 * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
1363 * @v: pointer to atomic_t
1364 *
1365 * Atomically updates @v to (@v - 1) with full ordering.
1366 *
1367 * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
1368 *
1369 * Return: @true if the resulting value of @v is zero, @false otherwise.
1370 */
571static __always_inline bool1371static __always_inline bool
572atomic_dec_and_test(atomic_t *v)1372atomic_dec_and_test(atomic_t *v)
573{1373{
574 kcsan_mb();1374 kcsan_mb();
575 instrument_atomic_read_write(v, sizeof(*v));1375 instrument_atomic_read_write(v, sizeof(*v));
576 return arch_atomic_dec_and_test(v);1376 return raw_atomic_dec_and_test(v);
577}1377}
578 1378
1379/**
1380 * atomic_inc_and_test() - atomic increment and test if zero with full ordering
1381 * @v: pointer to atomic_t
1382 *
1383 * Atomically updates @v to (@v + 1) with full ordering.
1384 *
1385 * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
1386 *
1387 * Return: @true if the resulting value of @v is zero, @false otherwise.
1388 */
579static __always_inline bool1389static __always_inline bool
580atomic_inc_and_test(atomic_t *v)1390atomic_inc_and_test(atomic_t *v)
581{1391{
582 kcsan_mb();1392 kcsan_mb();
583 instrument_atomic_read_write(v, sizeof(*v));1393 instrument_atomic_read_write(v, sizeof(*v));
584 return arch_atomic_inc_and_test(v);1394 return raw_atomic_inc_and_test(v);
585}1395}
586 1396
1397/**
1398 * atomic_add_negative() - atomic add and test if negative with full ordering
1399 * @i: int value to add
1400 * @v: pointer to atomic_t
1401 *
1402 * Atomically updates @v to (@v + @i) with full ordering.
1403 *
1404 * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
1405 *
1406 * Return: @true if the resulting value of @v is negative, @false otherwise.
1407 */
587static __always_inline bool1408static __always_inline bool
588atomic_add_negative(int i, atomic_t *v)1409atomic_add_negative(int i, atomic_t *v)
589{1410{
590 kcsan_mb();1411 kcsan_mb();
591 instrument_atomic_read_write(v, sizeof(*v));1412 instrument_atomic_read_write(v, sizeof(*v));
592 return arch_atomic_add_negative(i, v);1413 return raw_atomic_add_negative(i, v);
593}1414}
594 1415
1416/**
1417 * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
1418 * @i: int value to add
1419 * @v: pointer to atomic_t
1420 *
1421 * Atomically updates @v to (@v + @i) with acquire ordering.
1422 *
1423 * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
1424 *
1425 * Return: @true if the resulting value of @v is negative, @false otherwise.
1426 */
595static __always_inline bool1427static __always_inline bool
596atomic_add_negative_acquire(int i, atomic_t *v)1428atomic_add_negative_acquire(int i, atomic_t *v)
597{1429{
598 instrument_atomic_read_write(v, sizeof(*v));1430 instrument_atomic_read_write(v, sizeof(*v));
599 return arch_atomic_add_negative_acquire(i, v);1431 return raw_atomic_add_negative_acquire(i, v);
600}1432}
601 1433
1434/**
1435 * atomic_add_negative_release() - atomic add and test if negative with release ordering
1436 * @i: int value to add
1437 * @v: pointer to atomic_t
1438 *
1439 * Atomically updates @v to (@v + @i) with release ordering.
1440 *
1441 * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
1442 *
1443 * Return: @true if the resulting value of @v is negative, @false otherwise.
1444 */
602static __always_inline bool1445static __always_inline bool
603atomic_add_negative_release(int i, atomic_t *v)1446atomic_add_negative_release(int i, atomic_t *v)
604{1447{
605 kcsan_release();1448 kcsan_release();
606 instrument_atomic_read_write(v, sizeof(*v));1449 instrument_atomic_read_write(v, sizeof(*v));
607 return arch_atomic_add_negative_release(i, v);1450 return raw_atomic_add_negative_release(i, v);
608}1451}
609 1452
1453/**
1454 * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
1455 * @i: int value to add
1456 * @v: pointer to atomic_t
1457 *
1458 * Atomically updates @v to (@v + @i) with relaxed ordering.
1459 *
1460 * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
1461 *
1462 * Return: @true if the resulting value of @v is negative, @false otherwise.
1463 */
610static __always_inline bool1464static __always_inline bool
611atomic_add_negative_relaxed(int i, atomic_t *v)1465atomic_add_negative_relaxed(int i, atomic_t *v)
612{1466{
613 instrument_atomic_read_write(v, sizeof(*v));1467 instrument_atomic_read_write(v, sizeof(*v));
614 return arch_atomic_add_negative_relaxed(i, v);1468 return raw_atomic_add_negative_relaxed(i, v);
615}1469}
616 1470
1471/**
1472 * atomic_fetch_add_unless() - atomic add unless value with full ordering
1473 * @v: pointer to atomic_t
1474 * @a: int value to add
1475 * @u: int value to compare with
1476 *
1477 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1478 *
1479 * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
1480 *
1481 * Return: The original value of @v.
1482 */
617static __always_inline int1483static __always_inline int
618atomic_fetch_add_unless(atomic_t *v, int a, int u)1484atomic_fetch_add_unless(atomic_t *v, int a, int u)
619{1485{
620 kcsan_mb();1486 kcsan_mb();
621 instrument_atomic_read_write(v, sizeof(*v));1487 instrument_atomic_read_write(v, sizeof(*v));
622 return arch_atomic_fetch_add_unless(v, a, u);1488 return raw_atomic_fetch_add_unless(v, a, u);
623}1489}
624 1490
1491/**
1492 * atomic_add_unless() - atomic add unless value with full ordering
1493 * @v: pointer to atomic_t
1494 * @a: int value to add
1495 * @u: int value to compare with
1496 *
1497 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1498 *
1499 * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
1500 *
1501 * Return: @true if @v was updated, @false otherwise.
1502 */
625static __always_inline bool1503static __always_inline bool
626atomic_add_unless(atomic_t *v, int a, int u)1504atomic_add_unless(atomic_t *v, int a, int u)
627{1505{
628 kcsan_mb();1506 kcsan_mb();
629 instrument_atomic_read_write(v, sizeof(*v));1507 instrument_atomic_read_write(v, sizeof(*v));
630 return arch_atomic_add_unless(v, a, u);1508 return raw_atomic_add_unless(v, a, u);
631}1509}
632 1510
1511/**
1512 * atomic_inc_not_zero() - atomic increment unless zero with full ordering
1513 * @v: pointer to atomic_t
1514 *
1515 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
1516 *
1517 * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
1518 *
1519 * Return: @true if @v was updated, @false otherwise.
1520 */
633static __always_inline bool1521static __always_inline bool
634atomic_inc_not_zero(atomic_t *v)1522atomic_inc_not_zero(atomic_t *v)
635{1523{
636 kcsan_mb();1524 kcsan_mb();
637 instrument_atomic_read_write(v, sizeof(*v));1525 instrument_atomic_read_write(v, sizeof(*v));
638 return arch_atomic_inc_not_zero(v);1526 return raw_atomic_inc_not_zero(v);
639}1527}
640 1528
1529/**
1530 * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
1531 * @v: pointer to atomic_t
1532 *
1533 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
1534 *
1535 * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
1536 *
1537 * Return: @true if @v was updated, @false otherwise.
1538 */
641static __always_inline bool1539static __always_inline bool
642atomic_inc_unless_negative(atomic_t *v)1540atomic_inc_unless_negative(atomic_t *v)
643{1541{
644 kcsan_mb();1542 kcsan_mb();
645 instrument_atomic_read_write(v, sizeof(*v));1543 instrument_atomic_read_write(v, sizeof(*v));
646 return arch_atomic_inc_unless_negative(v);1544 return raw_atomic_inc_unless_negative(v);
647}1545}
648 1546
1547/**
1548 * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
1549 * @v: pointer to atomic_t
1550 *
1551 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
1552 *
1553 * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
1554 *
1555 * Return: @true if @v was updated, @false otherwise.
1556 */
649static __always_inline bool1557static __always_inline bool
650atomic_dec_unless_positive(atomic_t *v)1558atomic_dec_unless_positive(atomic_t *v)
651{1559{
652 kcsan_mb();1560 kcsan_mb();
653 instrument_atomic_read_write(v, sizeof(*v));1561 instrument_atomic_read_write(v, sizeof(*v));
654 return arch_atomic_dec_unless_positive(v);1562 return raw_atomic_dec_unless_positive(v);
655}1563}
656 1564
1565/**
1566 * atomic_dec_if_positive() - atomic decrement if positive with full ordering
1567 * @v: pointer to atomic_t
1568 *
1569 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
1570 *
1571 * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
1572 *
1573 * Return: The old value of (@v - 1), regardless of whether @v was updated.
1574 */
657static __always_inline int1575static __always_inline int
658atomic_dec_if_positive(atomic_t *v)1576atomic_dec_if_positive(atomic_t *v)
659{1577{
660 kcsan_mb();1578 kcsan_mb();
661 instrument_atomic_read_write(v, sizeof(*v));1579 instrument_atomic_read_write(v, sizeof(*v));
662 return arch_atomic_dec_if_positive(v);1580 return raw_atomic_dec_if_positive(v);
663}1581}
664 1582
1583/**
1584 * atomic64_read() - atomic load with relaxed ordering
1585 * @v: pointer to atomic64_t
1586 *
1587 * Atomically loads the value of @v with relaxed ordering.
1588 *
1589 * Unsafe to use in noinstr code; use raw_atomic64_read() there.
1590 *
1591 * Return: The value loaded from @v.
1592 */
665static __always_inline s641593static __always_inline s64
666atomic64_read(const atomic64_t *v)1594atomic64_read(const atomic64_t *v)
667{1595{
668 instrument_atomic_read(v, sizeof(*v));1596 instrument_atomic_read(v, sizeof(*v));
669 return arch_atomic64_read(v);1597 return raw_atomic64_read(v);
670}1598}
671 1599
1600/**
1601 * atomic64_read_acquire() - atomic load with acquire ordering
1602 * @v: pointer to atomic64_t
1603 *
1604 * Atomically loads the value of @v with acquire ordering.
1605 *
1606 * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
1607 *
1608 * Return: The value loaded from @v.
1609 */
672static __always_inline s641610static __always_inline s64
673atomic64_read_acquire(const atomic64_t *v)1611atomic64_read_acquire(const atomic64_t *v)
674{1612{
675 instrument_atomic_read(v, sizeof(*v));1613 instrument_atomic_read(v, sizeof(*v));
676 return arch_atomic64_read_acquire(v);1614 return raw_atomic64_read_acquire(v);
677}1615}
678 1616
1617/**
1618 * atomic64_set() - atomic set with relaxed ordering
1619 * @v: pointer to atomic64_t
1620 * @i: s64 value to assign
1621 *
1622 * Atomically sets @v to @i with relaxed ordering.
1623 *
1624 * Unsafe to use in noinstr code; use raw_atomic64_set() there.
1625 *
1626 * Return: Nothing.
1627 */
679static __always_inline void1628static __always_inline void
680atomic64_set(atomic64_t *v, s64 i)1629atomic64_set(atomic64_t *v, s64 i)
681{1630{
682 instrument_atomic_write(v, sizeof(*v));1631 instrument_atomic_write(v, sizeof(*v));
683 arch_atomic64_set(v, i);1632 raw_atomic64_set(v, i);
684}1633}
685 1634
1635/**
1636 * atomic64_set_release() - atomic set with release ordering
1637 * @v: pointer to atomic64_t
1638 * @i: s64 value to assign
1639 *
1640 * Atomically sets @v to @i with release ordering.
1641 *
1642 * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
1643 *
1644 * Return: Nothing.
1645 */
686static __always_inline void1646static __always_inline void
687atomic64_set_release(atomic64_t *v, s64 i)1647atomic64_set_release(atomic64_t *v, s64 i)
688{1648{
689 kcsan_release();1649 kcsan_release();
690 instrument_atomic_write(v, sizeof(*v));1650 instrument_atomic_write(v, sizeof(*v));
691 arch_atomic64_set_release(v, i);1651 raw_atomic64_set_release(v, i);
692}1652}
693 1653
1654/**
1655 * atomic64_add() - atomic add with relaxed ordering
1656 * @i: s64 value to add
1657 * @v: pointer to atomic64_t
1658 *
1659 * Atomically updates @v to (@v + @i) with relaxed ordering.
1660 *
1661 * Unsafe to use in noinstr code; use raw_atomic64_add() there.
1662 *
1663 * Return: Nothing.
1664 */
694static __always_inline void1665static __always_inline void
695atomic64_add(s64 i, atomic64_t *v)1666atomic64_add(s64 i, atomic64_t *v)
696{1667{
697 instrument_atomic_read_write(v, sizeof(*v));1668 instrument_atomic_read_write(v, sizeof(*v));
698 arch_atomic64_add(i, v);1669 raw_atomic64_add(i, v);
699}1670}
700 1671
1672/**
1673 * atomic64_add_return() - atomic add with full ordering
1674 * @i: s64 value to add
1675 * @v: pointer to atomic64_t
1676 *
1677 * Atomically updates @v to (@v + @i) with full ordering.
1678 *
1679 * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
1680 *
1681 * Return: The updated value of @v.
1682 */
701static __always_inline s641683static __always_inline s64
702atomic64_add_return(s64 i, atomic64_t *v)1684atomic64_add_return(s64 i, atomic64_t *v)
703{1685{
704 kcsan_mb();1686 kcsan_mb();
705 instrument_atomic_read_write(v, sizeof(*v));1687 instrument_atomic_read_write(v, sizeof(*v));
706 return arch_atomic64_add_return(i, v);1688 return raw_atomic64_add_return(i, v);
707}1689}
708 1690
1691/**
1692 * atomic64_add_return_acquire() - atomic add with acquire ordering
1693 * @i: s64 value to add
1694 * @v: pointer to atomic64_t
1695 *
1696 * Atomically updates @v to (@v + @i) with acquire ordering.
1697 *
1698 * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
1699 *
1700 * Return: The updated value of @v.
1701 */
709static __always_inline s641702static __always_inline s64
710atomic64_add_return_acquire(s64 i, atomic64_t *v)1703atomic64_add_return_acquire(s64 i, atomic64_t *v)
711{1704{
712 instrument_atomic_read_write(v, sizeof(*v));1705 instrument_atomic_read_write(v, sizeof(*v));
713 return arch_atomic64_add_return_acquire(i, v);1706 return raw_atomic64_add_return_acquire(i, v);
714}1707}
715 1708
1709/**
1710 * atomic64_add_return_release() - atomic add with release ordering
1711 * @i: s64 value to add
1712 * @v: pointer to atomic64_t
1713 *
1714 * Atomically updates @v to (@v + @i) with release ordering.
1715 *
1716 * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
1717 *
1718 * Return: The updated value of @v.
1719 */
716static __always_inline s641720static __always_inline s64
717atomic64_add_return_release(s64 i, atomic64_t *v)1721atomic64_add_return_release(s64 i, atomic64_t *v)
718{1722{
719 kcsan_release();1723 kcsan_release();
720 instrument_atomic_read_write(v, sizeof(*v));1724 instrument_atomic_read_write(v, sizeof(*v));
721 return arch_atomic64_add_return_release(i, v);1725 return raw_atomic64_add_return_release(i, v);
722}1726}
723 1727
1728/**
1729 * atomic64_add_return_relaxed() - atomic add with relaxed ordering
1730 * @i: s64 value to add
1731 * @v: pointer to atomic64_t
1732 *
1733 * Atomically updates @v to (@v + @i) with relaxed ordering.
1734 *
1735 * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
1736 *
1737 * Return: The updated value of @v.
1738 */
724static __always_inline s641739static __always_inline s64
725atomic64_add_return_relaxed(s64 i, atomic64_t *v)1740atomic64_add_return_relaxed(s64 i, atomic64_t *v)
726{1741{
727 instrument_atomic_read_write(v, sizeof(*v));1742 instrument_atomic_read_write(v, sizeof(*v));
728 return arch_atomic64_add_return_relaxed(i, v);1743 return raw_atomic64_add_return_relaxed(i, v);
729}1744}
730 1745
1746/**
1747 * atomic64_fetch_add() - atomic add with full ordering
1748 * @i: s64 value to add
1749 * @v: pointer to atomic64_t
1750 *
1751 * Atomically updates @v to (@v + @i) with full ordering.
1752 *
1753 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
1754 *
1755 * Return: The original value of @v.
1756 */
731static __always_inline s641757static __always_inline s64
732atomic64_fetch_add(s64 i, atomic64_t *v)1758atomic64_fetch_add(s64 i, atomic64_t *v)
733{1759{
734 kcsan_mb();1760 kcsan_mb();
735 instrument_atomic_read_write(v, sizeof(*v));1761 instrument_atomic_read_write(v, sizeof(*v));
736 return arch_atomic64_fetch_add(i, v);1762 return raw_atomic64_fetch_add(i, v);
737}1763}
738 1764
1765/**
1766 * atomic64_fetch_add_acquire() - atomic add with acquire ordering
1767 * @i: s64 value to add
1768 * @v: pointer to atomic64_t
1769 *
1770 * Atomically updates @v to (@v + @i) with acquire ordering.
1771 *
1772 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
1773 *
1774 * Return: The original value of @v.
1775 */
739static __always_inline s641776static __always_inline s64
740atomic64_fetch_add_acquire(s64 i, atomic64_t *v)1777atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
741{1778{
742 instrument_atomic_read_write(v, sizeof(*v));1779 instrument_atomic_read_write(v, sizeof(*v));
743 return arch_atomic64_fetch_add_acquire(i, v);1780 return raw_atomic64_fetch_add_acquire(i, v);
744}1781}
745 1782
1783/**
1784 * atomic64_fetch_add_release() - atomic add with release ordering
1785 * @i: s64 value to add
1786 * @v: pointer to atomic64_t
1787 *
1788 * Atomically updates @v to (@v + @i) with release ordering.
1789 *
1790 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
1791 *
1792 * Return: The original value of @v.
1793 */
746static __always_inline s641794static __always_inline s64
747atomic64_fetch_add_release(s64 i, atomic64_t *v)1795atomic64_fetch_add_release(s64 i, atomic64_t *v)
748{1796{
749 kcsan_release();1797 kcsan_release();
750 instrument_atomic_read_write(v, sizeof(*v));1798 instrument_atomic_read_write(v, sizeof(*v));
751 return arch_atomic64_fetch_add_release(i, v);1799 return raw_atomic64_fetch_add_release(i, v);
752}1800}
753 1801
1802/**
1803 * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
1804 * @i: s64 value to add
1805 * @v: pointer to atomic64_t
1806 *
1807 * Atomically updates @v to (@v + @i) with relaxed ordering.
1808 *
1809 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
1810 *
1811 * Return: The original value of @v.
1812 */
754static __always_inline s641813static __always_inline s64
755atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)1814atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
756{1815{
757 instrument_atomic_read_write(v, sizeof(*v));1816 instrument_atomic_read_write(v, sizeof(*v));
758 return arch_atomic64_fetch_add_relaxed(i, v);1817 return raw_atomic64_fetch_add_relaxed(i, v);
759}1818}
760 1819
1820/**
1821 * atomic64_sub() - atomic subtract with relaxed ordering
1822 * @i: s64 value to subtract
1823 * @v: pointer to atomic64_t
1824 *
1825 * Atomically updates @v to (@v - @i) with relaxed ordering.
1826 *
1827 * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
1828 *
1829 * Return: Nothing.
1830 */
761static __always_inline void1831static __always_inline void
762atomic64_sub(s64 i, atomic64_t *v)1832atomic64_sub(s64 i, atomic64_t *v)
763{1833{
764 instrument_atomic_read_write(v, sizeof(*v));1834 instrument_atomic_read_write(v, sizeof(*v));
765 arch_atomic64_sub(i, v);1835 raw_atomic64_sub(i, v);
766}1836}
767 1837
1838/**
1839 * atomic64_sub_return() - atomic subtract with full ordering
1840 * @i: s64 value to subtract
1841 * @v: pointer to atomic64_t
1842 *
1843 * Atomically updates @v to (@v - @i) with full ordering.
1844 *
1845 * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
1846 *
1847 * Return: The updated value of @v.
1848 */
768static __always_inline s641849static __always_inline s64
769atomic64_sub_return(s64 i, atomic64_t *v)1850atomic64_sub_return(s64 i, atomic64_t *v)
770{1851{
771 kcsan_mb();1852 kcsan_mb();
772 instrument_atomic_read_write(v, sizeof(*v));1853 instrument_atomic_read_write(v, sizeof(*v));
773 return arch_atomic64_sub_return(i, v);1854 return raw_atomic64_sub_return(i, v);
774}1855}
775 1856
1857/**
1858 * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
1859 * @i: s64 value to subtract
1860 * @v: pointer to atomic64_t
1861 *
1862 * Atomically updates @v to (@v - @i) with acquire ordering.
1863 *
1864 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
1865 *
1866 * Return: The updated value of @v.
1867 */
776static __always_inline s641868static __always_inline s64
777atomic64_sub_return_acquire(s64 i, atomic64_t *v)1869atomic64_sub_return_acquire(s64 i, atomic64_t *v)
778{1870{
779 instrument_atomic_read_write(v, sizeof(*v));1871 instrument_atomic_read_write(v, sizeof(*v));
780 return arch_atomic64_sub_return_acquire(i, v);1872 return raw_atomic64_sub_return_acquire(i, v);
781}1873}
782 1874
1875/**
1876 * atomic64_sub_return_release() - atomic subtract with release ordering
1877 * @i: s64 value to subtract
1878 * @v: pointer to atomic64_t
1879 *
1880 * Atomically updates @v to (@v - @i) with release ordering.
1881 *
1882 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
1883 *
1884 * Return: The updated value of @v.
1885 */
783static __always_inline s641886static __always_inline s64
784atomic64_sub_return_release(s64 i, atomic64_t *v)1887atomic64_sub_return_release(s64 i, atomic64_t *v)
785{1888{
786 kcsan_release();1889 kcsan_release();
787 instrument_atomic_read_write(v, sizeof(*v));1890 instrument_atomic_read_write(v, sizeof(*v));
788 return arch_atomic64_sub_return_release(i, v);1891 return raw_atomic64_sub_return_release(i, v);
789}1892}
790 1893
1894/**
1895 * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
1896 * @i: s64 value to subtract
1897 * @v: pointer to atomic64_t
1898 *
1899 * Atomically updates @v to (@v - @i) with relaxed ordering.
1900 *
1901 * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
1902 *
1903 * Return: The updated value of @v.
1904 */
791static __always_inline s641905static __always_inline s64
792atomic64_sub_return_relaxed(s64 i, atomic64_t *v)1906atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
793{1907{
794 instrument_atomic_read_write(v, sizeof(*v));1908 instrument_atomic_read_write(v, sizeof(*v));
795 return arch_atomic64_sub_return_relaxed(i, v);1909 return raw_atomic64_sub_return_relaxed(i, v);
796}1910}
797 1911
1912/**
1913 * atomic64_fetch_sub() - atomic subtract with full ordering
1914 * @i: s64 value to subtract
1915 * @v: pointer to atomic64_t
1916 *
1917 * Atomically updates @v to (@v - @i) with full ordering.
1918 *
1919 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
1920 *
1921 * Return: The original value of @v.
1922 */
798static __always_inline s641923static __always_inline s64
799atomic64_fetch_sub(s64 i, atomic64_t *v)1924atomic64_fetch_sub(s64 i, atomic64_t *v)
800{1925{
801 kcsan_mb();1926 kcsan_mb();
802 instrument_atomic_read_write(v, sizeof(*v));1927 instrument_atomic_read_write(v, sizeof(*v));
803 return arch_atomic64_fetch_sub(i, v);1928 return raw_atomic64_fetch_sub(i, v);
804}1929}
805 1930
1931/**
1932 * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
1933 * @i: s64 value to subtract
1934 * @v: pointer to atomic64_t
1935 *
1936 * Atomically updates @v to (@v - @i) with acquire ordering.
1937 *
1938 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
1939 *
1940 * Return: The original value of @v.
1941 */
806static __always_inline s641942static __always_inline s64
807atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)1943atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
808{1944{
809 instrument_atomic_read_write(v, sizeof(*v));1945 instrument_atomic_read_write(v, sizeof(*v));
810 return arch_atomic64_fetch_sub_acquire(i, v);1946 return raw_atomic64_fetch_sub_acquire(i, v);
811}1947}
812 1948
1949/**
1950 * atomic64_fetch_sub_release() - atomic subtract with release ordering
1951 * @i: s64 value to subtract
1952 * @v: pointer to atomic64_t
1953 *
1954 * Atomically updates @v to (@v - @i) with release ordering.
1955 *
1956 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
1957 *
1958 * Return: The original value of @v.
1959 */
813static __always_inline s641960static __always_inline s64
814atomic64_fetch_sub_release(s64 i, atomic64_t *v)1961atomic64_fetch_sub_release(s64 i, atomic64_t *v)
815{1962{
816 kcsan_release();1963 kcsan_release();
817 instrument_atomic_read_write(v, sizeof(*v));1964 instrument_atomic_read_write(v, sizeof(*v));
818 return arch_atomic64_fetch_sub_release(i, v);1965 return raw_atomic64_fetch_sub_release(i, v);
819}1966}
820 1967
1968/**
1969 * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
1970 * @i: s64 value to subtract
1971 * @v: pointer to atomic64_t
1972 *
1973 * Atomically updates @v to (@v - @i) with relaxed ordering.
1974 *
1975 * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
1976 *
1977 * Return: The original value of @v.
1978 */
821static __always_inline s641979static __always_inline s64
822atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)1980atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
823{1981{
824 instrument_atomic_read_write(v, sizeof(*v));1982 instrument_atomic_read_write(v, sizeof(*v));
825 return arch_atomic64_fetch_sub_relaxed(i, v);1983 return raw_atomic64_fetch_sub_relaxed(i, v);
826}1984}
827 1985
1986/**
1987 * atomic64_inc() - atomic increment with relaxed ordering
1988 * @v: pointer to atomic64_t
1989 *
1990 * Atomically updates @v to (@v + 1) with relaxed ordering.
1991 *
1992 * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
1993 *
1994 * Return: Nothing.
1995 */
828static __always_inline void1996static __always_inline void
829atomic64_inc(atomic64_t *v)1997atomic64_inc(atomic64_t *v)
830{1998{
831 instrument_atomic_read_write(v, sizeof(*v));1999 instrument_atomic_read_write(v, sizeof(*v));
832 arch_atomic64_inc(v);2000 raw_atomic64_inc(v);
833}2001}
834 2002
2003/**
2004 * atomic64_inc_return() - atomic increment with full ordering
2005 * @v: pointer to atomic64_t
2006 *
2007 * Atomically updates @v to (@v + 1) with full ordering.
2008 *
2009 * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
2010 *
2011 * Return: The updated value of @v.
2012 */
835static __always_inline s642013static __always_inline s64
836atomic64_inc_return(atomic64_t *v)2014atomic64_inc_return(atomic64_t *v)
837{2015{
838 kcsan_mb();2016 kcsan_mb();
839 instrument_atomic_read_write(v, sizeof(*v));2017 instrument_atomic_read_write(v, sizeof(*v));
840 return arch_atomic64_inc_return(v);2018 return raw_atomic64_inc_return(v);
841}2019}
842 2020
2021/**
2022 * atomic64_inc_return_acquire() - atomic increment with acquire ordering
2023 * @v: pointer to atomic64_t
2024 *
2025 * Atomically updates @v to (@v + 1) with acquire ordering.
2026 *
2027 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
2028 *
2029 * Return: The updated value of @v.
2030 */
843static __always_inline s642031static __always_inline s64
844atomic64_inc_return_acquire(atomic64_t *v)2032atomic64_inc_return_acquire(atomic64_t *v)
845{2033{
846 instrument_atomic_read_write(v, sizeof(*v));2034 instrument_atomic_read_write(v, sizeof(*v));
847 return arch_atomic64_inc_return_acquire(v);2035 return raw_atomic64_inc_return_acquire(v);
848}2036}
849 2037
2038/**
2039 * atomic64_inc_return_release() - atomic increment with release ordering
2040 * @v: pointer to atomic64_t
2041 *
2042 * Atomically updates @v to (@v + 1) with release ordering.
2043 *
2044 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
2045 *
2046 * Return: The updated value of @v.
2047 */
850static __always_inline s642048static __always_inline s64
851atomic64_inc_return_release(atomic64_t *v)2049atomic64_inc_return_release(atomic64_t *v)
852{2050{
853 kcsan_release();2051 kcsan_release();
854 instrument_atomic_read_write(v, sizeof(*v));2052 instrument_atomic_read_write(v, sizeof(*v));
855 return arch_atomic64_inc_return_release(v);2053 return raw_atomic64_inc_return_release(v);
856}2054}
857 2055
2056/**
2057 * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
2058 * @v: pointer to atomic64_t
2059 *
2060 * Atomically updates @v to (@v + 1) with relaxed ordering.
2061 *
2062 * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
2063 *
2064 * Return: The updated value of @v.
2065 */
858static __always_inline s642066static __always_inline s64
859atomic64_inc_return_relaxed(atomic64_t *v)2067atomic64_inc_return_relaxed(atomic64_t *v)
860{2068{
861 instrument_atomic_read_write(v, sizeof(*v));2069 instrument_atomic_read_write(v, sizeof(*v));
862 return arch_atomic64_inc_return_relaxed(v);2070 return raw_atomic64_inc_return_relaxed(v);
863}2071}
864 2072
2073/**
2074 * atomic64_fetch_inc() - atomic increment with full ordering
2075 * @v: pointer to atomic64_t
2076 *
2077 * Atomically updates @v to (@v + 1) with full ordering.
2078 *
2079 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
2080 *
2081 * Return: The original value of @v.
2082 */
865static __always_inline s642083static __always_inline s64
866atomic64_fetch_inc(atomic64_t *v)2084atomic64_fetch_inc(atomic64_t *v)
867{2085{
868 kcsan_mb();2086 kcsan_mb();
869 instrument_atomic_read_write(v, sizeof(*v));2087 instrument_atomic_read_write(v, sizeof(*v));
870 return arch_atomic64_fetch_inc(v);2088 return raw_atomic64_fetch_inc(v);
871}2089}
872 2090
2091/**
2092 * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
2093 * @v: pointer to atomic64_t
2094 *
2095 * Atomically updates @v to (@v + 1) with acquire ordering.
2096 *
2097 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
2098 *
2099 * Return: The original value of @v.
2100 */
873static __always_inline s642101static __always_inline s64
874atomic64_fetch_inc_acquire(atomic64_t *v)2102atomic64_fetch_inc_acquire(atomic64_t *v)
875{2103{
876 instrument_atomic_read_write(v, sizeof(*v));2104 instrument_atomic_read_write(v, sizeof(*v));
877 return arch_atomic64_fetch_inc_acquire(v);2105 return raw_atomic64_fetch_inc_acquire(v);
878}2106}
879 2107
2108/**
2109 * atomic64_fetch_inc_release() - atomic increment with release ordering
2110 * @v: pointer to atomic64_t
2111 *
2112 * Atomically updates @v to (@v + 1) with release ordering.
2113 *
2114 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
2115 *
2116 * Return: The original value of @v.
2117 */
880static __always_inline s642118static __always_inline s64
881atomic64_fetch_inc_release(atomic64_t *v)2119atomic64_fetch_inc_release(atomic64_t *v)
882{2120{
883 kcsan_release();2121 kcsan_release();
884 instrument_atomic_read_write(v, sizeof(*v));2122 instrument_atomic_read_write(v, sizeof(*v));
885 return arch_atomic64_fetch_inc_release(v);2123 return raw_atomic64_fetch_inc_release(v);
886}2124}
887 2125
2126/**
2127 * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
2128 * @v: pointer to atomic64_t
2129 *
2130 * Atomically updates @v to (@v + 1) with relaxed ordering.
2131 *
2132 * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
2133 *
2134 * Return: The original value of @v.
2135 */
888static __always_inline s642136static __always_inline s64
889atomic64_fetch_inc_relaxed(atomic64_t *v)2137atomic64_fetch_inc_relaxed(atomic64_t *v)
890{2138{
891 instrument_atomic_read_write(v, sizeof(*v));2139 instrument_atomic_read_write(v, sizeof(*v));
892 return arch_atomic64_fetch_inc_relaxed(v);2140 return raw_atomic64_fetch_inc_relaxed(v);
893}2141}
894 2142
2143/**
2144 * atomic64_dec() - atomic decrement with relaxed ordering
2145 * @v: pointer to atomic64_t
2146 *
2147 * Atomically updates @v to (@v - 1) with relaxed ordering.
2148 *
2149 * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
2150 *
2151 * Return: Nothing.
2152 */
895static __always_inline void2153static __always_inline void
896atomic64_dec(atomic64_t *v)2154atomic64_dec(atomic64_t *v)
897{2155{
898 instrument_atomic_read_write(v, sizeof(*v));2156 instrument_atomic_read_write(v, sizeof(*v));
899 arch_atomic64_dec(v);2157 raw_atomic64_dec(v);
900}2158}
901 2159
2160/**
2161 * atomic64_dec_return() - atomic decrement with full ordering
2162 * @v: pointer to atomic64_t
2163 *
2164 * Atomically updates @v to (@v - 1) with full ordering.
2165 *
2166 * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
2167 *
2168 * Return: The updated value of @v.
2169 */
902static __always_inline s642170static __always_inline s64
903atomic64_dec_return(atomic64_t *v)2171atomic64_dec_return(atomic64_t *v)
904{2172{
905 kcsan_mb();2173 kcsan_mb();
906 instrument_atomic_read_write(v, sizeof(*v));2174 instrument_atomic_read_write(v, sizeof(*v));
907 return arch_atomic64_dec_return(v);2175 return raw_atomic64_dec_return(v);
908}2176}
909 2177
2178/**
2179 * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
2180 * @v: pointer to atomic64_t
2181 *
2182 * Atomically updates @v to (@v - 1) with acquire ordering.
2183 *
2184 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
2185 *
2186 * Return: The updated value of @v.
2187 */
910static __always_inline s642188static __always_inline s64
911atomic64_dec_return_acquire(atomic64_t *v)2189atomic64_dec_return_acquire(atomic64_t *v)
912{2190{
913 instrument_atomic_read_write(v, sizeof(*v));2191 instrument_atomic_read_write(v, sizeof(*v));
914 return arch_atomic64_dec_return_acquire(v);2192 return raw_atomic64_dec_return_acquire(v);
915}2193}
916 2194
2195/**
2196 * atomic64_dec_return_release() - atomic decrement with release ordering
2197 * @v: pointer to atomic64_t
2198 *
2199 * Atomically updates @v to (@v - 1) with release ordering.
2200 *
2201 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
2202 *
2203 * Return: The updated value of @v.
2204 */
917static __always_inline s642205static __always_inline s64
918atomic64_dec_return_release(atomic64_t *v)2206atomic64_dec_return_release(atomic64_t *v)
919{2207{
920 kcsan_release();2208 kcsan_release();
921 instrument_atomic_read_write(v, sizeof(*v));2209 instrument_atomic_read_write(v, sizeof(*v));
922 return arch_atomic64_dec_return_release(v);2210 return raw_atomic64_dec_return_release(v);
923}2211}
924 2212
2213/**
2214 * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
2215 * @v: pointer to atomic64_t
2216 *
2217 * Atomically updates @v to (@v - 1) with relaxed ordering.
2218 *
2219 * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
2220 *
2221 * Return: The updated value of @v.
2222 */
925static __always_inline s642223static __always_inline s64
926atomic64_dec_return_relaxed(atomic64_t *v)2224atomic64_dec_return_relaxed(atomic64_t *v)
927{2225{
928 instrument_atomic_read_write(v, sizeof(*v));2226 instrument_atomic_read_write(v, sizeof(*v));
929 return arch_atomic64_dec_return_relaxed(v);2227 return raw_atomic64_dec_return_relaxed(v);
930}2228}
931 2229
2230/**
2231 * atomic64_fetch_dec() - atomic decrement with full ordering
2232 * @v: pointer to atomic64_t
2233 *
2234 * Atomically updates @v to (@v - 1) with full ordering.
2235 *
2236 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
2237 *
2238 * Return: The original value of @v.
2239 */
932static __always_inline s642240static __always_inline s64
933atomic64_fetch_dec(atomic64_t *v)2241atomic64_fetch_dec(atomic64_t *v)
934{2242{
935 kcsan_mb();2243 kcsan_mb();
936 instrument_atomic_read_write(v, sizeof(*v));2244 instrument_atomic_read_write(v, sizeof(*v));
937 return arch_atomic64_fetch_dec(v);2245 return raw_atomic64_fetch_dec(v);
938}2246}
939 2247
2248/**
2249 * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
2250 * @v: pointer to atomic64_t
2251 *
2252 * Atomically updates @v to (@v - 1) with acquire ordering.
2253 *
2254 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
2255 *
2256 * Return: The original value of @v.
2257 */
940static __always_inline s642258static __always_inline s64
941atomic64_fetch_dec_acquire(atomic64_t *v)2259atomic64_fetch_dec_acquire(atomic64_t *v)
942{2260{
943 instrument_atomic_read_write(v, sizeof(*v));2261 instrument_atomic_read_write(v, sizeof(*v));
944 return arch_atomic64_fetch_dec_acquire(v);2262 return raw_atomic64_fetch_dec_acquire(v);
945}2263}
946 2264
2265/**
2266 * atomic64_fetch_dec_release() - atomic decrement with release ordering
2267 * @v: pointer to atomic64_t
2268 *
2269 * Atomically updates @v to (@v - 1) with release ordering.
2270 *
2271 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
2272 *
2273 * Return: The original value of @v.
2274 */
947static __always_inline s642275static __always_inline s64
948atomic64_fetch_dec_release(atomic64_t *v)2276atomic64_fetch_dec_release(atomic64_t *v)
949{2277{
950 kcsan_release();2278 kcsan_release();
951 instrument_atomic_read_write(v, sizeof(*v));2279 instrument_atomic_read_write(v, sizeof(*v));
952 return arch_atomic64_fetch_dec_release(v);2280 return raw_atomic64_fetch_dec_release(v);
953}2281}
954 2282
2283/**
2284 * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
2285 * @v: pointer to atomic64_t
2286 *
2287 * Atomically updates @v to (@v - 1) with relaxed ordering.
2288 *
2289 * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
2290 *
2291 * Return: The original value of @v.
2292 */
955static __always_inline s642293static __always_inline s64
956atomic64_fetch_dec_relaxed(atomic64_t *v)2294atomic64_fetch_dec_relaxed(atomic64_t *v)
957{2295{
958 instrument_atomic_read_write(v, sizeof(*v));2296 instrument_atomic_read_write(v, sizeof(*v));
959 return arch_atomic64_fetch_dec_relaxed(v);2297 return raw_atomic64_fetch_dec_relaxed(v);
960}2298}
961 2299
2300/**
2301 * atomic64_and() - atomic bitwise AND with relaxed ordering
2302 * @i: s64 value
2303 * @v: pointer to atomic64_t
2304 *
2305 * Atomically updates @v to (@v & @i) with relaxed ordering.
2306 *
2307 * Unsafe to use in noinstr code; use raw_atomic64_and() there.
2308 *
2309 * Return: Nothing.
2310 */
962static __always_inline void2311static __always_inline void
963atomic64_and(s64 i, atomic64_t *v)2312atomic64_and(s64 i, atomic64_t *v)
964{2313{
965 instrument_atomic_read_write(v, sizeof(*v));2314 instrument_atomic_read_write(v, sizeof(*v));
966 arch_atomic64_and(i, v);2315 raw_atomic64_and(i, v);
967}2316}
968 2317
2318/**
2319 * atomic64_fetch_and() - atomic bitwise AND with full ordering
2320 * @i: s64 value
2321 * @v: pointer to atomic64_t
2322 *
2323 * Atomically updates @v to (@v & @i) with full ordering.
2324 *
2325 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
2326 *
2327 * Return: The original value of @v.
2328 */
969static __always_inline s642329static __always_inline s64
970atomic64_fetch_and(s64 i, atomic64_t *v)2330atomic64_fetch_and(s64 i, atomic64_t *v)
971{2331{
972 kcsan_mb();2332 kcsan_mb();
973 instrument_atomic_read_write(v, sizeof(*v));2333 instrument_atomic_read_write(v, sizeof(*v));
974 return arch_atomic64_fetch_and(i, v);2334 return raw_atomic64_fetch_and(i, v);
975}2335}
976 2336
2337/**
2338 * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
2339 * @i: s64 value
2340 * @v: pointer to atomic64_t
2341 *
2342 * Atomically updates @v to (@v & @i) with acquire ordering.
2343 *
2344 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
2345 *
2346 * Return: The original value of @v.
2347 */
977static __always_inline s642348static __always_inline s64
978atomic64_fetch_and_acquire(s64 i, atomic64_t *v)2349atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
979{2350{
980 instrument_atomic_read_write(v, sizeof(*v));2351 instrument_atomic_read_write(v, sizeof(*v));
981 return arch_atomic64_fetch_and_acquire(i, v);2352 return raw_atomic64_fetch_and_acquire(i, v);
982}2353}
983 2354
2355/**
2356 * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
2357 * @i: s64 value
2358 * @v: pointer to atomic64_t
2359 *
2360 * Atomically updates @v to (@v & @i) with release ordering.
2361 *
2362 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
2363 *
2364 * Return: The original value of @v.
2365 */
984static __always_inline s642366static __always_inline s64
985atomic64_fetch_and_release(s64 i, atomic64_t *v)2367atomic64_fetch_and_release(s64 i, atomic64_t *v)
986{2368{
987 kcsan_release();2369 kcsan_release();
988 instrument_atomic_read_write(v, sizeof(*v));2370 instrument_atomic_read_write(v, sizeof(*v));
989 return arch_atomic64_fetch_and_release(i, v);2371 return raw_atomic64_fetch_and_release(i, v);
990}2372}
991 2373
2374/**
2375 * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
2376 * @i: s64 value
2377 * @v: pointer to atomic64_t
2378 *
2379 * Atomically updates @v to (@v & @i) with relaxed ordering.
2380 *
2381 * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
2382 *
2383 * Return: The original value of @v.
2384 */
992static __always_inline s642385static __always_inline s64
993atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)2386atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
994{2387{
995 instrument_atomic_read_write(v, sizeof(*v));2388 instrument_atomic_read_write(v, sizeof(*v));
996 return arch_atomic64_fetch_and_relaxed(i, v);2389 return raw_atomic64_fetch_and_relaxed(i, v);
997}2390}
998 2391
2392/**
2393 * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
2394 * @i: s64 value
2395 * @v: pointer to atomic64_t
2396 *
2397 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2398 *
2399 * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
2400 *
2401 * Return: Nothing.
2402 */
999static __always_inline void2403static __always_inline void
1000atomic64_andnot(s64 i, atomic64_t *v)2404atomic64_andnot(s64 i, atomic64_t *v)
1001{2405{
1002 instrument_atomic_read_write(v, sizeof(*v));2406 instrument_atomic_read_write(v, sizeof(*v));
1003 arch_atomic64_andnot(i, v);2407 raw_atomic64_andnot(i, v);
1004}2408}
1005 2409
2410/**
2411 * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
2412 * @i: s64 value
2413 * @v: pointer to atomic64_t
2414 *
2415 * Atomically updates @v to (@v & ~@i) with full ordering.
2416 *
2417 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
2418 *
2419 * Return: The original value of @v.
2420 */
1006static __always_inline s642421static __always_inline s64
1007atomic64_fetch_andnot(s64 i, atomic64_t *v)2422atomic64_fetch_andnot(s64 i, atomic64_t *v)
1008{2423{
1009 kcsan_mb();2424 kcsan_mb();
1010 instrument_atomic_read_write(v, sizeof(*v));2425 instrument_atomic_read_write(v, sizeof(*v));
1011 return arch_atomic64_fetch_andnot(i, v);2426 return raw_atomic64_fetch_andnot(i, v);
1012}2427}
1013 2428
2429/**
2430 * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
2431 * @i: s64 value
2432 * @v: pointer to atomic64_t
2433 *
2434 * Atomically updates @v to (@v & ~@i) with acquire ordering.
2435 *
2436 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
2437 *
2438 * Return: The original value of @v.
2439 */
1014static __always_inline s642440static __always_inline s64
1015atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)2441atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1016{2442{
1017 instrument_atomic_read_write(v, sizeof(*v));2443 instrument_atomic_read_write(v, sizeof(*v));
1018 return arch_atomic64_fetch_andnot_acquire(i, v);2444 return raw_atomic64_fetch_andnot_acquire(i, v);
1019}2445}
1020 2446
2447/**
2448 * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
2449 * @i: s64 value
2450 * @v: pointer to atomic64_t
2451 *
2452 * Atomically updates @v to (@v & ~@i) with release ordering.
2453 *
2454 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
2455 *
2456 * Return: The original value of @v.
2457 */
1021static __always_inline s642458static __always_inline s64
1022atomic64_fetch_andnot_release(s64 i, atomic64_t *v)2459atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1023{2460{
1024 kcsan_release();2461 kcsan_release();
1025 instrument_atomic_read_write(v, sizeof(*v));2462 instrument_atomic_read_write(v, sizeof(*v));
1026 return arch_atomic64_fetch_andnot_release(i, v);2463 return raw_atomic64_fetch_andnot_release(i, v);
1027}2464}
1028 2465
2466/**
2467 * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
2468 * @i: s64 value
2469 * @v: pointer to atomic64_t
2470 *
2471 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
2472 *
2473 * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
2474 *
2475 * Return: The original value of @v.
2476 */
1029static __always_inline s642477static __always_inline s64
1030atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)2478atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1031{2479{
1032 instrument_atomic_read_write(v, sizeof(*v));2480 instrument_atomic_read_write(v, sizeof(*v));
1033 return arch_atomic64_fetch_andnot_relaxed(i, v);2481 return raw_atomic64_fetch_andnot_relaxed(i, v);
1034}2482}
1035 2483
2484/**
2485 * atomic64_or() - atomic bitwise OR with relaxed ordering
2486 * @i: s64 value
2487 * @v: pointer to atomic64_t
2488 *
2489 * Atomically updates @v to (@v | @i) with relaxed ordering.
2490 *
2491 * Unsafe to use in noinstr code; use raw_atomic64_or() there.
2492 *
2493 * Return: Nothing.
2494 */
1036static __always_inline void2495static __always_inline void
1037atomic64_or(s64 i, atomic64_t *v)2496atomic64_or(s64 i, atomic64_t *v)
1038{2497{
1039 instrument_atomic_read_write(v, sizeof(*v));2498 instrument_atomic_read_write(v, sizeof(*v));
1040 arch_atomic64_or(i, v);2499 raw_atomic64_or(i, v);
1041}2500}
1042 2501
2502/**
2503 * atomic64_fetch_or() - atomic bitwise OR with full ordering
2504 * @i: s64 value
2505 * @v: pointer to atomic64_t
2506 *
2507 * Atomically updates @v to (@v | @i) with full ordering.
2508 *
2509 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
2510 *
2511 * Return: The original value of @v.
2512 */
1043static __always_inline s642513static __always_inline s64
1044atomic64_fetch_or(s64 i, atomic64_t *v)2514atomic64_fetch_or(s64 i, atomic64_t *v)
1045{2515{
1046 kcsan_mb();2516 kcsan_mb();
1047 instrument_atomic_read_write(v, sizeof(*v));2517 instrument_atomic_read_write(v, sizeof(*v));
1048 return arch_atomic64_fetch_or(i, v);2518 return raw_atomic64_fetch_or(i, v);
1049}2519}
1050 2520
2521/**
2522 * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
2523 * @i: s64 value
2524 * @v: pointer to atomic64_t
2525 *
2526 * Atomically updates @v to (@v | @i) with acquire ordering.
2527 *
2528 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
2529 *
2530 * Return: The original value of @v.
2531 */
1051static __always_inline s642532static __always_inline s64
1052atomic64_fetch_or_acquire(s64 i, atomic64_t *v)2533atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1053{2534{
1054 instrument_atomic_read_write(v, sizeof(*v));2535 instrument_atomic_read_write(v, sizeof(*v));
1055 return arch_atomic64_fetch_or_acquire(i, v);2536 return raw_atomic64_fetch_or_acquire(i, v);
1056}2537}
1057 2538
2539/**
2540 * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
2541 * @i: s64 value
2542 * @v: pointer to atomic64_t
2543 *
2544 * Atomically updates @v to (@v | @i) with release ordering.
2545 *
2546 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
2547 *
2548 * Return: The original value of @v.
2549 */
1058static __always_inline s642550static __always_inline s64
1059atomic64_fetch_or_release(s64 i, atomic64_t *v)2551atomic64_fetch_or_release(s64 i, atomic64_t *v)
1060{2552{
1061 kcsan_release();2553 kcsan_release();
1062 instrument_atomic_read_write(v, sizeof(*v));2554 instrument_atomic_read_write(v, sizeof(*v));
1063 return arch_atomic64_fetch_or_release(i, v);2555 return raw_atomic64_fetch_or_release(i, v);
1064}2556}
1065 2557
2558/**
2559 * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
2560 * @i: s64 value
2561 * @v: pointer to atomic64_t
2562 *
2563 * Atomically updates @v to (@v | @i) with relaxed ordering.
2564 *
2565 * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
2566 *
2567 * Return: The original value of @v.
2568 */
1066static __always_inline s642569static __always_inline s64
1067atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)2570atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1068{2571{
1069 instrument_atomic_read_write(v, sizeof(*v));2572 instrument_atomic_read_write(v, sizeof(*v));
1070 return arch_atomic64_fetch_or_relaxed(i, v);2573 return raw_atomic64_fetch_or_relaxed(i, v);
1071}2574}
1072 2575
2576/**
2577 * atomic64_xor() - atomic bitwise XOR with relaxed ordering
2578 * @i: s64 value
2579 * @v: pointer to atomic64_t
2580 *
2581 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2582 *
2583 * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
2584 *
2585 * Return: Nothing.
2586 */
1073static __always_inline void2587static __always_inline void
1074atomic64_xor(s64 i, atomic64_t *v)2588atomic64_xor(s64 i, atomic64_t *v)
1075{2589{
1076 instrument_atomic_read_write(v, sizeof(*v));2590 instrument_atomic_read_write(v, sizeof(*v));
1077 arch_atomic64_xor(i, v);2591 raw_atomic64_xor(i, v);
1078}2592}
1079 2593
2594/**
2595 * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
2596 * @i: s64 value
2597 * @v: pointer to atomic64_t
2598 *
2599 * Atomically updates @v to (@v ^ @i) with full ordering.
2600 *
2601 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
2602 *
2603 * Return: The original value of @v.
2604 */
1080static __always_inline s642605static __always_inline s64
1081atomic64_fetch_xor(s64 i, atomic64_t *v)2606atomic64_fetch_xor(s64 i, atomic64_t *v)
1082{2607{
1083 kcsan_mb();2608 kcsan_mb();
1084 instrument_atomic_read_write(v, sizeof(*v));2609 instrument_atomic_read_write(v, sizeof(*v));
1085 return arch_atomic64_fetch_xor(i, v);2610 return raw_atomic64_fetch_xor(i, v);
1086}2611}
1087 2612
2613/**
2614 * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
2615 * @i: s64 value
2616 * @v: pointer to atomic64_t
2617 *
2618 * Atomically updates @v to (@v ^ @i) with acquire ordering.
2619 *
2620 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
2621 *
2622 * Return: The original value of @v.
2623 */
1088static __always_inline s642624static __always_inline s64
1089atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)2625atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1090{2626{
1091 instrument_atomic_read_write(v, sizeof(*v));2627 instrument_atomic_read_write(v, sizeof(*v));
1092 return arch_atomic64_fetch_xor_acquire(i, v);2628 return raw_atomic64_fetch_xor_acquire(i, v);
1093}2629}
1094 2630
2631/**
2632 * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
2633 * @i: s64 value
2634 * @v: pointer to atomic64_t
2635 *
2636 * Atomically updates @v to (@v ^ @i) with release ordering.
2637 *
2638 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
2639 *
2640 * Return: The original value of @v.
2641 */
1095static __always_inline s642642static __always_inline s64
1096atomic64_fetch_xor_release(s64 i, atomic64_t *v)2643atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1097{2644{
1098 kcsan_release();2645 kcsan_release();
1099 instrument_atomic_read_write(v, sizeof(*v));2646 instrument_atomic_read_write(v, sizeof(*v));
1100 return arch_atomic64_fetch_xor_release(i, v);2647 return raw_atomic64_fetch_xor_release(i, v);
1101}2648}
1102 2649
2650/**
2651 * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
2652 * @i: s64 value
2653 * @v: pointer to atomic64_t
2654 *
2655 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
2656 *
2657 * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
2658 *
2659 * Return: The original value of @v.
2660 */
1103static __always_inline s642661static __always_inline s64
1104atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)2662atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1105{2663{
1106 instrument_atomic_read_write(v, sizeof(*v));2664 instrument_atomic_read_write(v, sizeof(*v));
1107 return arch_atomic64_fetch_xor_relaxed(i, v);2665 return raw_atomic64_fetch_xor_relaxed(i, v);
1108}2666}
1109 2667
2668/**
2669 * atomic64_xchg() - atomic exchange with full ordering
2670 * @v: pointer to atomic64_t
2671 * @new: s64 value to assign
2672 *
2673 * Atomically updates @v to @new with full ordering.
2674 *
2675 * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
2676 *
2677 * Return: The original value of @v.
2678 */
1110static __always_inline s642679static __always_inline s64
1111atomic64_xchg(atomic64_t *v, s64 i)2680atomic64_xchg(atomic64_t *v, s64 new)
1112{2681{
1113 kcsan_mb();2682 kcsan_mb();
1114 instrument_atomic_read_write(v, sizeof(*v));2683 instrument_atomic_read_write(v, sizeof(*v));
1115 return arch_atomic64_xchg(v, i);2684 return raw_atomic64_xchg(v, new);
1116}2685}
1117 2686
2687/**
2688 * atomic64_xchg_acquire() - atomic exchange with acquire ordering
2689 * @v: pointer to atomic64_t
2690 * @new: s64 value to assign
2691 *
2692 * Atomically updates @v to @new with acquire ordering.
2693 *
2694 * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
2695 *
2696 * Return: The original value of @v.
2697 */
1118static __always_inline s642698static __always_inline s64
1119atomic64_xchg_acquire(atomic64_t *v, s64 i)2699atomic64_xchg_acquire(atomic64_t *v, s64 new)
1120{2700{
1121 instrument_atomic_read_write(v, sizeof(*v));2701 instrument_atomic_read_write(v, sizeof(*v));
1122 return arch_atomic64_xchg_acquire(v, i);2702 return raw_atomic64_xchg_acquire(v, new);
1123}2703}
1124 2704
2705/**
2706 * atomic64_xchg_release() - atomic exchange with release ordering
2707 * @v: pointer to atomic64_t
2708 * @new: s64 value to assign
2709 *
2710 * Atomically updates @v to @new with release ordering.
2711 *
2712 * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
2713 *
2714 * Return: The original value of @v.
2715 */
1125static __always_inline s642716static __always_inline s64
1126atomic64_xchg_release(atomic64_t *v, s64 i)2717atomic64_xchg_release(atomic64_t *v, s64 new)
1127{2718{
1128 kcsan_release();2719 kcsan_release();
1129 instrument_atomic_read_write(v, sizeof(*v));2720 instrument_atomic_read_write(v, sizeof(*v));
1130 return arch_atomic64_xchg_release(v, i);2721 return raw_atomic64_xchg_release(v, new);
1131}2722}
1132 2723
2724/**
2725 * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
2726 * @v: pointer to atomic64_t
2727 * @new: s64 value to assign
2728 *
2729 * Atomically updates @v to @new with relaxed ordering.
2730 *
2731 * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
2732 *
2733 * Return: The original value of @v.
2734 */
1133static __always_inline s642735static __always_inline s64
1134atomic64_xchg_relaxed(atomic64_t *v, s64 i)2736atomic64_xchg_relaxed(atomic64_t *v, s64 new)
1135{2737{
1136 instrument_atomic_read_write(v, sizeof(*v));2738 instrument_atomic_read_write(v, sizeof(*v));
1137 return arch_atomic64_xchg_relaxed(v, i);2739 return raw_atomic64_xchg_relaxed(v, new);
1138}2740}
1139 2741
2742/**
2743 * atomic64_cmpxchg() - atomic compare and exchange with full ordering
2744 * @v: pointer to atomic64_t
2745 * @old: s64 value to compare with
2746 * @new: s64 value to assign
2747 *
2748 * If (@v == @old), atomically updates @v to @new with full ordering.
2749 *
2750 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
2751 *
2752 * Return: The original value of @v.
2753 */
1140static __always_inline s642754static __always_inline s64
1141atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)2755atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1142{2756{
1143 kcsan_mb();2757 kcsan_mb();
1144 instrument_atomic_read_write(v, sizeof(*v));2758 instrument_atomic_read_write(v, sizeof(*v));
1145 return arch_atomic64_cmpxchg(v, old, new);2759 return raw_atomic64_cmpxchg(v, old, new);
1146}2760}
1147 2761
2762/**
2763 * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2764 * @v: pointer to atomic64_t
2765 * @old: s64 value to compare with
2766 * @new: s64 value to assign
2767 *
2768 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2769 *
2770 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
2771 *
2772 * Return: The original value of @v.
2773 */
1148static __always_inline s642774static __always_inline s64
1149atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)2775atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1150{2776{
1151 instrument_atomic_read_write(v, sizeof(*v));2777 instrument_atomic_read_write(v, sizeof(*v));
1152 return arch_atomic64_cmpxchg_acquire(v, old, new);2778 return raw_atomic64_cmpxchg_acquire(v, old, new);
1153}2779}
1154 2780
2781/**
2782 * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
2783 * @v: pointer to atomic64_t
2784 * @old: s64 value to compare with
2785 * @new: s64 value to assign
2786 *
2787 * If (@v == @old), atomically updates @v to @new with release ordering.
2788 *
2789 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
2790 *
2791 * Return: The original value of @v.
2792 */
1155static __always_inline s642793static __always_inline s64
1156atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)2794atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1157{2795{
1158 kcsan_release();2796 kcsan_release();
1159 instrument_atomic_read_write(v, sizeof(*v));2797 instrument_atomic_read_write(v, sizeof(*v));
1160 return arch_atomic64_cmpxchg_release(v, old, new);2798 return raw_atomic64_cmpxchg_release(v, old, new);
1161}2799}
1162 2800
2801/**
2802 * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2803 * @v: pointer to atomic64_t
2804 * @old: s64 value to compare with
2805 * @new: s64 value to assign
2806 *
2807 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2808 *
2809 * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
2810 *
2811 * Return: The original value of @v.
2812 */
1163static __always_inline s642813static __always_inline s64
1164atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)2814atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1165{2815{
1166 instrument_atomic_read_write(v, sizeof(*v));2816 instrument_atomic_read_write(v, sizeof(*v));
1167 return arch_atomic64_cmpxchg_relaxed(v, old, new);2817 return raw_atomic64_cmpxchg_relaxed(v, old, new);
1168}2818}
1169 2819
2820/**
2821 * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
2822 * @v: pointer to atomic64_t
2823 * @old: pointer to s64 value to compare with
2824 * @new: s64 value to assign
2825 *
2826 * If (@v == @old), atomically updates @v to @new with full ordering.
2827 * Otherwise, updates @old to the current value of @v.
2828 *
2829 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
2830 *
2831 * Return: @true if the exchange occured, @false otherwise.
2832 */
1170static __always_inline bool2833static __always_inline bool
1171atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)2834atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1172{2835{
1173 kcsan_mb();2836 kcsan_mb();
1174 instrument_atomic_read_write(v, sizeof(*v));2837 instrument_atomic_read_write(v, sizeof(*v));
1175 instrument_atomic_read_write(old, sizeof(*old));2838 instrument_atomic_read_write(old, sizeof(*old));
1176 return arch_atomic64_try_cmpxchg(v, old, new);2839 return raw_atomic64_try_cmpxchg(v, old, new);
1177}2840}
1178 2841
2842/**
2843 * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2844 * @v: pointer to atomic64_t
2845 * @old: pointer to s64 value to compare with
2846 * @new: s64 value to assign
2847 *
2848 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2849 * Otherwise, updates @old to the current value of @v.
2850 *
2851 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
2852 *
2853 * Return: @true if the exchange occured, @false otherwise.
2854 */
1179static __always_inline bool2855static __always_inline bool
1180atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)2856atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1181{2857{
1182 instrument_atomic_read_write(v, sizeof(*v));2858 instrument_atomic_read_write(v, sizeof(*v));
1183 instrument_atomic_read_write(old, sizeof(*old));2859 instrument_atomic_read_write(old, sizeof(*old));
1184 return arch_atomic64_try_cmpxchg_acquire(v, old, new);2860 return raw_atomic64_try_cmpxchg_acquire(v, old, new);
1185}2861}
1186 2862
2863/**
2864 * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
2865 * @v: pointer to atomic64_t
2866 * @old: pointer to s64 value to compare with
2867 * @new: s64 value to assign
2868 *
2869 * If (@v == @old), atomically updates @v to @new with release ordering.
2870 * Otherwise, updates @old to the current value of @v.
2871 *
2872 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
2873 *
2874 * Return: @true if the exchange occured, @false otherwise.
2875 */
1187static __always_inline bool2876static __always_inline bool
1188atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)2877atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1189{2878{
1190 kcsan_release();2879 kcsan_release();
1191 instrument_atomic_read_write(v, sizeof(*v));2880 instrument_atomic_read_write(v, sizeof(*v));
1192 instrument_atomic_read_write(old, sizeof(*old));2881 instrument_atomic_read_write(old, sizeof(*old));
1193 return arch_atomic64_try_cmpxchg_release(v, old, new);2882 return raw_atomic64_try_cmpxchg_release(v, old, new);
1194}2883}
1195 2884
2885/**
2886 * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2887 * @v: pointer to atomic64_t
2888 * @old: pointer to s64 value to compare with
2889 * @new: s64 value to assign
2890 *
2891 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2892 * Otherwise, updates @old to the current value of @v.
2893 *
2894 * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
2895 *
2896 * Return: @true if the exchange occured, @false otherwise.
2897 */
1196static __always_inline bool2898static __always_inline bool
1197atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)2899atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1198{2900{
1199 instrument_atomic_read_write(v, sizeof(*v));2901 instrument_atomic_read_write(v, sizeof(*v));
1200 instrument_atomic_read_write(old, sizeof(*old));2902 instrument_atomic_read_write(old, sizeof(*old));
1201 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);2903 return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
1202}2904}
1203 2905
2906/**
2907 * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
2908 * @i: s64 value to add
2909 * @v: pointer to atomic64_t
2910 *
2911 * Atomically updates @v to (@v - @i) with full ordering.
2912 *
2913 * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
2914 *
2915 * Return: @true if the resulting value of @v is zero, @false otherwise.
2916 */
1204static __always_inline bool2917static __always_inline bool
1205atomic64_sub_and_test(s64 i, atomic64_t *v)2918atomic64_sub_and_test(s64 i, atomic64_t *v)
1206{2919{
1207 kcsan_mb();2920 kcsan_mb();
1208 instrument_atomic_read_write(v, sizeof(*v));2921 instrument_atomic_read_write(v, sizeof(*v));
1209 return arch_atomic64_sub_and_test(i, v);2922 return raw_atomic64_sub_and_test(i, v);
1210}2923}
1211 2924
2925/**
2926 * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
2927 * @v: pointer to atomic64_t
2928 *
2929 * Atomically updates @v to (@v - 1) with full ordering.
2930 *
2931 * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
2932 *
2933 * Return: @true if the resulting value of @v is zero, @false otherwise.
2934 */
1212static __always_inline bool2935static __always_inline bool
1213atomic64_dec_and_test(atomic64_t *v)2936atomic64_dec_and_test(atomic64_t *v)
1214{2937{
1215 kcsan_mb();2938 kcsan_mb();
1216 instrument_atomic_read_write(v, sizeof(*v));2939 instrument_atomic_read_write(v, sizeof(*v));
1217 return arch_atomic64_dec_and_test(v);2940 return raw_atomic64_dec_and_test(v);
1218}2941}
1219 2942
2943/**
2944 * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
2945 * @v: pointer to atomic64_t
2946 *
2947 * Atomically updates @v to (@v + 1) with full ordering.
2948 *
2949 * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
2950 *
2951 * Return: @true if the resulting value of @v is zero, @false otherwise.
2952 */
1220static __always_inline bool2953static __always_inline bool
1221atomic64_inc_and_test(atomic64_t *v)2954atomic64_inc_and_test(atomic64_t *v)
1222{2955{
1223 kcsan_mb();2956 kcsan_mb();
1224 instrument_atomic_read_write(v, sizeof(*v));2957 instrument_atomic_read_write(v, sizeof(*v));
1225 return arch_atomic64_inc_and_test(v);2958 return raw_atomic64_inc_and_test(v);
1226}2959}
1227 2960
2961/**
2962 * atomic64_add_negative() - atomic add and test if negative with full ordering
2963 * @i: s64 value to add
2964 * @v: pointer to atomic64_t
2965 *
2966 * Atomically updates @v to (@v + @i) with full ordering.
2967 *
2968 * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
2969 *
2970 * Return: @true if the resulting value of @v is negative, @false otherwise.
2971 */
1228static __always_inline bool2972static __always_inline bool
1229atomic64_add_negative(s64 i, atomic64_t *v)2973atomic64_add_negative(s64 i, atomic64_t *v)
1230{2974{
1231 kcsan_mb();2975 kcsan_mb();
1232 instrument_atomic_read_write(v, sizeof(*v));2976 instrument_atomic_read_write(v, sizeof(*v));
1233 return arch_atomic64_add_negative(i, v);2977 return raw_atomic64_add_negative(i, v);
1234}2978}
1235 2979
2980/**
2981 * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
2982 * @i: s64 value to add
2983 * @v: pointer to atomic64_t
2984 *
2985 * Atomically updates @v to (@v + @i) with acquire ordering.
2986 *
2987 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
2988 *
2989 * Return: @true if the resulting value of @v is negative, @false otherwise.
2990 */
1236static __always_inline bool2991static __always_inline bool
1237atomic64_add_negative_acquire(s64 i, atomic64_t *v)2992atomic64_add_negative_acquire(s64 i, atomic64_t *v)
1238{2993{
1239 instrument_atomic_read_write(v, sizeof(*v));2994 instrument_atomic_read_write(v, sizeof(*v));
1240 return arch_atomic64_add_negative_acquire(i, v);2995 return raw_atomic64_add_negative_acquire(i, v);
1241}2996}
1242 2997
2998/**
2999 * atomic64_add_negative_release() - atomic add and test if negative with release ordering
3000 * @i: s64 value to add
3001 * @v: pointer to atomic64_t
3002 *
3003 * Atomically updates @v to (@v + @i) with release ordering.
3004 *
3005 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
3006 *
3007 * Return: @true if the resulting value of @v is negative, @false otherwise.
3008 */
1243static __always_inline bool3009static __always_inline bool
1244atomic64_add_negative_release(s64 i, atomic64_t *v)3010atomic64_add_negative_release(s64 i, atomic64_t *v)
1245{3011{
1246 kcsan_release();3012 kcsan_release();
1247 instrument_atomic_read_write(v, sizeof(*v));3013 instrument_atomic_read_write(v, sizeof(*v));
1248 return arch_atomic64_add_negative_release(i, v);3014 return raw_atomic64_add_negative_release(i, v);
1249}3015}
1250 3016
3017/**
3018 * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
3019 * @i: s64 value to add
3020 * @v: pointer to atomic64_t
3021 *
3022 * Atomically updates @v to (@v + @i) with relaxed ordering.
3023 *
3024 * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
3025 *
3026 * Return: @true if the resulting value of @v is negative, @false otherwise.
3027 */
1251static __always_inline bool3028static __always_inline bool
1252atomic64_add_negative_relaxed(s64 i, atomic64_t *v)3029atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
1253{3030{
1254 instrument_atomic_read_write(v, sizeof(*v));3031 instrument_atomic_read_write(v, sizeof(*v));
1255 return arch_atomic64_add_negative_relaxed(i, v);3032 return raw_atomic64_add_negative_relaxed(i, v);
1256}3033}
1257 3034
3035/**
3036 * atomic64_fetch_add_unless() - atomic add unless value with full ordering
3037 * @v: pointer to atomic64_t
3038 * @a: s64 value to add
3039 * @u: s64 value to compare with
3040 *
3041 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3042 *
3043 * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
3044 *
3045 * Return: The original value of @v.
3046 */
1258static __always_inline s643047static __always_inline s64
1259atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)3048atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1260{3049{
1261 kcsan_mb();3050 kcsan_mb();
1262 instrument_atomic_read_write(v, sizeof(*v));3051 instrument_atomic_read_write(v, sizeof(*v));
1263 return arch_atomic64_fetch_add_unless(v, a, u);3052 return raw_atomic64_fetch_add_unless(v, a, u);
1264}3053}
1265 3054
3055/**
3056 * atomic64_add_unless() - atomic add unless value with full ordering
3057 * @v: pointer to atomic64_t
3058 * @a: s64 value to add
3059 * @u: s64 value to compare with
3060 *
3061 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
3062 *
3063 * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
3064 *
3065 * Return: @true if @v was updated, @false otherwise.
3066 */
1266static __always_inline bool3067static __always_inline bool
1267atomic64_add_unless(atomic64_t *v, s64 a, s64 u)3068atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1268{3069{
1269 kcsan_mb();3070 kcsan_mb();
1270 instrument_atomic_read_write(v, sizeof(*v));3071 instrument_atomic_read_write(v, sizeof(*v));
1271 return arch_atomic64_add_unless(v, a, u);3072 return raw_atomic64_add_unless(v, a, u);
1272}3073}
1273 3074
3075/**
3076 * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
3077 * @v: pointer to atomic64_t
3078 *
3079 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
3080 *
3081 * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
3082 *
3083 * Return: @true if @v was updated, @false otherwise.
3084 */
1274static __always_inline bool3085static __always_inline bool
1275atomic64_inc_not_zero(atomic64_t *v)3086atomic64_inc_not_zero(atomic64_t *v)
1276{3087{
1277 kcsan_mb();3088 kcsan_mb();
1278 instrument_atomic_read_write(v, sizeof(*v));3089 instrument_atomic_read_write(v, sizeof(*v));
1279 return arch_atomic64_inc_not_zero(v);3090 return raw_atomic64_inc_not_zero(v);
1280}3091}
1281 3092
3093/**
3094 * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
3095 * @v: pointer to atomic64_t
3096 *
3097 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
3098 *
3099 * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
3100 *
3101 * Return: @true if @v was updated, @false otherwise.
3102 */
1282static __always_inline bool3103static __always_inline bool
1283atomic64_inc_unless_negative(atomic64_t *v)3104atomic64_inc_unless_negative(atomic64_t *v)
1284{3105{
1285 kcsan_mb();3106 kcsan_mb();
1286 instrument_atomic_read_write(v, sizeof(*v));3107 instrument_atomic_read_write(v, sizeof(*v));
1287 return arch_atomic64_inc_unless_negative(v);3108 return raw_atomic64_inc_unless_negative(v);
1288}3109}
1289 3110
3111/**
3112 * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
3113 * @v: pointer to atomic64_t
3114 *
3115 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
3116 *
3117 * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
3118 *
3119 * Return: @true if @v was updated, @false otherwise.
3120 */
1290static __always_inline bool3121static __always_inline bool
1291atomic64_dec_unless_positive(atomic64_t *v)3122atomic64_dec_unless_positive(atomic64_t *v)
1292{3123{
1293 kcsan_mb();3124 kcsan_mb();
1294 instrument_atomic_read_write(v, sizeof(*v));3125 instrument_atomic_read_write(v, sizeof(*v));
1295 return arch_atomic64_dec_unless_positive(v);3126 return raw_atomic64_dec_unless_positive(v);
1296}3127}
1297 3128
3129/**
3130 * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
3131 * @v: pointer to atomic64_t
3132 *
3133 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
3134 *
3135 * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
3136 *
3137 * Return: The old value of (@v - 1), regardless of whether @v was updated.
3138 */
1298static __always_inline s643139static __always_inline s64
1299atomic64_dec_if_positive(atomic64_t *v)3140atomic64_dec_if_positive(atomic64_t *v)
1300{3141{
1301 kcsan_mb();3142 kcsan_mb();
1302 instrument_atomic_read_write(v, sizeof(*v));3143 instrument_atomic_read_write(v, sizeof(*v));
1303 return arch_atomic64_dec_if_positive(v);3144 return raw_atomic64_dec_if_positive(v);
1304}3145}
1305 3146
3147/**
3148 * atomic_long_read() - atomic load with relaxed ordering
3149 * @v: pointer to atomic_long_t
3150 *
3151 * Atomically loads the value of @v with relaxed ordering.
3152 *
3153 * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
3154 *
3155 * Return: The value loaded from @v.
3156 */
1306static __always_inline long3157static __always_inline long
1307atomic_long_read(const atomic_long_t *v)3158atomic_long_read(const atomic_long_t *v)
1308{3159{
1309 instrument_atomic_read(v, sizeof(*v));3160 instrument_atomic_read(v, sizeof(*v));
1310 return arch_atomic_long_read(v);3161 return raw_atomic_long_read(v);
1311}3162}
1312 3163
3164/**
3165 * atomic_long_read_acquire() - atomic load with acquire ordering
3166 * @v: pointer to atomic_long_t
3167 *
3168 * Atomically loads the value of @v with acquire ordering.
3169 *
3170 * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
3171 *
3172 * Return: The value loaded from @v.
3173 */
1313static __always_inline long3174static __always_inline long
1314atomic_long_read_acquire(const atomic_long_t *v)3175atomic_long_read_acquire(const atomic_long_t *v)
1315{3176{
1316 instrument_atomic_read(v, sizeof(*v));3177 instrument_atomic_read(v, sizeof(*v));
1317 return arch_atomic_long_read_acquire(v);3178 return raw_atomic_long_read_acquire(v);
1318}3179}
1319 3180
3181/**
3182 * atomic_long_set() - atomic set with relaxed ordering
3183 * @v: pointer to atomic_long_t
3184 * @i: long value to assign
3185 *
3186 * Atomically sets @v to @i with relaxed ordering.
3187 *
3188 * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
3189 *
3190 * Return: Nothing.
3191 */
1320static __always_inline void3192static __always_inline void
1321atomic_long_set(atomic_long_t *v, long i)3193atomic_long_set(atomic_long_t *v, long i)
1322{3194{
1323 instrument_atomic_write(v, sizeof(*v));3195 instrument_atomic_write(v, sizeof(*v));
1324 arch_atomic_long_set(v, i);3196 raw_atomic_long_set(v, i);
1325}3197}
1326 3198
3199/**
3200 * atomic_long_set_release() - atomic set with release ordering
3201 * @v: pointer to atomic_long_t
3202 * @i: long value to assign
3203 *
3204 * Atomically sets @v to @i with release ordering.
3205 *
3206 * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
3207 *
3208 * Return: Nothing.
3209 */
1327static __always_inline void3210static __always_inline void
1328atomic_long_set_release(atomic_long_t *v, long i)3211atomic_long_set_release(atomic_long_t *v, long i)
1329{3212{
1330 kcsan_release();3213 kcsan_release();
1331 instrument_atomic_write(v, sizeof(*v));3214 instrument_atomic_write(v, sizeof(*v));
1332 arch_atomic_long_set_release(v, i);3215 raw_atomic_long_set_release(v, i);
1333}3216}
1334 3217
3218/**
3219 * atomic_long_add() - atomic add with relaxed ordering
3220 * @i: long value to add
3221 * @v: pointer to atomic_long_t
3222 *
3223 * Atomically updates @v to (@v + @i) with relaxed ordering.
3224 *
3225 * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
3226 *
3227 * Return: Nothing.
3228 */
1335static __always_inline void3229static __always_inline void
1336atomic_long_add(long i, atomic_long_t *v)3230atomic_long_add(long i, atomic_long_t *v)
1337{3231{
1338 instrument_atomic_read_write(v, sizeof(*v));3232 instrument_atomic_read_write(v, sizeof(*v));
1339 arch_atomic_long_add(i, v);3233 raw_atomic_long_add(i, v);
1340}3234}
1341 3235
3236/**
3237 * atomic_long_add_return() - atomic add with full ordering
3238 * @i: long value to add
3239 * @v: pointer to atomic_long_t
3240 *
3241 * Atomically updates @v to (@v + @i) with full ordering.
3242 *
3243 * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
3244 *
3245 * Return: The updated value of @v.
3246 */
1342static __always_inline long3247static __always_inline long
1343atomic_long_add_return(long i, atomic_long_t *v)3248atomic_long_add_return(long i, atomic_long_t *v)
1344{3249{
1345 kcsan_mb();3250 kcsan_mb();
1346 instrument_atomic_read_write(v, sizeof(*v));3251 instrument_atomic_read_write(v, sizeof(*v));
1347 return arch_atomic_long_add_return(i, v);3252 return raw_atomic_long_add_return(i, v);
1348}3253}
1349 3254
3255/**
3256 * atomic_long_add_return_acquire() - atomic add with acquire ordering
3257 * @i: long value to add
3258 * @v: pointer to atomic_long_t
3259 *
3260 * Atomically updates @v to (@v + @i) with acquire ordering.
3261 *
3262 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
3263 *
3264 * Return: The updated value of @v.
3265 */
1350static __always_inline long3266static __always_inline long
1351atomic_long_add_return_acquire(long i, atomic_long_t *v)3267atomic_long_add_return_acquire(long i, atomic_long_t *v)
1352{3268{
1353 instrument_atomic_read_write(v, sizeof(*v));3269 instrument_atomic_read_write(v, sizeof(*v));
1354 return arch_atomic_long_add_return_acquire(i, v);3270 return raw_atomic_long_add_return_acquire(i, v);
1355}3271}
1356 3272
3273/**
3274 * atomic_long_add_return_release() - atomic add with release ordering
3275 * @i: long value to add
3276 * @v: pointer to atomic_long_t
3277 *
3278 * Atomically updates @v to (@v + @i) with release ordering.
3279 *
3280 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
3281 *
3282 * Return: The updated value of @v.
3283 */
1357static __always_inline long3284static __always_inline long
1358atomic_long_add_return_release(long i, atomic_long_t *v)3285atomic_long_add_return_release(long i, atomic_long_t *v)
1359{3286{
1360 kcsan_release();3287 kcsan_release();
1361 instrument_atomic_read_write(v, sizeof(*v));3288 instrument_atomic_read_write(v, sizeof(*v));
1362 return arch_atomic_long_add_return_release(i, v);3289 return raw_atomic_long_add_return_release(i, v);
1363}3290}
1364 3291
3292/**
3293 * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
3294 * @i: long value to add
3295 * @v: pointer to atomic_long_t
3296 *
3297 * Atomically updates @v to (@v + @i) with relaxed ordering.
3298 *
3299 * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
3300 *
3301 * Return: The updated value of @v.
3302 */
1365static __always_inline long3303static __always_inline long
1366atomic_long_add_return_relaxed(long i, atomic_long_t *v)3304atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1367{3305{
1368 instrument_atomic_read_write(v, sizeof(*v));3306 instrument_atomic_read_write(v, sizeof(*v));
1369 return arch_atomic_long_add_return_relaxed(i, v);3307 return raw_atomic_long_add_return_relaxed(i, v);
1370}3308}
1371 3309
3310/**
3311 * atomic_long_fetch_add() - atomic add with full ordering
3312 * @i: long value to add
3313 * @v: pointer to atomic_long_t
3314 *
3315 * Atomically updates @v to (@v + @i) with full ordering.
3316 *
3317 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
3318 *
3319 * Return: The original value of @v.
3320 */
1372static __always_inline long3321static __always_inline long
1373atomic_long_fetch_add(long i, atomic_long_t *v)3322atomic_long_fetch_add(long i, atomic_long_t *v)
1374{3323{
1375 kcsan_mb();3324 kcsan_mb();
1376 instrument_atomic_read_write(v, sizeof(*v));3325 instrument_atomic_read_write(v, sizeof(*v));
1377 return arch_atomic_long_fetch_add(i, v);3326 return raw_atomic_long_fetch_add(i, v);
1378}3327}
1379 3328
3329/**
3330 * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
3331 * @i: long value to add
3332 * @v: pointer to atomic_long_t
3333 *
3334 * Atomically updates @v to (@v + @i) with acquire ordering.
3335 *
3336 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
3337 *
3338 * Return: The original value of @v.
3339 */
1380static __always_inline long3340static __always_inline long
1381atomic_long_fetch_add_acquire(long i, atomic_long_t *v)3341atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1382{3342{
1383 instrument_atomic_read_write(v, sizeof(*v));3343 instrument_atomic_read_write(v, sizeof(*v));
1384 return arch_atomic_long_fetch_add_acquire(i, v);3344 return raw_atomic_long_fetch_add_acquire(i, v);
1385}3345}
1386 3346
3347/**
3348 * atomic_long_fetch_add_release() - atomic add with release ordering
3349 * @i: long value to add
3350 * @v: pointer to atomic_long_t
3351 *
3352 * Atomically updates @v to (@v + @i) with release ordering.
3353 *
3354 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
3355 *
3356 * Return: The original value of @v.
3357 */
1387static __always_inline long3358static __always_inline long
1388atomic_long_fetch_add_release(long i, atomic_long_t *v)3359atomic_long_fetch_add_release(long i, atomic_long_t *v)
1389{3360{
1390 kcsan_release();3361 kcsan_release();
1391 instrument_atomic_read_write(v, sizeof(*v));3362 instrument_atomic_read_write(v, sizeof(*v));
1392 return arch_atomic_long_fetch_add_release(i, v);3363 return raw_atomic_long_fetch_add_release(i, v);
1393}3364}
1394 3365
3366/**
3367 * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
3368 * @i: long value to add
3369 * @v: pointer to atomic_long_t
3370 *
3371 * Atomically updates @v to (@v + @i) with relaxed ordering.
3372 *
3373 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
3374 *
3375 * Return: The original value of @v.
3376 */
1395static __always_inline long3377static __always_inline long
1396atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)3378atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1397{3379{
1398 instrument_atomic_read_write(v, sizeof(*v));3380 instrument_atomic_read_write(v, sizeof(*v));
1399 return arch_atomic_long_fetch_add_relaxed(i, v);3381 return raw_atomic_long_fetch_add_relaxed(i, v);
1400}3382}
1401 3383
3384/**
3385 * atomic_long_sub() - atomic subtract with relaxed ordering
3386 * @i: long value to subtract
3387 * @v: pointer to atomic_long_t
3388 *
3389 * Atomically updates @v to (@v - @i) with relaxed ordering.
3390 *
3391 * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
3392 *
3393 * Return: Nothing.
3394 */
1402static __always_inline void3395static __always_inline void
1403atomic_long_sub(long i, atomic_long_t *v)3396atomic_long_sub(long i, atomic_long_t *v)
1404{3397{
1405 instrument_atomic_read_write(v, sizeof(*v));3398 instrument_atomic_read_write(v, sizeof(*v));
1406 arch_atomic_long_sub(i, v);3399 raw_atomic_long_sub(i, v);
1407}3400}
1408 3401
3402/**
3403 * atomic_long_sub_return() - atomic subtract with full ordering
3404 * @i: long value to subtract
3405 * @v: pointer to atomic_long_t
3406 *
3407 * Atomically updates @v to (@v - @i) with full ordering.
3408 *
3409 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
3410 *
3411 * Return: The updated value of @v.
3412 */
1409static __always_inline long3413static __always_inline long
1410atomic_long_sub_return(long i, atomic_long_t *v)3414atomic_long_sub_return(long i, atomic_long_t *v)
1411{3415{
1412 kcsan_mb();3416 kcsan_mb();
1413 instrument_atomic_read_write(v, sizeof(*v));3417 instrument_atomic_read_write(v, sizeof(*v));
1414 return arch_atomic_long_sub_return(i, v);3418 return raw_atomic_long_sub_return(i, v);
1415}3419}
1416 3420
3421/**
3422 * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
3423 * @i: long value to subtract
3424 * @v: pointer to atomic_long_t
3425 *
3426 * Atomically updates @v to (@v - @i) with acquire ordering.
3427 *
3428 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
3429 *
3430 * Return: The updated value of @v.
3431 */
1417static __always_inline long3432static __always_inline long
1418atomic_long_sub_return_acquire(long i, atomic_long_t *v)3433atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1419{3434{
1420 instrument_atomic_read_write(v, sizeof(*v));3435 instrument_atomic_read_write(v, sizeof(*v));
1421 return arch_atomic_long_sub_return_acquire(i, v);3436 return raw_atomic_long_sub_return_acquire(i, v);
1422}3437}
1423 3438
3439/**
3440 * atomic_long_sub_return_release() - atomic subtract with release ordering
3441 * @i: long value to subtract
3442 * @v: pointer to atomic_long_t
3443 *
3444 * Atomically updates @v to (@v - @i) with release ordering.
3445 *
3446 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
3447 *
3448 * Return: The updated value of @v.
3449 */
1424static __always_inline long3450static __always_inline long
1425atomic_long_sub_return_release(long i, atomic_long_t *v)3451atomic_long_sub_return_release(long i, atomic_long_t *v)
1426{3452{
1427 kcsan_release();3453 kcsan_release();
1428 instrument_atomic_read_write(v, sizeof(*v));3454 instrument_atomic_read_write(v, sizeof(*v));
1429 return arch_atomic_long_sub_return_release(i, v);3455 return raw_atomic_long_sub_return_release(i, v);
1430}3456}
1431 3457
3458/**
3459 * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
3460 * @i: long value to subtract
3461 * @v: pointer to atomic_long_t
3462 *
3463 * Atomically updates @v to (@v - @i) with relaxed ordering.
3464 *
3465 * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
3466 *
3467 * Return: The updated value of @v.
3468 */
1432static __always_inline long3469static __always_inline long
1433atomic_long_sub_return_relaxed(long i, atomic_long_t *v)3470atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1434{3471{
1435 instrument_atomic_read_write(v, sizeof(*v));3472 instrument_atomic_read_write(v, sizeof(*v));
1436 return arch_atomic_long_sub_return_relaxed(i, v);3473 return raw_atomic_long_sub_return_relaxed(i, v);
1437}3474}
1438 3475
3476/**
3477 * atomic_long_fetch_sub() - atomic subtract with full ordering
3478 * @i: long value to subtract
3479 * @v: pointer to atomic_long_t
3480 *
3481 * Atomically updates @v to (@v - @i) with full ordering.
3482 *
3483 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
3484 *
3485 * Return: The original value of @v.
3486 */
1439static __always_inline long3487static __always_inline long
1440atomic_long_fetch_sub(long i, atomic_long_t *v)3488atomic_long_fetch_sub(long i, atomic_long_t *v)
1441{3489{
1442 kcsan_mb();3490 kcsan_mb();
1443 instrument_atomic_read_write(v, sizeof(*v));3491 instrument_atomic_read_write(v, sizeof(*v));
1444 return arch_atomic_long_fetch_sub(i, v);3492 return raw_atomic_long_fetch_sub(i, v);
1445}3493}
1446 3494
3495/**
3496 * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
3497 * @i: long value to subtract
3498 * @v: pointer to atomic_long_t
3499 *
3500 * Atomically updates @v to (@v - @i) with acquire ordering.
3501 *
3502 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
3503 *
3504 * Return: The original value of @v.
3505 */
1447static __always_inline long3506static __always_inline long
1448atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)3507atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1449{3508{
1450 instrument_atomic_read_write(v, sizeof(*v));3509 instrument_atomic_read_write(v, sizeof(*v));
1451 return arch_atomic_long_fetch_sub_acquire(i, v);3510 return raw_atomic_long_fetch_sub_acquire(i, v);
1452}3511}
1453 3512
3513/**
3514 * atomic_long_fetch_sub_release() - atomic subtract with release ordering
3515 * @i: long value to subtract
3516 * @v: pointer to atomic_long_t
3517 *
3518 * Atomically updates @v to (@v - @i) with release ordering.
3519 *
3520 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
3521 *
3522 * Return: The original value of @v.
3523 */
1454static __always_inline long3524static __always_inline long
1455atomic_long_fetch_sub_release(long i, atomic_long_t *v)3525atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1456{3526{
1457 kcsan_release();3527 kcsan_release();
1458 instrument_atomic_read_write(v, sizeof(*v));3528 instrument_atomic_read_write(v, sizeof(*v));
1459 return arch_atomic_long_fetch_sub_release(i, v);3529 return raw_atomic_long_fetch_sub_release(i, v);
1460}3530}
1461 3531
3532/**
3533 * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3534 * @i: long value to subtract
3535 * @v: pointer to atomic_long_t
3536 *
3537 * Atomically updates @v to (@v - @i) with relaxed ordering.
3538 *
3539 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
3540 *
3541 * Return: The original value of @v.
3542 */
1462static __always_inline long3543static __always_inline long
1463atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)3544atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1464{3545{
1465 instrument_atomic_read_write(v, sizeof(*v));3546 instrument_atomic_read_write(v, sizeof(*v));
1466 return arch_atomic_long_fetch_sub_relaxed(i, v);3547 return raw_atomic_long_fetch_sub_relaxed(i, v);
1467}3548}
1468 3549
3550/**
3551 * atomic_long_inc() - atomic increment with relaxed ordering
3552 * @v: pointer to atomic_long_t
3553 *
3554 * Atomically updates @v to (@v + 1) with relaxed ordering.
3555 *
3556 * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
3557 *
3558 * Return: Nothing.
3559 */
1469static __always_inline void3560static __always_inline void
1470atomic_long_inc(atomic_long_t *v)3561atomic_long_inc(atomic_long_t *v)
1471{3562{
1472 instrument_atomic_read_write(v, sizeof(*v));3563 instrument_atomic_read_write(v, sizeof(*v));
1473 arch_atomic_long_inc(v);3564 raw_atomic_long_inc(v);
1474}3565}
1475 3566
3567/**
3568 * atomic_long_inc_return() - atomic increment with full ordering
3569 * @v: pointer to atomic_long_t
3570 *
3571 * Atomically updates @v to (@v + 1) with full ordering.
3572 *
3573 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
3574 *
3575 * Return: The updated value of @v.
3576 */
1476static __always_inline long3577static __always_inline long
1477atomic_long_inc_return(atomic_long_t *v)3578atomic_long_inc_return(atomic_long_t *v)
1478{3579{
1479 kcsan_mb();3580 kcsan_mb();
1480 instrument_atomic_read_write(v, sizeof(*v));3581 instrument_atomic_read_write(v, sizeof(*v));
1481 return arch_atomic_long_inc_return(v);3582 return raw_atomic_long_inc_return(v);
1482}3583}
1483 3584
3585/**
3586 * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
3587 * @v: pointer to atomic_long_t
3588 *
3589 * Atomically updates @v to (@v + 1) with acquire ordering.
3590 *
3591 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
3592 *
3593 * Return: The updated value of @v.
3594 */
1484static __always_inline long3595static __always_inline long
1485atomic_long_inc_return_acquire(atomic_long_t *v)3596atomic_long_inc_return_acquire(atomic_long_t *v)
1486{3597{
1487 instrument_atomic_read_write(v, sizeof(*v));3598 instrument_atomic_read_write(v, sizeof(*v));
1488 return arch_atomic_long_inc_return_acquire(v);3599 return raw_atomic_long_inc_return_acquire(v);
1489}3600}
1490 3601
3602/**
3603 * atomic_long_inc_return_release() - atomic increment with release ordering
3604 * @v: pointer to atomic_long_t
3605 *
3606 * Atomically updates @v to (@v + 1) with release ordering.
3607 *
3608 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
3609 *
3610 * Return: The updated value of @v.
3611 */
1491static __always_inline long3612static __always_inline long
1492atomic_long_inc_return_release(atomic_long_t *v)3613atomic_long_inc_return_release(atomic_long_t *v)
1493{3614{
1494 kcsan_release();3615 kcsan_release();
1495 instrument_atomic_read_write(v, sizeof(*v));3616 instrument_atomic_read_write(v, sizeof(*v));
1496 return arch_atomic_long_inc_return_release(v);3617 return raw_atomic_long_inc_return_release(v);
1497}3618}
1498 3619
3620/**
3621 * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
3622 * @v: pointer to atomic_long_t
3623 *
3624 * Atomically updates @v to (@v + 1) with relaxed ordering.
3625 *
3626 * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
3627 *
3628 * Return: The updated value of @v.
3629 */
1499static __always_inline long3630static __always_inline long
1500atomic_long_inc_return_relaxed(atomic_long_t *v)3631atomic_long_inc_return_relaxed(atomic_long_t *v)
1501{3632{
1502 instrument_atomic_read_write(v, sizeof(*v));3633 instrument_atomic_read_write(v, sizeof(*v));
1503 return arch_atomic_long_inc_return_relaxed(v);3634 return raw_atomic_long_inc_return_relaxed(v);
1504}3635}
1505 3636
3637/**
3638 * atomic_long_fetch_inc() - atomic increment with full ordering
3639 * @v: pointer to atomic_long_t
3640 *
3641 * Atomically updates @v to (@v + 1) with full ordering.
3642 *
3643 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
3644 *
3645 * Return: The original value of @v.
3646 */
1506static __always_inline long3647static __always_inline long
1507atomic_long_fetch_inc(atomic_long_t *v)3648atomic_long_fetch_inc(atomic_long_t *v)
1508{3649{
1509 kcsan_mb();3650 kcsan_mb();
1510 instrument_atomic_read_write(v, sizeof(*v));3651 instrument_atomic_read_write(v, sizeof(*v));
1511 return arch_atomic_long_fetch_inc(v);3652 return raw_atomic_long_fetch_inc(v);
1512}3653}
1513 3654
3655/**
3656 * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
3657 * @v: pointer to atomic_long_t
3658 *
3659 * Atomically updates @v to (@v + 1) with acquire ordering.
3660 *
3661 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
3662 *
3663 * Return: The original value of @v.
3664 */
1514static __always_inline long3665static __always_inline long
1515atomic_long_fetch_inc_acquire(atomic_long_t *v)3666atomic_long_fetch_inc_acquire(atomic_long_t *v)
1516{3667{
1517 instrument_atomic_read_write(v, sizeof(*v));3668 instrument_atomic_read_write(v, sizeof(*v));
1518 return arch_atomic_long_fetch_inc_acquire(v);3669 return raw_atomic_long_fetch_inc_acquire(v);
1519}3670}
1520 3671
3672/**
3673 * atomic_long_fetch_inc_release() - atomic increment with release ordering
3674 * @v: pointer to atomic_long_t
3675 *
3676 * Atomically updates @v to (@v + 1) with release ordering.
3677 *
3678 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
3679 *
3680 * Return: The original value of @v.
3681 */
1521static __always_inline long3682static __always_inline long
1522atomic_long_fetch_inc_release(atomic_long_t *v)3683atomic_long_fetch_inc_release(atomic_long_t *v)
1523{3684{
1524 kcsan_release();3685 kcsan_release();
1525 instrument_atomic_read_write(v, sizeof(*v));3686 instrument_atomic_read_write(v, sizeof(*v));
1526 return arch_atomic_long_fetch_inc_release(v);3687 return raw_atomic_long_fetch_inc_release(v);
1527}3688}
1528 3689
3690/**
3691 * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
3692 * @v: pointer to atomic_long_t
3693 *
3694 * Atomically updates @v to (@v + 1) with relaxed ordering.
3695 *
3696 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
3697 *
3698 * Return: The original value of @v.
3699 */
1529static __always_inline long3700static __always_inline long
1530atomic_long_fetch_inc_relaxed(atomic_long_t *v)3701atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1531{3702{
1532 instrument_atomic_read_write(v, sizeof(*v));3703 instrument_atomic_read_write(v, sizeof(*v));
1533 return arch_atomic_long_fetch_inc_relaxed(v);3704 return raw_atomic_long_fetch_inc_relaxed(v);
1534}3705}
1535 3706
3707/**
3708 * atomic_long_dec() - atomic decrement with relaxed ordering
3709 * @v: pointer to atomic_long_t
3710 *
3711 * Atomically updates @v to (@v - 1) with relaxed ordering.
3712 *
3713 * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
3714 *
3715 * Return: Nothing.
3716 */
1536static __always_inline void3717static __always_inline void
1537atomic_long_dec(atomic_long_t *v)3718atomic_long_dec(atomic_long_t *v)
1538{3719{
1539 instrument_atomic_read_write(v, sizeof(*v));3720 instrument_atomic_read_write(v, sizeof(*v));
1540 arch_atomic_long_dec(v);3721 raw_atomic_long_dec(v);
1541}3722}
1542 3723
3724/**
3725 * atomic_long_dec_return() - atomic decrement with full ordering
3726 * @v: pointer to atomic_long_t
3727 *
3728 * Atomically updates @v to (@v - 1) with full ordering.
3729 *
3730 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
3731 *
3732 * Return: The updated value of @v.
3733 */
1543static __always_inline long3734static __always_inline long
1544atomic_long_dec_return(atomic_long_t *v)3735atomic_long_dec_return(atomic_long_t *v)
1545{3736{
1546 kcsan_mb();3737 kcsan_mb();
1547 instrument_atomic_read_write(v, sizeof(*v));3738 instrument_atomic_read_write(v, sizeof(*v));
1548 return arch_atomic_long_dec_return(v);3739 return raw_atomic_long_dec_return(v);
1549}3740}
1550 3741
3742/**
3743 * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
3744 * @v: pointer to atomic_long_t
3745 *
3746 * Atomically updates @v to (@v - 1) with acquire ordering.
3747 *
3748 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
3749 *
3750 * Return: The updated value of @v.
3751 */
1551static __always_inline long3752static __always_inline long
1552atomic_long_dec_return_acquire(atomic_long_t *v)3753atomic_long_dec_return_acquire(atomic_long_t *v)
1553{3754{
1554 instrument_atomic_read_write(v, sizeof(*v));3755 instrument_atomic_read_write(v, sizeof(*v));
1555 return arch_atomic_long_dec_return_acquire(v);3756 return raw_atomic_long_dec_return_acquire(v);
1556}3757}
1557 3758
3759/**
3760 * atomic_long_dec_return_release() - atomic decrement with release ordering
3761 * @v: pointer to atomic_long_t
3762 *
3763 * Atomically updates @v to (@v - 1) with release ordering.
3764 *
3765 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
3766 *
3767 * Return: The updated value of @v.
3768 */
1558static __always_inline long3769static __always_inline long
1559atomic_long_dec_return_release(atomic_long_t *v)3770atomic_long_dec_return_release(atomic_long_t *v)
1560{3771{
1561 kcsan_release();3772 kcsan_release();
1562 instrument_atomic_read_write(v, sizeof(*v));3773 instrument_atomic_read_write(v, sizeof(*v));
1563 return arch_atomic_long_dec_return_release(v);3774 return raw_atomic_long_dec_return_release(v);
1564}3775}
1565 3776
3777/**
3778 * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
3779 * @v: pointer to atomic_long_t
3780 *
3781 * Atomically updates @v to (@v - 1) with relaxed ordering.
3782 *
3783 * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
3784 *
3785 * Return: The updated value of @v.
3786 */
1566static __always_inline long3787static __always_inline long
1567atomic_long_dec_return_relaxed(atomic_long_t *v)3788atomic_long_dec_return_relaxed(atomic_long_t *v)
1568{3789{
1569 instrument_atomic_read_write(v, sizeof(*v));3790 instrument_atomic_read_write(v, sizeof(*v));
1570 return arch_atomic_long_dec_return_relaxed(v);3791 return raw_atomic_long_dec_return_relaxed(v);
1571}3792}
1572 3793
3794/**
3795 * atomic_long_fetch_dec() - atomic decrement with full ordering
3796 * @v: pointer to atomic_long_t
3797 *
3798 * Atomically updates @v to (@v - 1) with full ordering.
3799 *
3800 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
3801 *
3802 * Return: The original value of @v.
3803 */
1573static __always_inline long3804static __always_inline long
1574atomic_long_fetch_dec(atomic_long_t *v)3805atomic_long_fetch_dec(atomic_long_t *v)
1575{3806{
1576 kcsan_mb();3807 kcsan_mb();
1577 instrument_atomic_read_write(v, sizeof(*v));3808 instrument_atomic_read_write(v, sizeof(*v));
1578 return arch_atomic_long_fetch_dec(v);3809 return raw_atomic_long_fetch_dec(v);
1579}3810}
1580 3811
3812/**
3813 * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
3814 * @v: pointer to atomic_long_t
3815 *
3816 * Atomically updates @v to (@v - 1) with acquire ordering.
3817 *
3818 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
3819 *
3820 * Return: The original value of @v.
3821 */
1581static __always_inline long3822static __always_inline long
1582atomic_long_fetch_dec_acquire(atomic_long_t *v)3823atomic_long_fetch_dec_acquire(atomic_long_t *v)
1583{3824{
1584 instrument_atomic_read_write(v, sizeof(*v));3825 instrument_atomic_read_write(v, sizeof(*v));
1585 return arch_atomic_long_fetch_dec_acquire(v);3826 return raw_atomic_long_fetch_dec_acquire(v);
1586}3827}
1587 3828
3829/**
3830 * atomic_long_fetch_dec_release() - atomic decrement with release ordering
3831 * @v: pointer to atomic_long_t
3832 *
3833 * Atomically updates @v to (@v - 1) with release ordering.
3834 *
3835 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
3836 *
3837 * Return: The original value of @v.
3838 */
1588static __always_inline long3839static __always_inline long
1589atomic_long_fetch_dec_release(atomic_long_t *v)3840atomic_long_fetch_dec_release(atomic_long_t *v)
1590{3841{
1591 kcsan_release();3842 kcsan_release();
1592 instrument_atomic_read_write(v, sizeof(*v));3843 instrument_atomic_read_write(v, sizeof(*v));
1593 return arch_atomic_long_fetch_dec_release(v);3844 return raw_atomic_long_fetch_dec_release(v);
1594}3845}
1595 3846
3847/**
3848 * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3849 * @v: pointer to atomic_long_t
3850 *
3851 * Atomically updates @v to (@v - 1) with relaxed ordering.
3852 *
3853 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
3854 *
3855 * Return: The original value of @v.
3856 */
1596static __always_inline long3857static __always_inline long
1597atomic_long_fetch_dec_relaxed(atomic_long_t *v)3858atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1598{3859{
1599 instrument_atomic_read_write(v, sizeof(*v));3860 instrument_atomic_read_write(v, sizeof(*v));
1600 return arch_atomic_long_fetch_dec_relaxed(v);3861 return raw_atomic_long_fetch_dec_relaxed(v);
1601}3862}
1602 3863
3864/**
3865 * atomic_long_and() - atomic bitwise AND with relaxed ordering
3866 * @i: long value
3867 * @v: pointer to atomic_long_t
3868 *
3869 * Atomically updates @v to (@v & @i) with relaxed ordering.
3870 *
3871 * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
3872 *
3873 * Return: Nothing.
3874 */
1603static __always_inline void3875static __always_inline void
1604atomic_long_and(long i, atomic_long_t *v)3876atomic_long_and(long i, atomic_long_t *v)
1605{3877{
1606 instrument_atomic_read_write(v, sizeof(*v));3878 instrument_atomic_read_write(v, sizeof(*v));
1607 arch_atomic_long_and(i, v);3879 raw_atomic_long_and(i, v);
1608}3880}
1609 3881
3882/**
3883 * atomic_long_fetch_and() - atomic bitwise AND with full ordering
3884 * @i: long value
3885 * @v: pointer to atomic_long_t
3886 *
3887 * Atomically updates @v to (@v & @i) with full ordering.
3888 *
3889 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
3890 *
3891 * Return: The original value of @v.
3892 */
1610static __always_inline long3893static __always_inline long
1611atomic_long_fetch_and(long i, atomic_long_t *v)3894atomic_long_fetch_and(long i, atomic_long_t *v)
1612{3895{
1613 kcsan_mb();3896 kcsan_mb();
1614 instrument_atomic_read_write(v, sizeof(*v));3897 instrument_atomic_read_write(v, sizeof(*v));
1615 return arch_atomic_long_fetch_and(i, v);3898 return raw_atomic_long_fetch_and(i, v);
1616}3899}
1617 3900
3901/**
3902 * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3903 * @i: long value
3904 * @v: pointer to atomic_long_t
3905 *
3906 * Atomically updates @v to (@v & @i) with acquire ordering.
3907 *
3908 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
3909 *
3910 * Return: The original value of @v.
3911 */
1618static __always_inline long3912static __always_inline long
1619atomic_long_fetch_and_acquire(long i, atomic_long_t *v)3913atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1620{3914{
1621 instrument_atomic_read_write(v, sizeof(*v));3915 instrument_atomic_read_write(v, sizeof(*v));
1622 return arch_atomic_long_fetch_and_acquire(i, v);3916 return raw_atomic_long_fetch_and_acquire(i, v);
1623}3917}
1624 3918
3919/**
3920 * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
3921 * @i: long value
3922 * @v: pointer to atomic_long_t
3923 *
3924 * Atomically updates @v to (@v & @i) with release ordering.
3925 *
3926 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
3927 *
3928 * Return: The original value of @v.
3929 */
1625static __always_inline long3930static __always_inline long
1626atomic_long_fetch_and_release(long i, atomic_long_t *v)3931atomic_long_fetch_and_release(long i, atomic_long_t *v)
1627{3932{
1628 kcsan_release();3933 kcsan_release();
1629 instrument_atomic_read_write(v, sizeof(*v));3934 instrument_atomic_read_write(v, sizeof(*v));
1630 return arch_atomic_long_fetch_and_release(i, v);3935 return raw_atomic_long_fetch_and_release(i, v);
1631}3936}
1632 3937
3938/**
3939 * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3940 * @i: long value
3941 * @v: pointer to atomic_long_t
3942 *
3943 * Atomically updates @v to (@v & @i) with relaxed ordering.
3944 *
3945 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
3946 *
3947 * Return: The original value of @v.
3948 */
1633static __always_inline long3949static __always_inline long
1634atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)3950atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1635{3951{
1636 instrument_atomic_read_write(v, sizeof(*v));3952 instrument_atomic_read_write(v, sizeof(*v));
1637 return arch_atomic_long_fetch_and_relaxed(i, v);3953 return raw_atomic_long_fetch_and_relaxed(i, v);
1638}3954}
1639 3955
3956/**
3957 * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
3958 * @i: long value
3959 * @v: pointer to atomic_long_t
3960 *
3961 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3962 *
3963 * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
3964 *
3965 * Return: Nothing.
3966 */
1640static __always_inline void3967static __always_inline void
1641atomic_long_andnot(long i, atomic_long_t *v)3968atomic_long_andnot(long i, atomic_long_t *v)
1642{3969{
1643 instrument_atomic_read_write(v, sizeof(*v));3970 instrument_atomic_read_write(v, sizeof(*v));
1644 arch_atomic_long_andnot(i, v);3971 raw_atomic_long_andnot(i, v);
1645}3972}
1646 3973
3974/**
3975 * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
3976 * @i: long value
3977 * @v: pointer to atomic_long_t
3978 *
3979 * Atomically updates @v to (@v & ~@i) with full ordering.
3980 *
3981 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
3982 *
3983 * Return: The original value of @v.
3984 */
1647static __always_inline long3985static __always_inline long
1648atomic_long_fetch_andnot(long i, atomic_long_t *v)3986atomic_long_fetch_andnot(long i, atomic_long_t *v)
1649{3987{
1650 kcsan_mb();3988 kcsan_mb();
1651 instrument_atomic_read_write(v, sizeof(*v));3989 instrument_atomic_read_write(v, sizeof(*v));
1652 return arch_atomic_long_fetch_andnot(i, v);3990 return raw_atomic_long_fetch_andnot(i, v);
1653}3991}
1654 3992
3993/**
3994 * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3995 * @i: long value
3996 * @v: pointer to atomic_long_t
3997 *
3998 * Atomically updates @v to (@v & ~@i) with acquire ordering.
3999 *
4000 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
4001 *
4002 * Return: The original value of @v.
4003 */
1655static __always_inline long4004static __always_inline long
1656atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)4005atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1657{4006{
1658 instrument_atomic_read_write(v, sizeof(*v));4007 instrument_atomic_read_write(v, sizeof(*v));
1659 return arch_atomic_long_fetch_andnot_acquire(i, v);4008 return raw_atomic_long_fetch_andnot_acquire(i, v);
1660}4009}
1661 4010
4011/**
4012 * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
4013 * @i: long value
4014 * @v: pointer to atomic_long_t
4015 *
4016 * Atomically updates @v to (@v & ~@i) with release ordering.
4017 *
4018 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
4019 *
4020 * Return: The original value of @v.
4021 */
1662static __always_inline long4022static __always_inline long
1663atomic_long_fetch_andnot_release(long i, atomic_long_t *v)4023atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1664{4024{
1665 kcsan_release();4025 kcsan_release();
1666 instrument_atomic_read_write(v, sizeof(*v));4026 instrument_atomic_read_write(v, sizeof(*v));
1667 return arch_atomic_long_fetch_andnot_release(i, v);4027 return raw_atomic_long_fetch_andnot_release(i, v);
1668}4028}
1669 4029
4030/**
4031 * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
4032 * @i: long value
4033 * @v: pointer to atomic_long_t
4034 *
4035 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
4036 *
4037 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
4038 *
4039 * Return: The original value of @v.
4040 */
1670static __always_inline long4041static __always_inline long
1671atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)4042atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1672{4043{
1673 instrument_atomic_read_write(v, sizeof(*v));4044 instrument_atomic_read_write(v, sizeof(*v));
1674 return arch_atomic_long_fetch_andnot_relaxed(i, v);4045 return raw_atomic_long_fetch_andnot_relaxed(i, v);
1675}4046}
1676 4047
4048/**
4049 * atomic_long_or() - atomic bitwise OR with relaxed ordering
4050 * @i: long value
4051 * @v: pointer to atomic_long_t
4052 *
4053 * Atomically updates @v to (@v | @i) with relaxed ordering.
4054 *
4055 * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
4056 *
4057 * Return: Nothing.
4058 */
1677static __always_inline void4059static __always_inline void
1678atomic_long_or(long i, atomic_long_t *v)4060atomic_long_or(long i, atomic_long_t *v)
1679{4061{
1680 instrument_atomic_read_write(v, sizeof(*v));4062 instrument_atomic_read_write(v, sizeof(*v));
1681 arch_atomic_long_or(i, v);4063 raw_atomic_long_or(i, v);
1682}4064}
1683 4065
4066/**
4067 * atomic_long_fetch_or() - atomic bitwise OR with full ordering
4068 * @i: long value
4069 * @v: pointer to atomic_long_t
4070 *
4071 * Atomically updates @v to (@v | @i) with full ordering.
4072 *
4073 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
4074 *
4075 * Return: The original value of @v.
4076 */
1684static __always_inline long4077static __always_inline long
1685atomic_long_fetch_or(long i, atomic_long_t *v)4078atomic_long_fetch_or(long i, atomic_long_t *v)
1686{4079{
1687 kcsan_mb();4080 kcsan_mb();
1688 instrument_atomic_read_write(v, sizeof(*v));4081 instrument_atomic_read_write(v, sizeof(*v));
1689 return arch_atomic_long_fetch_or(i, v);4082 return raw_atomic_long_fetch_or(i, v);
1690}4083}
1691 4084
4085/**
4086 * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
4087 * @i: long value
4088 * @v: pointer to atomic_long_t
4089 *
4090 * Atomically updates @v to (@v | @i) with acquire ordering.
4091 *
4092 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
4093 *
4094 * Return: The original value of @v.
4095 */
1692static __always_inline long4096static __always_inline long
1693atomic_long_fetch_or_acquire(long i, atomic_long_t *v)4097atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1694{4098{
1695 instrument_atomic_read_write(v, sizeof(*v));4099 instrument_atomic_read_write(v, sizeof(*v));
1696 return arch_atomic_long_fetch_or_acquire(i, v);4100 return raw_atomic_long_fetch_or_acquire(i, v);
1697}4101}
1698 4102
4103/**
4104 * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
4105 * @i: long value
4106 * @v: pointer to atomic_long_t
4107 *
4108 * Atomically updates @v to (@v | @i) with release ordering.
4109 *
4110 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
4111 *
4112 * Return: The original value of @v.
4113 */
1699static __always_inline long4114static __always_inline long
1700atomic_long_fetch_or_release(long i, atomic_long_t *v)4115atomic_long_fetch_or_release(long i, atomic_long_t *v)
1701{4116{
1702 kcsan_release();4117 kcsan_release();
1703 instrument_atomic_read_write(v, sizeof(*v));4118 instrument_atomic_read_write(v, sizeof(*v));
1704 return arch_atomic_long_fetch_or_release(i, v);4119 return raw_atomic_long_fetch_or_release(i, v);
1705}4120}
1706 4121
4122/**
4123 * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
4124 * @i: long value
4125 * @v: pointer to atomic_long_t
4126 *
4127 * Atomically updates @v to (@v | @i) with relaxed ordering.
4128 *
4129 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
4130 *
4131 * Return: The original value of @v.
4132 */
1707static __always_inline long4133static __always_inline long
1708atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)4134atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1709{4135{
1710 instrument_atomic_read_write(v, sizeof(*v));4136 instrument_atomic_read_write(v, sizeof(*v));
1711 return arch_atomic_long_fetch_or_relaxed(i, v);4137 return raw_atomic_long_fetch_or_relaxed(i, v);
1712}4138}
1713 4139
4140/**
4141 * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
4142 * @i: long value
4143 * @v: pointer to atomic_long_t
4144 *
4145 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4146 *
4147 * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
4148 *
4149 * Return: Nothing.
4150 */
1714static __always_inline void4151static __always_inline void
1715atomic_long_xor(long i, atomic_long_t *v)4152atomic_long_xor(long i, atomic_long_t *v)
1716{4153{
1717 instrument_atomic_read_write(v, sizeof(*v));4154 instrument_atomic_read_write(v, sizeof(*v));
1718 arch_atomic_long_xor(i, v);4155 raw_atomic_long_xor(i, v);
1719}4156}
1720 4157
4158/**
4159 * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
4160 * @i: long value
4161 * @v: pointer to atomic_long_t
4162 *
4163 * Atomically updates @v to (@v ^ @i) with full ordering.
4164 *
4165 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
4166 *
4167 * Return: The original value of @v.
4168 */
1721static __always_inline long4169static __always_inline long
1722atomic_long_fetch_xor(long i, atomic_long_t *v)4170atomic_long_fetch_xor(long i, atomic_long_t *v)
1723{4171{
1724 kcsan_mb();4172 kcsan_mb();
1725 instrument_atomic_read_write(v, sizeof(*v));4173 instrument_atomic_read_write(v, sizeof(*v));
1726 return arch_atomic_long_fetch_xor(i, v);4174 return raw_atomic_long_fetch_xor(i, v);
1727}4175}
1728 4176
4177/**
4178 * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
4179 * @i: long value
4180 * @v: pointer to atomic_long_t
4181 *
4182 * Atomically updates @v to (@v ^ @i) with acquire ordering.
4183 *
4184 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
4185 *
4186 * Return: The original value of @v.
4187 */
1729static __always_inline long4188static __always_inline long
1730atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)4189atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1731{4190{
1732 instrument_atomic_read_write(v, sizeof(*v));4191 instrument_atomic_read_write(v, sizeof(*v));
1733 return arch_atomic_long_fetch_xor_acquire(i, v);4192 return raw_atomic_long_fetch_xor_acquire(i, v);
1734}4193}
1735 4194
4195/**
4196 * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
4197 * @i: long value
4198 * @v: pointer to atomic_long_t
4199 *
4200 * Atomically updates @v to (@v ^ @i) with release ordering.
4201 *
4202 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
4203 *
4204 * Return: The original value of @v.
4205 */
1736static __always_inline long4206static __always_inline long
1737atomic_long_fetch_xor_release(long i, atomic_long_t *v)4207atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1738{4208{
1739 kcsan_release();4209 kcsan_release();
1740 instrument_atomic_read_write(v, sizeof(*v));4210 instrument_atomic_read_write(v, sizeof(*v));
1741 return arch_atomic_long_fetch_xor_release(i, v);4211 return raw_atomic_long_fetch_xor_release(i, v);
1742}4212}
1743 4213
4214/**
4215 * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
4216 * @i: long value
4217 * @v: pointer to atomic_long_t
4218 *
4219 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4220 *
4221 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
4222 *
4223 * Return: The original value of @v.
4224 */
1744static __always_inline long4225static __always_inline long
1745atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)4226atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1746{4227{
1747 instrument_atomic_read_write(v, sizeof(*v));4228 instrument_atomic_read_write(v, sizeof(*v));
1748 return arch_atomic_long_fetch_xor_relaxed(i, v);4229 return raw_atomic_long_fetch_xor_relaxed(i, v);
1749}4230}
1750 4231
4232/**
4233 * atomic_long_xchg() - atomic exchange with full ordering
4234 * @v: pointer to atomic_long_t
4235 * @new: long value to assign
4236 *
4237 * Atomically updates @v to @new with full ordering.
4238 *
4239 * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
4240 *
4241 * Return: The original value of @v.
4242 */
1751static __always_inline long4243static __always_inline long
1752atomic_long_xchg(atomic_long_t *v, long i)4244atomic_long_xchg(atomic_long_t *v, long new)
1753{4245{
1754 kcsan_mb();4246 kcsan_mb();
1755 instrument_atomic_read_write(v, sizeof(*v));4247 instrument_atomic_read_write(v, sizeof(*v));
1756 return arch_atomic_long_xchg(v, i);4248 return raw_atomic_long_xchg(v, new);
1757}4249}
1758 4250
4251/**
4252 * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
4253 * @v: pointer to atomic_long_t
4254 * @new: long value to assign
4255 *
4256 * Atomically updates @v to @new with acquire ordering.
4257 *
4258 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
4259 *
4260 * Return: The original value of @v.
4261 */
1759static __always_inline long4262static __always_inline long
1760atomic_long_xchg_acquire(atomic_long_t *v, long i)4263atomic_long_xchg_acquire(atomic_long_t *v, long new)
1761{4264{
1762 instrument_atomic_read_write(v, sizeof(*v));4265 instrument_atomic_read_write(v, sizeof(*v));
1763 return arch_atomic_long_xchg_acquire(v, i);4266 return raw_atomic_long_xchg_acquire(v, new);
1764}4267}
1765 4268
4269/**
4270 * atomic_long_xchg_release() - atomic exchange with release ordering
4271 * @v: pointer to atomic_long_t
4272 * @new: long value to assign
4273 *
4274 * Atomically updates @v to @new with release ordering.
4275 *
4276 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
4277 *
4278 * Return: The original value of @v.
4279 */
1766static __always_inline long4280static __always_inline long
1767atomic_long_xchg_release(atomic_long_t *v, long i)4281atomic_long_xchg_release(atomic_long_t *v, long new)
1768{4282{
1769 kcsan_release();4283 kcsan_release();
1770 instrument_atomic_read_write(v, sizeof(*v));4284 instrument_atomic_read_write(v, sizeof(*v));
1771 return arch_atomic_long_xchg_release(v, i);4285 return raw_atomic_long_xchg_release(v, new);
1772}4286}
1773 4287
4288/**
4289 * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
4290 * @v: pointer to atomic_long_t
4291 * @new: long value to assign
4292 *
4293 * Atomically updates @v to @new with relaxed ordering.
4294 *
4295 * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
4296 *
4297 * Return: The original value of @v.
4298 */
1774static __always_inline long4299static __always_inline long
1775atomic_long_xchg_relaxed(atomic_long_t *v, long i)4300atomic_long_xchg_relaxed(atomic_long_t *v, long new)
1776{4301{
1777 instrument_atomic_read_write(v, sizeof(*v));4302 instrument_atomic_read_write(v, sizeof(*v));
1778 return arch_atomic_long_xchg_relaxed(v, i);4303 return raw_atomic_long_xchg_relaxed(v, new);
1779}4304}
1780 4305
4306/**
4307 * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
4308 * @v: pointer to atomic_long_t
4309 * @old: long value to compare with
4310 * @new: long value to assign
4311 *
4312 * If (@v == @old), atomically updates @v to @new with full ordering.
4313 *
4314 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
4315 *
4316 * Return: The original value of @v.
4317 */
1781static __always_inline long4318static __always_inline long
1782atomic_long_cmpxchg(atomic_long_t *v, long old, long new)4319atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1783{4320{
1784 kcsan_mb();4321 kcsan_mb();
1785 instrument_atomic_read_write(v, sizeof(*v));4322 instrument_atomic_read_write(v, sizeof(*v));
1786 return arch_atomic_long_cmpxchg(v, old, new);4323 return raw_atomic_long_cmpxchg(v, old, new);
1787}4324}
1788 4325
4326/**
4327 * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4328 * @v: pointer to atomic_long_t
4329 * @old: long value to compare with
4330 * @new: long value to assign
4331 *
4332 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4333 *
4334 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
4335 *
4336 * Return: The original value of @v.
4337 */
1789static __always_inline long4338static __always_inline long
1790atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)4339atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1791{4340{
1792 instrument_atomic_read_write(v, sizeof(*v));4341 instrument_atomic_read_write(v, sizeof(*v));
1793 return arch_atomic_long_cmpxchg_acquire(v, old, new);4342 return raw_atomic_long_cmpxchg_acquire(v, old, new);
1794}4343}
1795 4344
4345/**
4346 * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
4347 * @v: pointer to atomic_long_t
4348 * @old: long value to compare with
4349 * @new: long value to assign
4350 *
4351 * If (@v == @old), atomically updates @v to @new with release ordering.
4352 *
4353 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
4354 *
4355 * Return: The original value of @v.
4356 */
1796static __always_inline long4357static __always_inline long
1797atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)4358atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1798{4359{
1799 kcsan_release();4360 kcsan_release();
1800 instrument_atomic_read_write(v, sizeof(*v));4361 instrument_atomic_read_write(v, sizeof(*v));
1801 return arch_atomic_long_cmpxchg_release(v, old, new);4362 return raw_atomic_long_cmpxchg_release(v, old, new);
1802}4363}
1803 4364
4365/**
4366 * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4367 * @v: pointer to atomic_long_t
4368 * @old: long value to compare with
4369 * @new: long value to assign
4370 *
4371 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4372 *
4373 * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
4374 *
4375 * Return: The original value of @v.
4376 */
1804static __always_inline long4377static __always_inline long
1805atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)4378atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1806{4379{
1807 instrument_atomic_read_write(v, sizeof(*v));4380 instrument_atomic_read_write(v, sizeof(*v));
1808 return arch_atomic_long_cmpxchg_relaxed(v, old, new);4381 return raw_atomic_long_cmpxchg_relaxed(v, old, new);
1809}4382}
1810 4383
4384/**
4385 * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
4386 * @v: pointer to atomic_long_t
4387 * @old: pointer to long value to compare with
4388 * @new: long value to assign
4389 *
4390 * If (@v == @old), atomically updates @v to @new with full ordering.
4391 * Otherwise, updates @old to the current value of @v.
4392 *
4393 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
4394 *
4395 * Return: @true if the exchange occured, @false otherwise.
4396 */
1811static __always_inline bool4397static __always_inline bool
1812atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)4398atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1813{4399{
1814 kcsan_mb();4400 kcsan_mb();
1815 instrument_atomic_read_write(v, sizeof(*v));4401 instrument_atomic_read_write(v, sizeof(*v));
1816 instrument_atomic_read_write(old, sizeof(*old));4402 instrument_atomic_read_write(old, sizeof(*old));
1817 return arch_atomic_long_try_cmpxchg(v, old, new);4403 return raw_atomic_long_try_cmpxchg(v, old, new);
1818}4404}
1819 4405
4406/**
4407 * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4408 * @v: pointer to atomic_long_t
4409 * @old: pointer to long value to compare with
4410 * @new: long value to assign
4411 *
4412 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4413 * Otherwise, updates @old to the current value of @v.
4414 *
4415 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
4416 *
4417 * Return: @true if the exchange occured, @false otherwise.
4418 */
1820static __always_inline bool4419static __always_inline bool
1821atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)4420atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1822{4421{
1823 instrument_atomic_read_write(v, sizeof(*v));4422 instrument_atomic_read_write(v, sizeof(*v));
1824 instrument_atomic_read_write(old, sizeof(*old));4423 instrument_atomic_read_write(old, sizeof(*old));
1825 return arch_atomic_long_try_cmpxchg_acquire(v, old, new);4424 return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
1826}4425}
1827 4426
4427/**
4428 * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
4429 * @v: pointer to atomic_long_t
4430 * @old: pointer to long value to compare with
4431 * @new: long value to assign
4432 *
4433 * If (@v == @old), atomically updates @v to @new with release ordering.
4434 * Otherwise, updates @old to the current value of @v.
4435 *
4436 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
4437 *
4438 * Return: @true if the exchange occured, @false otherwise.
4439 */
1828static __always_inline bool4440static __always_inline bool
1829atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)4441atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1830{4442{
1831 kcsan_release();4443 kcsan_release();
1832 instrument_atomic_read_write(v, sizeof(*v));4444 instrument_atomic_read_write(v, sizeof(*v));
1833 instrument_atomic_read_write(old, sizeof(*old));4445 instrument_atomic_read_write(old, sizeof(*old));
1834 return arch_atomic_long_try_cmpxchg_release(v, old, new);4446 return raw_atomic_long_try_cmpxchg_release(v, old, new);
1835}4447}
1836 4448
4449/**
4450 * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4451 * @v: pointer to atomic_long_t
4452 * @old: pointer to long value to compare with
4453 * @new: long value to assign
4454 *
4455 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4456 * Otherwise, updates @old to the current value of @v.
4457 *
4458 * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
4459 *
4460 * Return: @true if the exchange occured, @false otherwise.
4461 */
1837static __always_inline bool4462static __always_inline bool
1838atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)4463atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1839{4464{
1840 instrument_atomic_read_write(v, sizeof(*v));4465 instrument_atomic_read_write(v, sizeof(*v));
1841 instrument_atomic_read_write(old, sizeof(*old));4466 instrument_atomic_read_write(old, sizeof(*old));
1842 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);4467 return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
1843}4468}
1844 4469
4470/**
4471 * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
4472 * @i: long value to add
4473 * @v: pointer to atomic_long_t
4474 *
4475 * Atomically updates @v to (@v - @i) with full ordering.
4476 *
4477 * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
4478 *
4479 * Return: @true if the resulting value of @v is zero, @false otherwise.
4480 */
1845static __always_inline bool4481static __always_inline bool
1846atomic_long_sub_and_test(long i, atomic_long_t *v)4482atomic_long_sub_and_test(long i, atomic_long_t *v)
1847{4483{
1848 kcsan_mb();4484 kcsan_mb();
1849 instrument_atomic_read_write(v, sizeof(*v));4485 instrument_atomic_read_write(v, sizeof(*v));
1850 return arch_atomic_long_sub_and_test(i, v);4486 return raw_atomic_long_sub_and_test(i, v);
1851}4487}
1852 4488
4489/**
4490 * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
4491 * @v: pointer to atomic_long_t
4492 *
4493 * Atomically updates @v to (@v - 1) with full ordering.
4494 *
4495 * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
4496 *
4497 * Return: @true if the resulting value of @v is zero, @false otherwise.
4498 */
1853static __always_inline bool4499static __always_inline bool
1854atomic_long_dec_and_test(atomic_long_t *v)4500atomic_long_dec_and_test(atomic_long_t *v)
1855{4501{
1856 kcsan_mb();4502 kcsan_mb();
1857 instrument_atomic_read_write(v, sizeof(*v));4503 instrument_atomic_read_write(v, sizeof(*v));
1858 return arch_atomic_long_dec_and_test(v);4504 return raw_atomic_long_dec_and_test(v);
1859}4505}
1860 4506
4507/**
4508 * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
4509 * @v: pointer to atomic_long_t
4510 *
4511 * Atomically updates @v to (@v + 1) with full ordering.
4512 *
4513 * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
4514 *
4515 * Return: @true if the resulting value of @v is zero, @false otherwise.
4516 */
1861static __always_inline bool4517static __always_inline bool
1862atomic_long_inc_and_test(atomic_long_t *v)4518atomic_long_inc_and_test(atomic_long_t *v)
1863{4519{
1864 kcsan_mb();4520 kcsan_mb();
1865 instrument_atomic_read_write(v, sizeof(*v));4521 instrument_atomic_read_write(v, sizeof(*v));
1866 return arch_atomic_long_inc_and_test(v);4522 return raw_atomic_long_inc_and_test(v);
1867}4523}
1868 4524
4525/**
4526 * atomic_long_add_negative() - atomic add and test if negative with full ordering
4527 * @i: long value to add
4528 * @v: pointer to atomic_long_t
4529 *
4530 * Atomically updates @v to (@v + @i) with full ordering.
4531 *
4532 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
4533 *
4534 * Return: @true if the resulting value of @v is negative, @false otherwise.
4535 */
1869static __always_inline bool4536static __always_inline bool
1870atomic_long_add_negative(long i, atomic_long_t *v)4537atomic_long_add_negative(long i, atomic_long_t *v)
1871{4538{
1872 kcsan_mb();4539 kcsan_mb();
1873 instrument_atomic_read_write(v, sizeof(*v));4540 instrument_atomic_read_write(v, sizeof(*v));
1874 return arch_atomic_long_add_negative(i, v);4541 return raw_atomic_long_add_negative(i, v);
1875}4542}
1876 4543
4544/**
4545 * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
4546 * @i: long value to add
4547 * @v: pointer to atomic_long_t
4548 *
4549 * Atomically updates @v to (@v + @i) with acquire ordering.
4550 *
4551 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
4552 *
4553 * Return: @true if the resulting value of @v is negative, @false otherwise.
4554 */
1877static __always_inline bool4555static __always_inline bool
1878atomic_long_add_negative_acquire(long i, atomic_long_t *v)4556atomic_long_add_negative_acquire(long i, atomic_long_t *v)
1879{4557{
1880 instrument_atomic_read_write(v, sizeof(*v));4558 instrument_atomic_read_write(v, sizeof(*v));
1881 return arch_atomic_long_add_negative_acquire(i, v);4559 return raw_atomic_long_add_negative_acquire(i, v);
1882}4560}
1883 4561
4562/**
4563 * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
4564 * @i: long value to add
4565 * @v: pointer to atomic_long_t
4566 *
4567 * Atomically updates @v to (@v + @i) with release ordering.
4568 *
4569 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
4570 *
4571 * Return: @true if the resulting value of @v is negative, @false otherwise.
4572 */
1884static __always_inline bool4573static __always_inline bool
1885atomic_long_add_negative_release(long i, atomic_long_t *v)4574atomic_long_add_negative_release(long i, atomic_long_t *v)
1886{4575{
1887 kcsan_release();4576 kcsan_release();
1888 instrument_atomic_read_write(v, sizeof(*v));4577 instrument_atomic_read_write(v, sizeof(*v));
1889 return arch_atomic_long_add_negative_release(i, v);4578 return raw_atomic_long_add_negative_release(i, v);
1890}4579}
1891 4580
4581/**
4582 * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4583 * @i: long value to add
4584 * @v: pointer to atomic_long_t
4585 *
4586 * Atomically updates @v to (@v + @i) with relaxed ordering.
4587 *
4588 * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
4589 *
4590 * Return: @true if the resulting value of @v is negative, @false otherwise.
4591 */
1892static __always_inline bool4592static __always_inline bool
1893atomic_long_add_negative_relaxed(long i, atomic_long_t *v)4593atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1894{4594{
1895 instrument_atomic_read_write(v, sizeof(*v));4595 instrument_atomic_read_write(v, sizeof(*v));
1896 return arch_atomic_long_add_negative_relaxed(i, v);4596 return raw_atomic_long_add_negative_relaxed(i, v);
1897}4597}
1898 4598
4599/**
4600 * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
4601 * @v: pointer to atomic_long_t
4602 * @a: long value to add
4603 * @u: long value to compare with
4604 *
4605 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4606 *
4607 * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
4608 *
4609 * Return: The original value of @v.
4610 */
1899static __always_inline long4611static __always_inline long
1900atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)4612atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1901{4613{
1902 kcsan_mb();4614 kcsan_mb();
1903 instrument_atomic_read_write(v, sizeof(*v));4615 instrument_atomic_read_write(v, sizeof(*v));
1904 return arch_atomic_long_fetch_add_unless(v, a, u);4616 return raw_atomic_long_fetch_add_unless(v, a, u);
1905}4617}
1906 4618
4619/**
4620 * atomic_long_add_unless() - atomic add unless value with full ordering
4621 * @v: pointer to atomic_long_t
4622 * @a: long value to add
4623 * @u: long value to compare with
4624 *
4625 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4626 *
4627 * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
4628 *
4629 * Return: @true if @v was updated, @false otherwise.
4630 */
1907static __always_inline bool4631static __always_inline bool
1908atomic_long_add_unless(atomic_long_t *v, long a, long u)4632atomic_long_add_unless(atomic_long_t *v, long a, long u)
1909{4633{
1910 kcsan_mb();4634 kcsan_mb();
1911 instrument_atomic_read_write(v, sizeof(*v));4635 instrument_atomic_read_write(v, sizeof(*v));
1912 return arch_atomic_long_add_unless(v, a, u);4636 return raw_atomic_long_add_unless(v, a, u);
1913}4637}
1914 4638
4639/**
4640 * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
4641 * @v: pointer to atomic_long_t
4642 *
4643 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4644 *
4645 * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
4646 *
4647 * Return: @true if @v was updated, @false otherwise.
4648 */
1915static __always_inline bool4649static __always_inline bool
1916atomic_long_inc_not_zero(atomic_long_t *v)4650atomic_long_inc_not_zero(atomic_long_t *v)
1917{4651{
1918 kcsan_mb();4652 kcsan_mb();
1919 instrument_atomic_read_write(v, sizeof(*v));4653 instrument_atomic_read_write(v, sizeof(*v));
1920 return arch_atomic_long_inc_not_zero(v);4654 return raw_atomic_long_inc_not_zero(v);
1921}4655}
1922 4656
4657/**
4658 * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
4659 * @v: pointer to atomic_long_t
4660 *
4661 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4662 *
4663 * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
4664 *
4665 * Return: @true if @v was updated, @false otherwise.
4666 */
1923static __always_inline bool4667static __always_inline bool
1924atomic_long_inc_unless_negative(atomic_long_t *v)4668atomic_long_inc_unless_negative(atomic_long_t *v)
1925{4669{
1926 kcsan_mb();4670 kcsan_mb();
1927 instrument_atomic_read_write(v, sizeof(*v));4671 instrument_atomic_read_write(v, sizeof(*v));
1928 return arch_atomic_long_inc_unless_negative(v);4672 return raw_atomic_long_inc_unless_negative(v);
1929}4673}
1930 4674
4675/**
4676 * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
4677 * @v: pointer to atomic_long_t
4678 *
4679 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4680 *
4681 * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.

347 more lines aren’t shown here.

include/linux/atomic/atomic-long.h+1435−687
@@ -21,1030 +21,1778 @@ typedef atomic_t atomic_long_t; · 20 unmodified lines
21#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed21#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
22#endif22#endif
23 23
24#ifdef CONFIG_64BIT24/**
25 25 * raw_atomic_long_read() - atomic load with relaxed ordering
26static __always_inline long26 * @v: pointer to atomic_long_t
27arch_atomic_long_read(const atomic_long_t *v)27 *
28{28 * Atomically loads the value of @v with relaxed ordering.
29 return arch_atomic64_read(v);29 *
30}30 * Safe to use in noinstr code; prefer atomic_long_read() elsewhere.
31 31 *
32static __always_inline long32 * Return: The value loaded from @v.
33arch_atomic_long_read_acquire(const atomic_long_t *v)33 */
34{34static __always_inline long
35 return arch_atomic64_read_acquire(v);35raw_atomic_long_read(const atomic_long_t *v)
36}
37
38static __always_inline void
39arch_atomic_long_set(atomic_long_t *v, long i)
40{
41 arch_atomic64_set(v, i);
42}
43
44static __always_inline void
45arch_atomic_long_set_release(atomic_long_t *v, long i)
46{
47 arch_atomic64_set_release(v, i);
48}
49
50static __always_inline void
51arch_atomic_long_add(long i, atomic_long_t *v)
52{
53 arch_atomic64_add(i, v);
54}
55
56static __always_inline long
57arch_atomic_long_add_return(long i, atomic_long_t *v)
58{
59 return arch_atomic64_add_return(i, v);
60}
61
62static __always_inline long
63arch_atomic_long_add_return_acquire(long i, atomic_long_t *v)
64{
65 return arch_atomic64_add_return_acquire(i, v);
66}
67
68static __always_inline long
69arch_atomic_long_add_return_release(long i, atomic_long_t *v)
70{
71 return arch_atomic64_add_return_release(i, v);
72}
73
74static __always_inline long
75arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
76{
77 return arch_atomic64_add_return_relaxed(i, v);
78}
79
80static __always_inline long
81arch_atomic_long_fetch_add(long i, atomic_long_t *v)
82{
83 return arch_atomic64_fetch_add(i, v);
84}
85
86static __always_inline long
87arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
88{
89 return arch_atomic64_fetch_add_acquire(i, v);
90}
91
92static __always_inline long
93arch_atomic_long_fetch_add_release(long i, atomic_long_t *v)
94{
95 return arch_atomic64_fetch_add_release(i, v);
96}
97
98static __always_inline long
99arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
100{
101 return arch_atomic64_fetch_add_relaxed(i, v);
102}
103
104static __always_inline void
105arch_atomic_long_sub(long i, atomic_long_t *v)
106{
107 arch_atomic64_sub(i, v);
108}
109
110static __always_inline long
111arch_atomic_long_sub_return(long i, atomic_long_t *v)
112{
113 return arch_atomic64_sub_return(i, v);
114}
115
116static __always_inline long
117arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
118{
119 return arch_atomic64_sub_return_acquire(i, v);
120}
121
122static __always_inline long
123arch_atomic_long_sub_return_release(long i, atomic_long_t *v)
124{
125 return arch_atomic64_sub_return_release(i, v);
126}
127
128static __always_inline long
129arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
130{
131 return arch_atomic64_sub_return_relaxed(i, v);
132}
133
134static __always_inline long
135arch_atomic_long_fetch_sub(long i, atomic_long_t *v)
136{
137 return arch_atomic64_fetch_sub(i, v);
138}
139
140static __always_inline long
141arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
142{
143 return arch_atomic64_fetch_sub_acquire(i, v);
144}
145
146static __always_inline long
147arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
148{
149 return arch_atomic64_fetch_sub_release(i, v);
150}
151
152static __always_inline long
153arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
154{
155 return arch_atomic64_fetch_sub_relaxed(i, v);
156}
157
158static __always_inline void
159arch_atomic_long_inc(atomic_long_t *v)
160{
161 arch_atomic64_inc(v);
162}
163
164static __always_inline long
165arch_atomic_long_inc_return(atomic_long_t *v)
166{
167 return arch_atomic64_inc_return(v);
168}
169
170static __always_inline long
171arch_atomic_long_inc_return_acquire(atomic_long_t *v)
172{
173 return arch_atomic64_inc_return_acquire(v);
174}
175
176static __always_inline long
177arch_atomic_long_inc_return_release(atomic_long_t *v)
178{
179 return arch_atomic64_inc_return_release(v);
180}
181
182static __always_inline long
183arch_atomic_long_inc_return_relaxed(atomic_long_t *v)
184{
185 return arch_atomic64_inc_return_relaxed(v);
186}
187
188static __always_inline long
189arch_atomic_long_fetch_inc(atomic_long_t *v)
190{
191 return arch_atomic64_fetch_inc(v);
192}
193
194static __always_inline long
195arch_atomic_long_fetch_inc_acquire(atomic_long_t *v)
196{
197 return arch_atomic64_fetch_inc_acquire(v);
198}
199
200static __always_inline long
201arch_atomic_long_fetch_inc_release(atomic_long_t *v)
202{
203 return arch_atomic64_fetch_inc_release(v);
204}
205
206static __always_inline long
207arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
208{
209 return arch_atomic64_fetch_inc_relaxed(v);
210}
211
212static __always_inline void
213arch_atomic_long_dec(atomic_long_t *v)
214{
215 arch_atomic64_dec(v);
216}
217
218static __always_inline long
219arch_atomic_long_dec_return(atomic_long_t *v)
220{
221 return arch_atomic64_dec_return(v);
222}
223
224static __always_inline long
225arch_atomic_long_dec_return_acquire(atomic_long_t *v)
226{
227 return arch_atomic64_dec_return_acquire(v);
228}
229
230static __always_inline long
231arch_atomic_long_dec_return_release(atomic_long_t *v)
232{
233 return arch_atomic64_dec_return_release(v);
234}
235
236static __always_inline long
237arch_atomic_long_dec_return_relaxed(atomic_long_t *v)
238{
239 return arch_atomic64_dec_return_relaxed(v);
240}
241
242static __always_inline long
243arch_atomic_long_fetch_dec(atomic_long_t *v)
244{
245 return arch_atomic64_fetch_dec(v);
246}
247
248static __always_inline long
249arch_atomic_long_fetch_dec_acquire(atomic_long_t *v)
250{
251 return arch_atomic64_fetch_dec_acquire(v);
252}
253
254static __always_inline long
255arch_atomic_long_fetch_dec_release(atomic_long_t *v)
256{
257 return arch_atomic64_fetch_dec_release(v);
258}
259
260static __always_inline long
261arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
262{
263 return arch_atomic64_fetch_dec_relaxed(v);
264}
265
266static __always_inline void
267arch_atomic_long_and(long i, atomic_long_t *v)
268{
269 arch_atomic64_and(i, v);
270}
271
272static __always_inline long
273arch_atomic_long_fetch_and(long i, atomic_long_t *v)
274{
275 return arch_atomic64_fetch_and(i, v);
276}
277
278static __always_inline long
279arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
280{
281 return arch_atomic64_fetch_and_acquire(i, v);
282}
283
284static __always_inline long
285arch_atomic_long_fetch_and_release(long i, atomic_long_t *v)
286{
287 return arch_atomic64_fetch_and_release(i, v);
288}
289
290static __always_inline long
291arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
292{
293 return arch_atomic64_fetch_and_relaxed(i, v);
294}
295
296static __always_inline void
297arch_atomic_long_andnot(long i, atomic_long_t *v)
298{
299 arch_atomic64_andnot(i, v);
300}
301
302static __always_inline long
303arch_atomic_long_fetch_andnot(long i, atomic_long_t *v)
304{
305 return arch_atomic64_fetch_andnot(i, v);
306}
307
308static __always_inline long
309arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
310{
311 return arch_atomic64_fetch_andnot_acquire(i, v);
312}
313
314static __always_inline long
315arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
316{
317 return arch_atomic64_fetch_andnot_release(i, v);
318}
319
320static __always_inline long
321arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
322{
323 return arch_atomic64_fetch_andnot_relaxed(i, v);
324}
325
326static __always_inline void
327arch_atomic_long_or(long i, atomic_long_t *v)
328{
329 arch_atomic64_or(i, v);
330}
331
332static __always_inline long
333arch_atomic_long_fetch_or(long i, atomic_long_t *v)
334{
335 return arch_atomic64_fetch_or(i, v);
336}
337
338static __always_inline long
339arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
340{
341 return arch_atomic64_fetch_or_acquire(i, v);
342}
343
344static __always_inline long
345arch_atomic_long_fetch_or_release(long i, atomic_long_t *v)
346{
347 return arch_atomic64_fetch_or_release(i, v);
348}
349
350static __always_inline long
351arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
352{
353 return arch_atomic64_fetch_or_relaxed(i, v);
354}
355
356static __always_inline void
357arch_atomic_long_xor(long i, atomic_long_t *v)
358{
359 arch_atomic64_xor(i, v);
360}
361
362static __always_inline long
363arch_atomic_long_fetch_xor(long i, atomic_long_t *v)
364{
365 return arch_atomic64_fetch_xor(i, v);
366}
367
368static __always_inline long
369arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
370{
371 return arch_atomic64_fetch_xor_acquire(i, v);
372}
373
374static __always_inline long
375arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
376{
377 return arch_atomic64_fetch_xor_release(i, v);
378}
379
380static __always_inline long
381arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
382{
383 return arch_atomic64_fetch_xor_relaxed(i, v);
384}
385
386static __always_inline long
387arch_atomic_long_xchg(atomic_long_t *v, long i)
388{
389 return arch_atomic64_xchg(v, i);
390}
391
392static __always_inline long
393arch_atomic_long_xchg_acquire(atomic_long_t *v, long i)
394{
395 return arch_atomic64_xchg_acquire(v, i);
396}
397
398static __always_inline long
399arch_atomic_long_xchg_release(atomic_long_t *v, long i)
400{
401 return arch_atomic64_xchg_release(v, i);
402}
403
404static __always_inline long
405arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i)
406{
407 return arch_atomic64_xchg_relaxed(v, i);
408}
409
410static __always_inline long
411arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
412{
413 return arch_atomic64_cmpxchg(v, old, new);
414}
415
416static __always_inline long
417arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
418{
419 return arch_atomic64_cmpxchg_acquire(v, old, new);
420}
421
422static __always_inline long
423arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
424{
425 return arch_atomic64_cmpxchg_release(v, old, new);
426}
427
428static __always_inline long
429arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
430{
431 return arch_atomic64_cmpxchg_relaxed(v, old, new);
432}
433
434static __always_inline bool
435arch_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
436{
437 return arch_atomic64_try_cmpxchg(v, (s64 *)old, new);
438}
439
440static __always_inline bool
441arch_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
442{
443 return arch_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
444}
445
446static __always_inline bool
447arch_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
448{
449 return arch_atomic64_try_cmpxchg_release(v, (s64 *)old, new);
450}
451
452static __always_inline bool
453arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
454{
455 return arch_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
456}
457
458static __always_inline bool
459arch_atomic_long_sub_and_test(long i, atomic_long_t *v)
460{
461 return arch_atomic64_sub_and_test(i, v);
462}
463
464static __always_inline bool
465arch_atomic_long_dec_and_test(atomic_long_t *v)
466{
467 return arch_atomic64_dec_and_test(v);
468}
469
470static __always_inline bool
471arch_atomic_long_inc_and_test(atomic_long_t *v)
472{
473 return arch_atomic64_inc_and_test(v);
474}
475
476static __always_inline bool
477arch_atomic_long_add_negative(long i, atomic_long_t *v)
478{
479 return arch_atomic64_add_negative(i, v);
480}
481
482static __always_inline bool
483arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
484{
485 return arch_atomic64_add_negative_acquire(i, v);
486}
487
488static __always_inline bool
489arch_atomic_long_add_negative_release(long i, atomic_long_t *v)
490{
491 return arch_atomic64_add_negative_release(i, v);
492}
493
494static __always_inline bool
495arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
496{
497 return arch_atomic64_add_negative_relaxed(i, v);
498}
499
500static __always_inline long
501arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
502{
503 return arch_atomic64_fetch_add_unless(v, a, u);
504}
505
506static __always_inline bool
507arch_atomic_long_add_unless(atomic_long_t *v, long a, long u)
508{
509 return arch_atomic64_add_unless(v, a, u);
510}
511
512static __always_inline bool
513arch_atomic_long_inc_not_zero(atomic_long_t *v)
514{
515 return arch_atomic64_inc_not_zero(v);
516}
517
518static __always_inline bool
519arch_atomic_long_inc_unless_negative(atomic_long_t *v)
520{
521 return arch_atomic64_inc_unless_negative(v);
522}
523
524static __always_inline bool
525arch_atomic_long_dec_unless_positive(atomic_long_t *v)
526{
527 return arch_atomic64_dec_unless_positive(v);
528}
529
530static __always_inline long
531arch_atomic_long_dec_if_positive(atomic_long_t *v)
532{
533 return arch_atomic64_dec_if_positive(v);
534}
535
536#else /* CONFIG_64BIT */
537
538static __always_inline long
539arch_atomic_long_read(const atomic_long_t *v)
540{36{
541 return arch_atomic_read(v);37#ifdef CONFIG_64BIT
38 return raw_atomic64_read(v);
39#else
40 return raw_atomic_read(v);
41#endif
542}42}
543 43
44/**
45 * raw_atomic_long_read_acquire() - atomic load with acquire ordering
46 * @v: pointer to atomic_long_t
47 *
48 * Atomically loads the value of @v with acquire ordering.
49 *
50 * Safe to use in noinstr code; prefer atomic_long_read_acquire() elsewhere.
51 *
52 * Return: The value loaded from @v.
53 */
544static __always_inline long54static __always_inline long
545arch_atomic_long_read_acquire(const atomic_long_t *v)55raw_atomic_long_read_acquire(const atomic_long_t *v)
546{56{
547 return arch_atomic_read_acquire(v);57#ifdef CONFIG_64BIT
58 return raw_atomic64_read_acquire(v);
59#else
60 return raw_atomic_read_acquire(v);
61#endif
548}62}
549 63
64/**
65 * raw_atomic_long_set() - atomic set with relaxed ordering
66 * @v: pointer to atomic_long_t
67 * @i: long value to assign
68 *
69 * Atomically sets @v to @i with relaxed ordering.
70 *
71 * Safe to use in noinstr code; prefer atomic_long_set() elsewhere.
72 *
73 * Return: Nothing.
74 */
550static __always_inline void75static __always_inline void
551arch_atomic_long_set(atomic_long_t *v, long i)76raw_atomic_long_set(atomic_long_t *v, long i)
552{77{
553 arch_atomic_set(v, i);78#ifdef CONFIG_64BIT
79 raw_atomic64_set(v, i);
80#else
81 raw_atomic_set(v, i);
82#endif
554}83}
555 84
85/**
86 * raw_atomic_long_set_release() - atomic set with release ordering
87 * @v: pointer to atomic_long_t
88 * @i: long value to assign
89 *
90 * Atomically sets @v to @i with release ordering.
91 *
92 * Safe to use in noinstr code; prefer atomic_long_set_release() elsewhere.
93 *
94 * Return: Nothing.
95 */
556static __always_inline void96static __always_inline void
557arch_atomic_long_set_release(atomic_long_t *v, long i)97raw_atomic_long_set_release(atomic_long_t *v, long i)
558{98{
559 arch_atomic_set_release(v, i);99#ifdef CONFIG_64BIT
100 raw_atomic64_set_release(v, i);
101#else
102 raw_atomic_set_release(v, i);
103#endif
560}104}
561 105
106/**
107 * raw_atomic_long_add() - atomic add with relaxed ordering
108 * @i: long value to add
109 * @v: pointer to atomic_long_t
110 *
111 * Atomically updates @v to (@v + @i) with relaxed ordering.
112 *
113 * Safe to use in noinstr code; prefer atomic_long_add() elsewhere.
114 *
115 * Return: Nothing.
116 */
562static __always_inline void117static __always_inline void
563arch_atomic_long_add(long i, atomic_long_t *v)118raw_atomic_long_add(long i, atomic_long_t *v)
564{119{
565 arch_atomic_add(i, v);120#ifdef CONFIG_64BIT
121 raw_atomic64_add(i, v);
122#else
123 raw_atomic_add(i, v);
124#endif
566}125}
567 126
127/**
128 * raw_atomic_long_add_return() - atomic add with full ordering
129 * @i: long value to add
130 * @v: pointer to atomic_long_t
131 *
132 * Atomically updates @v to (@v + @i) with full ordering.
133 *
134 * Safe to use in noinstr code; prefer atomic_long_add_return() elsewhere.
135 *
136 * Return: The updated value of @v.
137 */
568static __always_inline long138static __always_inline long
569arch_atomic_long_add_return(long i, atomic_long_t *v)139raw_atomic_long_add_return(long i, atomic_long_t *v)
570{140{
571 return arch_atomic_add_return(i, v);141#ifdef CONFIG_64BIT
142 return raw_atomic64_add_return(i, v);
143#else
144 return raw_atomic_add_return(i, v);
145#endif
572}146}
573 147
148/**
149 * raw_atomic_long_add_return_acquire() - atomic add with acquire ordering
150 * @i: long value to add
151 * @v: pointer to atomic_long_t
152 *
153 * Atomically updates @v to (@v + @i) with acquire ordering.
154 *
155 * Safe to use in noinstr code; prefer atomic_long_add_return_acquire() elsewhere.
156 *
157 * Return: The updated value of @v.
158 */
574static __always_inline long159static __always_inline long
575arch_atomic_long_add_return_acquire(long i, atomic_long_t *v)160raw_atomic_long_add_return_acquire(long i, atomic_long_t *v)
576{161{
577 return arch_atomic_add_return_acquire(i, v);162#ifdef CONFIG_64BIT
163 return raw_atomic64_add_return_acquire(i, v);
164#else
165 return raw_atomic_add_return_acquire(i, v);
166#endif
578}167}
579 168
169/**
170 * raw_atomic_long_add_return_release() - atomic add with release ordering
171 * @i: long value to add
172 * @v: pointer to atomic_long_t
173 *
174 * Atomically updates @v to (@v + @i) with release ordering.
175 *
176 * Safe to use in noinstr code; prefer atomic_long_add_return_release() elsewhere.
177 *
178 * Return: The updated value of @v.
179 */
580static __always_inline long180static __always_inline long
581arch_atomic_long_add_return_release(long i, atomic_long_t *v)181raw_atomic_long_add_return_release(long i, atomic_long_t *v)
582{182{
583 return arch_atomic_add_return_release(i, v);183#ifdef CONFIG_64BIT
184 return raw_atomic64_add_return_release(i, v);
185#else
186 return raw_atomic_add_return_release(i, v);
187#endif
584}188}
585 189
190/**
191 * raw_atomic_long_add_return_relaxed() - atomic add with relaxed ordering
192 * @i: long value to add
193 * @v: pointer to atomic_long_t
194 *
195 * Atomically updates @v to (@v + @i) with relaxed ordering.
196 *
197 * Safe to use in noinstr code; prefer atomic_long_add_return_relaxed() elsewhere.
198 *
199 * Return: The updated value of @v.
200 */
586static __always_inline long201static __always_inline long
587arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v)202raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
588{203{
589 return arch_atomic_add_return_relaxed(i, v);204#ifdef CONFIG_64BIT
205 return raw_atomic64_add_return_relaxed(i, v);
206#else
207 return raw_atomic_add_return_relaxed(i, v);
208#endif
590}209}
591 210
211/**
212 * raw_atomic_long_fetch_add() - atomic add with full ordering
213 * @i: long value to add
214 * @v: pointer to atomic_long_t
215 *
216 * Atomically updates @v to (@v + @i) with full ordering.
217 *
218 * Safe to use in noinstr code; prefer atomic_long_fetch_add() elsewhere.
219 *
220 * Return: The original value of @v.
221 */
592static __always_inline long222static __always_inline long
593arch_atomic_long_fetch_add(long i, atomic_long_t *v)223raw_atomic_long_fetch_add(long i, atomic_long_t *v)
594{224{
595 return arch_atomic_fetch_add(i, v);225#ifdef CONFIG_64BIT
226 return raw_atomic64_fetch_add(i, v);
227#else
228 return raw_atomic_fetch_add(i, v);
229#endif
596}230}
597 231
232/**
233 * raw_atomic_long_fetch_add_acquire() - atomic add with acquire ordering
234 * @i: long value to add
235 * @v: pointer to atomic_long_t
236 *
237 * Atomically updates @v to (@v + @i) with acquire ordering.
238 *
239 * Safe to use in noinstr code; prefer atomic_long_fetch_add_acquire() elsewhere.
240 *
241 * Return: The original value of @v.
242 */
598static __always_inline long243static __always_inline long
599arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)244raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
600{245{
601 return arch_atomic_fetch_add_acquire(i, v);246#ifdef CONFIG_64BIT
247 return raw_atomic64_fetch_add_acquire(i, v);
248#else
249 return raw_atomic_fetch_add_acquire(i, v);
250#endif
602}251}
603 252
253/**
254 * raw_atomic_long_fetch_add_release() - atomic add with release ordering
255 * @i: long value to add
256 * @v: pointer to atomic_long_t
257 *
258 * Atomically updates @v to (@v + @i) with release ordering.
259 *
260 * Safe to use in noinstr code; prefer atomic_long_fetch_add_release() elsewhere.
261 *
262 * Return: The original value of @v.
263 */
604static __always_inline long264static __always_inline long
605arch_atomic_long_fetch_add_release(long i, atomic_long_t *v)265raw_atomic_long_fetch_add_release(long i, atomic_long_t *v)
606{266{
607 return arch_atomic_fetch_add_release(i, v);267#ifdef CONFIG_64BIT
268 return raw_atomic64_fetch_add_release(i, v);
269#else
270 return raw_atomic_fetch_add_release(i, v);
271#endif
608}272}
609 273
274/**
275 * raw_atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
276 * @i: long value to add
277 * @v: pointer to atomic_long_t
278 *
279 * Atomically updates @v to (@v + @i) with relaxed ordering.
280 *
281 * Safe to use in noinstr code; prefer atomic_long_fetch_add_relaxed() elsewhere.
282 *
283 * Return: The original value of @v.
284 */
610static __always_inline long285static __always_inline long
611arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)286raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
612{287{
613 return arch_atomic_fetch_add_relaxed(i, v);288#ifdef CONFIG_64BIT
289 return raw_atomic64_fetch_add_relaxed(i, v);
290#else
291 return raw_atomic_fetch_add_relaxed(i, v);
292#endif
614}293}
615 294
295/**
296 * raw_atomic_long_sub() - atomic subtract with relaxed ordering
297 * @i: long value to subtract
298 * @v: pointer to atomic_long_t
299 *
300 * Atomically updates @v to (@v - @i) with relaxed ordering.
301 *
302 * Safe to use in noinstr code; prefer atomic_long_sub() elsewhere.
303 *
304 * Return: Nothing.
305 */
616static __always_inline void306static __always_inline void
617arch_atomic_long_sub(long i, atomic_long_t *v)307raw_atomic_long_sub(long i, atomic_long_t *v)
618{308{
619 arch_atomic_sub(i, v);309#ifdef CONFIG_64BIT
310 raw_atomic64_sub(i, v);
311#else
312 raw_atomic_sub(i, v);
313#endif
620}314}
621 315
316/**
317 * raw_atomic_long_sub_return() - atomic subtract with full ordering
318 * @i: long value to subtract
319 * @v: pointer to atomic_long_t
320 *
321 * Atomically updates @v to (@v - @i) with full ordering.
322 *
323 * Safe to use in noinstr code; prefer atomic_long_sub_return() elsewhere.
324 *
325 * Return: The updated value of @v.
326 */
622static __always_inline long327static __always_inline long
623arch_atomic_long_sub_return(long i, atomic_long_t *v)328raw_atomic_long_sub_return(long i, atomic_long_t *v)
624{329{
625 return arch_atomic_sub_return(i, v);330#ifdef CONFIG_64BIT
331 return raw_atomic64_sub_return(i, v);
332#else
333 return raw_atomic_sub_return(i, v);
334#endif
626}335}
627 336
337/**
338 * raw_atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
339 * @i: long value to subtract
340 * @v: pointer to atomic_long_t
341 *
342 * Atomically updates @v to (@v - @i) with acquire ordering.
343 *
344 * Safe to use in noinstr code; prefer atomic_long_sub_return_acquire() elsewhere.
345 *
346 * Return: The updated value of @v.
347 */
628static __always_inline long348static __always_inline long
629arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v)349raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
630{350{
631 return arch_atomic_sub_return_acquire(i, v);351#ifdef CONFIG_64BIT
352 return raw_atomic64_sub_return_acquire(i, v);
353#else
354 return raw_atomic_sub_return_acquire(i, v);
355#endif
632}356}
633 357
358/**
359 * raw_atomic_long_sub_return_release() - atomic subtract with release ordering
360 * @i: long value to subtract
361 * @v: pointer to atomic_long_t
362 *
363 * Atomically updates @v to (@v - @i) with release ordering.
364 *
365 * Safe to use in noinstr code; prefer atomic_long_sub_return_release() elsewhere.
366 *
367 * Return: The updated value of @v.
368 */
634static __always_inline long369static __always_inline long
635arch_atomic_long_sub_return_release(long i, atomic_long_t *v)370raw_atomic_long_sub_return_release(long i, atomic_long_t *v)
636{371{
637 return arch_atomic_sub_return_release(i, v);372#ifdef CONFIG_64BIT
373 return raw_atomic64_sub_return_release(i, v);
374#else
375 return raw_atomic_sub_return_release(i, v);
376#endif
638}377}
639 378
379/**
380 * raw_atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
381 * @i: long value to subtract
382 * @v: pointer to atomic_long_t
383 *
384 * Atomically updates @v to (@v - @i) with relaxed ordering.
385 *
386 * Safe to use in noinstr code; prefer atomic_long_sub_return_relaxed() elsewhere.
387 *
388 * Return: The updated value of @v.
389 */
640static __always_inline long390static __always_inline long
641arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)391raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
642{392{
643 return arch_atomic_sub_return_relaxed(i, v);393#ifdef CONFIG_64BIT
394 return raw_atomic64_sub_return_relaxed(i, v);
395#else
396 return raw_atomic_sub_return_relaxed(i, v);
397#endif
644}398}
645 399
400/**
401 * raw_atomic_long_fetch_sub() - atomic subtract with full ordering
402 * @i: long value to subtract
403 * @v: pointer to atomic_long_t
404 *
405 * Atomically updates @v to (@v - @i) with full ordering.
406 *
407 * Safe to use in noinstr code; prefer atomic_long_fetch_sub() elsewhere.
408 *
409 * Return: The original value of @v.
410 */
646static __always_inline long411static __always_inline long
647arch_atomic_long_fetch_sub(long i, atomic_long_t *v)412raw_atomic_long_fetch_sub(long i, atomic_long_t *v)
648{413{
649 return arch_atomic_fetch_sub(i, v);414#ifdef CONFIG_64BIT
415 return raw_atomic64_fetch_sub(i, v);
416#else
417 return raw_atomic_fetch_sub(i, v);
418#endif
650}419}
651 420
421/**
422 * raw_atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
423 * @i: long value to subtract
424 * @v: pointer to atomic_long_t
425 *
426 * Atomically updates @v to (@v - @i) with acquire ordering.
427 *
428 * Safe to use in noinstr code; prefer atomic_long_fetch_sub_acquire() elsewhere.
429 *
430 * Return: The original value of @v.
431 */
652static __always_inline long432static __always_inline long
653arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)433raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
654{434{
655 return arch_atomic_fetch_sub_acquire(i, v);435#ifdef CONFIG_64BIT
436 return raw_atomic64_fetch_sub_acquire(i, v);
437#else
438 return raw_atomic_fetch_sub_acquire(i, v);
439#endif
656}440}
657 441
442/**
443 * raw_atomic_long_fetch_sub_release() - atomic subtract with release ordering
444 * @i: long value to subtract
445 * @v: pointer to atomic_long_t
446 *
447 * Atomically updates @v to (@v - @i) with release ordering.
448 *
449 * Safe to use in noinstr code; prefer atomic_long_fetch_sub_release() elsewhere.
450 *
451 * Return: The original value of @v.
452 */
658static __always_inline long453static __always_inline long
659arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v)454raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
660{455{
661 return arch_atomic_fetch_sub_release(i, v);456#ifdef CONFIG_64BIT
457 return raw_atomic64_fetch_sub_release(i, v);
458#else
459 return raw_atomic_fetch_sub_release(i, v);
460#endif
662}461}
663 462
463/**
464 * raw_atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
465 * @i: long value to subtract
466 * @v: pointer to atomic_long_t
467 *
468 * Atomically updates @v to (@v - @i) with relaxed ordering.
469 *
470 * Safe to use in noinstr code; prefer atomic_long_fetch_sub_relaxed() elsewhere.
471 *
472 * Return: The original value of @v.
473 */
664static __always_inline long474static __always_inline long
665arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)475raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
666{476{
667 return arch_atomic_fetch_sub_relaxed(i, v);477#ifdef CONFIG_64BIT
478 return raw_atomic64_fetch_sub_relaxed(i, v);
479#else
480 return raw_atomic_fetch_sub_relaxed(i, v);
481#endif
668}482}
669 483
484/**
485 * raw_atomic_long_inc() - atomic increment with relaxed ordering
486 * @v: pointer to atomic_long_t
487 *
488 * Atomically updates @v to (@v + 1) with relaxed ordering.
489 *
490 * Safe to use in noinstr code; prefer atomic_long_inc() elsewhere.
491 *
492 * Return: Nothing.
493 */
670static __always_inline void494static __always_inline void
671arch_atomic_long_inc(atomic_long_t *v)495raw_atomic_long_inc(atomic_long_t *v)
672{496{
673 arch_atomic_inc(v);497#ifdef CONFIG_64BIT
498 raw_atomic64_inc(v);
499#else
500 raw_atomic_inc(v);
501#endif
674}502}
675 503
504/**
505 * raw_atomic_long_inc_return() - atomic increment with full ordering
506 * @v: pointer to atomic_long_t
507 *
508 * Atomically updates @v to (@v + 1) with full ordering.
509 *
510 * Safe to use in noinstr code; prefer atomic_long_inc_return() elsewhere.
511 *
512 * Return: The updated value of @v.
513 */
676static __always_inline long514static __always_inline long
677arch_atomic_long_inc_return(atomic_long_t *v)515raw_atomic_long_inc_return(atomic_long_t *v)
678{516{
679 return arch_atomic_inc_return(v);517#ifdef CONFIG_64BIT
518 return raw_atomic64_inc_return(v);
519#else
520 return raw_atomic_inc_return(v);
521#endif
680}522}
681 523
524/**
525 * raw_atomic_long_inc_return_acquire() - atomic increment with acquire ordering
526 * @v: pointer to atomic_long_t
527 *
528 * Atomically updates @v to (@v + 1) with acquire ordering.
529 *
530 * Safe to use in noinstr code; prefer atomic_long_inc_return_acquire() elsewhere.
531 *
532 * Return: The updated value of @v.
533 */
682static __always_inline long534static __always_inline long
683arch_atomic_long_inc_return_acquire(atomic_long_t *v)535raw_atomic_long_inc_return_acquire(atomic_long_t *v)
684{536{
685 return arch_atomic_inc_return_acquire(v);537#ifdef CONFIG_64BIT
538 return raw_atomic64_inc_return_acquire(v);
539#else
540 return raw_atomic_inc_return_acquire(v);
541#endif
686}542}
687 543
544/**
545 * raw_atomic_long_inc_return_release() - atomic increment with release ordering
546 * @v: pointer to atomic_long_t
547 *
548 * Atomically updates @v to (@v + 1) with release ordering.
549 *
550 * Safe to use in noinstr code; prefer atomic_long_inc_return_release() elsewhere.
551 *
552 * Return: The updated value of @v.
553 */
688static __always_inline long554static __always_inline long
689arch_atomic_long_inc_return_release(atomic_long_t *v)555raw_atomic_long_inc_return_release(atomic_long_t *v)
690{556{
691 return arch_atomic_inc_return_release(v);557#ifdef CONFIG_64BIT
558 return raw_atomic64_inc_return_release(v);
559#else
560 return raw_atomic_inc_return_release(v);
561#endif
692}562}
693 563
564/**
565 * raw_atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
566 * @v: pointer to atomic_long_t
567 *
568 * Atomically updates @v to (@v + 1) with relaxed ordering.
569 *
570 * Safe to use in noinstr code; prefer atomic_long_inc_return_relaxed() elsewhere.
571 *
572 * Return: The updated value of @v.
573 */
694static __always_inline long574static __always_inline long
695arch_atomic_long_inc_return_relaxed(atomic_long_t *v)575raw_atomic_long_inc_return_relaxed(atomic_long_t *v)
696{576{
697 return arch_atomic_inc_return_relaxed(v);577#ifdef CONFIG_64BIT
578 return raw_atomic64_inc_return_relaxed(v);
579#else
580 return raw_atomic_inc_return_relaxed(v);
581#endif
698}582}
699 583
584/**
585 * raw_atomic_long_fetch_inc() - atomic increment with full ordering
586 * @v: pointer to atomic_long_t
587 *
588 * Atomically updates @v to (@v + 1) with full ordering.
589 *
590 * Safe to use in noinstr code; prefer atomic_long_fetch_inc() elsewhere.
591 *
592 * Return: The original value of @v.
593 */
700static __always_inline long594static __always_inline long
701arch_atomic_long_fetch_inc(atomic_long_t *v)595raw_atomic_long_fetch_inc(atomic_long_t *v)
702{596{
703 return arch_atomic_fetch_inc(v);597#ifdef CONFIG_64BIT
598 return raw_atomic64_fetch_inc(v);
599#else
600 return raw_atomic_fetch_inc(v);
601#endif
704}602}
705 603
604/**
605 * raw_atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
606 * @v: pointer to atomic_long_t
607 *
608 * Atomically updates @v to (@v + 1) with acquire ordering.
609 *
610 * Safe to use in noinstr code; prefer atomic_long_fetch_inc_acquire() elsewhere.
611 *
612 * Return: The original value of @v.
613 */
706static __always_inline long614static __always_inline long
707arch_atomic_long_fetch_inc_acquire(atomic_long_t *v)615raw_atomic_long_fetch_inc_acquire(atomic_long_t *v)
708{616{
709 return arch_atomic_fetch_inc_acquire(v);617#ifdef CONFIG_64BIT
618 return raw_atomic64_fetch_inc_acquire(v);
619#else
620 return raw_atomic_fetch_inc_acquire(v);
621#endif
710}622}
711 623
624/**
625 * raw_atomic_long_fetch_inc_release() - atomic increment with release ordering
626 * @v: pointer to atomic_long_t
627 *
628 * Atomically updates @v to (@v + 1) with release ordering.
629 *
630 * Safe to use in noinstr code; prefer atomic_long_fetch_inc_release() elsewhere.
631 *
632 * Return: The original value of @v.
633 */
712static __always_inline long634static __always_inline long
713arch_atomic_long_fetch_inc_release(atomic_long_t *v)635raw_atomic_long_fetch_inc_release(atomic_long_t *v)
714{636{
715 return arch_atomic_fetch_inc_release(v);637#ifdef CONFIG_64BIT
638 return raw_atomic64_fetch_inc_release(v);
639#else
640 return raw_atomic_fetch_inc_release(v);
641#endif
716}642}
717 643
644/**
645 * raw_atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
646 * @v: pointer to atomic_long_t
647 *
648 * Atomically updates @v to (@v + 1) with relaxed ordering.
649 *
650 * Safe to use in noinstr code; prefer atomic_long_fetch_inc_relaxed() elsewhere.
651 *
652 * Return: The original value of @v.
653 */
718static __always_inline long654static __always_inline long
719arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v)655raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
720{656{
721 return arch_atomic_fetch_inc_relaxed(v);657#ifdef CONFIG_64BIT
658 return raw_atomic64_fetch_inc_relaxed(v);
659#else
660 return raw_atomic_fetch_inc_relaxed(v);
661#endif
722}662}
723 663
664/**
665 * raw_atomic_long_dec() - atomic decrement with relaxed ordering
666 * @v: pointer to atomic_long_t
667 *
668 * Atomically updates @v to (@v - 1) with relaxed ordering.
669 *
670 * Safe to use in noinstr code; prefer atomic_long_dec() elsewhere.
671 *
672 * Return: Nothing.
673 */
724static __always_inline void674static __always_inline void
725arch_atomic_long_dec(atomic_long_t *v)675raw_atomic_long_dec(atomic_long_t *v)
726{676{
727 arch_atomic_dec(v);677#ifdef CONFIG_64BIT
678 raw_atomic64_dec(v);
679#else
680 raw_atomic_dec(v);
681#endif
728}682}
729 683
684/**
685 * raw_atomic_long_dec_return() - atomic decrement with full ordering
686 * @v: pointer to atomic_long_t
687 *
688 * Atomically updates @v to (@v - 1) with full ordering.
689 *
690 * Safe to use in noinstr code; prefer atomic_long_dec_return() elsewhere.
691 *
692 * Return: The updated value of @v.
693 */
730static __always_inline long694static __always_inline long
731arch_atomic_long_dec_return(atomic_long_t *v)695raw_atomic_long_dec_return(atomic_long_t *v)
732{696{
733 return arch_atomic_dec_return(v);697#ifdef CONFIG_64BIT
698 return raw_atomic64_dec_return(v);
699#else
700 return raw_atomic_dec_return(v);
701#endif
734}702}
735 703
704/**
705 * raw_atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
706 * @v: pointer to atomic_long_t
707 *
708 * Atomically updates @v to (@v - 1) with acquire ordering.
709 *
710 * Safe to use in noinstr code; prefer atomic_long_dec_return_acquire() elsewhere.
711 *
712 * Return: The updated value of @v.
713 */
736static __always_inline long714static __always_inline long
737arch_atomic_long_dec_return_acquire(atomic_long_t *v)715raw_atomic_long_dec_return_acquire(atomic_long_t *v)
738{716{
739 return arch_atomic_dec_return_acquire(v);717#ifdef CONFIG_64BIT
718 return raw_atomic64_dec_return_acquire(v);
719#else
720 return raw_atomic_dec_return_acquire(v);
721#endif
740}722}
741 723
724/**
725 * raw_atomic_long_dec_return_release() - atomic decrement with release ordering
726 * @v: pointer to atomic_long_t
727 *
728 * Atomically updates @v to (@v - 1) with release ordering.
729 *
730 * Safe to use in noinstr code; prefer atomic_long_dec_return_release() elsewhere.
731 *
732 * Return: The updated value of @v.
733 */
742static __always_inline long734static __always_inline long
743arch_atomic_long_dec_return_release(atomic_long_t *v)735raw_atomic_long_dec_return_release(atomic_long_t *v)
744{736{
745 return arch_atomic_dec_return_release(v);737#ifdef CONFIG_64BIT
738 return raw_atomic64_dec_return_release(v);
739#else
740 return raw_atomic_dec_return_release(v);
741#endif
746}742}
747 743
744/**
745 * raw_atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
746 * @v: pointer to atomic_long_t
747 *
748 * Atomically updates @v to (@v - 1) with relaxed ordering.
749 *
750 * Safe to use in noinstr code; prefer atomic_long_dec_return_relaxed() elsewhere.
751 *
752 * Return: The updated value of @v.
753 */
748static __always_inline long754static __always_inline long
749arch_atomic_long_dec_return_relaxed(atomic_long_t *v)755raw_atomic_long_dec_return_relaxed(atomic_long_t *v)
750{756{
751 return arch_atomic_dec_return_relaxed(v);757#ifdef CONFIG_64BIT
758 return raw_atomic64_dec_return_relaxed(v);
759#else
760 return raw_atomic_dec_return_relaxed(v);
761#endif
752}762}
753 763
764/**
765 * raw_atomic_long_fetch_dec() - atomic decrement with full ordering
766 * @v: pointer to atomic_long_t
767 *
768 * Atomically updates @v to (@v - 1) with full ordering.
769 *
770 * Safe to use in noinstr code; prefer atomic_long_fetch_dec() elsewhere.
771 *
772 * Return: The original value of @v.
773 */
754static __always_inline long774static __always_inline long
755arch_atomic_long_fetch_dec(atomic_long_t *v)775raw_atomic_long_fetch_dec(atomic_long_t *v)
756{776{
757 return arch_atomic_fetch_dec(v);777#ifdef CONFIG_64BIT
778 return raw_atomic64_fetch_dec(v);
779#else
780 return raw_atomic_fetch_dec(v);
781#endif
758}782}
759 783
784/**
785 * raw_atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
786 * @v: pointer to atomic_long_t
787 *
788 * Atomically updates @v to (@v - 1) with acquire ordering.
789 *
790 * Safe to use in noinstr code; prefer atomic_long_fetch_dec_acquire() elsewhere.
791 *
792 * Return: The original value of @v.
793 */
760static __always_inline long794static __always_inline long
761arch_atomic_long_fetch_dec_acquire(atomic_long_t *v)795raw_atomic_long_fetch_dec_acquire(atomic_long_t *v)
762{796{
763 return arch_atomic_fetch_dec_acquire(v);797#ifdef CONFIG_64BIT
798 return raw_atomic64_fetch_dec_acquire(v);
799#else
800 return raw_atomic_fetch_dec_acquire(v);
801#endif
764}802}
765 803
804/**
805 * raw_atomic_long_fetch_dec_release() - atomic decrement with release ordering
806 * @v: pointer to atomic_long_t
807 *
808 * Atomically updates @v to (@v - 1) with release ordering.
809 *
810 * Safe to use in noinstr code; prefer atomic_long_fetch_dec_release() elsewhere.
811 *
812 * Return: The original value of @v.
813 */
766static __always_inline long814static __always_inline long
767arch_atomic_long_fetch_dec_release(atomic_long_t *v)815raw_atomic_long_fetch_dec_release(atomic_long_t *v)
768{816{
769 return arch_atomic_fetch_dec_release(v);817#ifdef CONFIG_64BIT
818 return raw_atomic64_fetch_dec_release(v);
819#else
820 return raw_atomic_fetch_dec_release(v);
821#endif
770}822}
771 823
824/**
825 * raw_atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
826 * @v: pointer to atomic_long_t
827 *
828 * Atomically updates @v to (@v - 1) with relaxed ordering.
829 *
830 * Safe to use in noinstr code; prefer atomic_long_fetch_dec_relaxed() elsewhere.
831 *
832 * Return: The original value of @v.
833 */
772static __always_inline long834static __always_inline long
773arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v)835raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
774{836{
775 return arch_atomic_fetch_dec_relaxed(v);837#ifdef CONFIG_64BIT
838 return raw_atomic64_fetch_dec_relaxed(v);
839#else
840 return raw_atomic_fetch_dec_relaxed(v);
841#endif
776}842}
777 843
844/**
845 * raw_atomic_long_and() - atomic bitwise AND with relaxed ordering
846 * @i: long value
847 * @v: pointer to atomic_long_t
848 *
849 * Atomically updates @v to (@v & @i) with relaxed ordering.
850 *
851 * Safe to use in noinstr code; prefer atomic_long_and() elsewhere.
852 *
853 * Return: Nothing.
854 */
778static __always_inline void855static __always_inline void
779arch_atomic_long_and(long i, atomic_long_t *v)856raw_atomic_long_and(long i, atomic_long_t *v)
780{857{
781 arch_atomic_and(i, v);858#ifdef CONFIG_64BIT
859 raw_atomic64_and(i, v);
860#else
861 raw_atomic_and(i, v);
862#endif
782}863}
783 864
865/**
866 * raw_atomic_long_fetch_and() - atomic bitwise AND with full ordering
867 * @i: long value
868 * @v: pointer to atomic_long_t
869 *
870 * Atomically updates @v to (@v & @i) with full ordering.
871 *
872 * Safe to use in noinstr code; prefer atomic_long_fetch_and() elsewhere.
873 *
874 * Return: The original value of @v.
875 */
784static __always_inline long876static __always_inline long
785arch_atomic_long_fetch_and(long i, atomic_long_t *v)877raw_atomic_long_fetch_and(long i, atomic_long_t *v)
786{878{
787 return arch_atomic_fetch_and(i, v);879#ifdef CONFIG_64BIT
880 return raw_atomic64_fetch_and(i, v);
881#else
882 return raw_atomic_fetch_and(i, v);
883#endif
788}884}
789 885
886/**
887 * raw_atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
888 * @i: long value
889 * @v: pointer to atomic_long_t
890 *
891 * Atomically updates @v to (@v & @i) with acquire ordering.
892 *
893 * Safe to use in noinstr code; prefer atomic_long_fetch_and_acquire() elsewhere.
894 *
895 * Return: The original value of @v.
896 */
790static __always_inline long897static __always_inline long
791arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)898raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
792{899{
793 return arch_atomic_fetch_and_acquire(i, v);900#ifdef CONFIG_64BIT
901 return raw_atomic64_fetch_and_acquire(i, v);
902#else
903 return raw_atomic_fetch_and_acquire(i, v);
904#endif
794}905}
795 906
907/**
908 * raw_atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
909 * @i: long value
910 * @v: pointer to atomic_long_t
911 *
912 * Atomically updates @v to (@v & @i) with release ordering.
913 *
914 * Safe to use in noinstr code; prefer atomic_long_fetch_and_release() elsewhere.
915 *
916 * Return: The original value of @v.
917 */
796static __always_inline long918static __always_inline long
797arch_atomic_long_fetch_and_release(long i, atomic_long_t *v)919raw_atomic_long_fetch_and_release(long i, atomic_long_t *v)
798{920{
799 return arch_atomic_fetch_and_release(i, v);921#ifdef CONFIG_64BIT
922 return raw_atomic64_fetch_and_release(i, v);
923#else
924 return raw_atomic_fetch_and_release(i, v);
925#endif
800}926}
801 927
928/**
929 * raw_atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
930 * @i: long value
931 * @v: pointer to atomic_long_t
932 *
933 * Atomically updates @v to (@v & @i) with relaxed ordering.
934 *
935 * Safe to use in noinstr code; prefer atomic_long_fetch_and_relaxed() elsewhere.
936 *
937 * Return: The original value of @v.
938 */
802static __always_inline long939static __always_inline long
803arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)940raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
804{941{
805 return arch_atomic_fetch_and_relaxed(i, v);942#ifdef CONFIG_64BIT
943 return raw_atomic64_fetch_and_relaxed(i, v);
944#else
945 return raw_atomic_fetch_and_relaxed(i, v);
946#endif
806}947}
807 948
949/**
950 * raw_atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
951 * @i: long value
952 * @v: pointer to atomic_long_t
953 *
954 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
955 *
956 * Safe to use in noinstr code; prefer atomic_long_andnot() elsewhere.
957 *
958 * Return: Nothing.
959 */
808static __always_inline void960static __always_inline void
809arch_atomic_long_andnot(long i, atomic_long_t *v)961raw_atomic_long_andnot(long i, atomic_long_t *v)
810{962{
811 arch_atomic_andnot(i, v);963#ifdef CONFIG_64BIT
964 raw_atomic64_andnot(i, v);
965#else
966 raw_atomic_andnot(i, v);
967#endif
812}968}
813 969
970/**
971 * raw_atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
972 * @i: long value
973 * @v: pointer to atomic_long_t
974 *
975 * Atomically updates @v to (@v & ~@i) with full ordering.
976 *
977 * Safe to use in noinstr code; prefer atomic_long_fetch_andnot() elsewhere.
978 *
979 * Return: The original value of @v.
980 */
814static __always_inline long981static __always_inline long
815arch_atomic_long_fetch_andnot(long i, atomic_long_t *v)982raw_atomic_long_fetch_andnot(long i, atomic_long_t *v)
816{983{
817 return arch_atomic_fetch_andnot(i, v);984#ifdef CONFIG_64BIT
985 return raw_atomic64_fetch_andnot(i, v);
986#else
987 return raw_atomic_fetch_andnot(i, v);
988#endif
818}989}
819 990
991/**
992 * raw_atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
993 * @i: long value
994 * @v: pointer to atomic_long_t
995 *
996 * Atomically updates @v to (@v & ~@i) with acquire ordering.
997 *
998 * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_acquire() elsewhere.
999 *
1000 * Return: The original value of @v.
1001 */
820static __always_inline long1002static __always_inline long
821arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)1003raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
822{1004{
823 return arch_atomic_fetch_andnot_acquire(i, v);1005#ifdef CONFIG_64BIT
1006 return raw_atomic64_fetch_andnot_acquire(i, v);
1007#else
1008 return raw_atomic_fetch_andnot_acquire(i, v);
1009#endif
824}1010}
825 1011
1012/**
1013 * raw_atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1014 * @i: long value
1015 * @v: pointer to atomic_long_t
1016 *
1017 * Atomically updates @v to (@v & ~@i) with release ordering.
1018 *
1019 * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_release() elsewhere.
1020 *
1021 * Return: The original value of @v.
1022 */
826static __always_inline long1023static __always_inline long
827arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)1024raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
828{1025{
829 return arch_atomic_fetch_andnot_release(i, v);1026#ifdef CONFIG_64BIT
1027 return raw_atomic64_fetch_andnot_release(i, v);
1028#else
1029 return raw_atomic_fetch_andnot_release(i, v);
1030#endif
830}1031}
831 1032
1033/**
1034 * raw_atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1035 * @i: long value
1036 * @v: pointer to atomic_long_t
1037 *
1038 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1039 *
1040 * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_relaxed() elsewhere.
1041 *
1042 * Return: The original value of @v.
1043 */
832static __always_inline long1044static __always_inline long
833arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)1045raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
834{1046{
835 return arch_atomic_fetch_andnot_relaxed(i, v);1047#ifdef CONFIG_64BIT
1048 return raw_atomic64_fetch_andnot_relaxed(i, v);
1049#else
1050 return raw_atomic_fetch_andnot_relaxed(i, v);
1051#endif
836}1052}
837 1053
1054/**
1055 * raw_atomic_long_or() - atomic bitwise OR with relaxed ordering
1056 * @i: long value
1057 * @v: pointer to atomic_long_t
1058 *
1059 * Atomically updates @v to (@v | @i) with relaxed ordering.
1060 *
1061 * Safe to use in noinstr code; prefer atomic_long_or() elsewhere.
1062 *
1063 * Return: Nothing.
1064 */
838static __always_inline void1065static __always_inline void
839arch_atomic_long_or(long i, atomic_long_t *v)1066raw_atomic_long_or(long i, atomic_long_t *v)
840{1067{
841 arch_atomic_or(i, v);1068#ifdef CONFIG_64BIT
1069 raw_atomic64_or(i, v);
1070#else
1071 raw_atomic_or(i, v);
1072#endif
842}1073}
843 1074
1075/**
1076 * raw_atomic_long_fetch_or() - atomic bitwise OR with full ordering
1077 * @i: long value
1078 * @v: pointer to atomic_long_t
1079 *
1080 * Atomically updates @v to (@v | @i) with full ordering.
1081 *
1082 * Safe to use in noinstr code; prefer atomic_long_fetch_or() elsewhere.
1083 *
1084 * Return: The original value of @v.
1085 */
844static __always_inline long1086static __always_inline long
845arch_atomic_long_fetch_or(long i, atomic_long_t *v)1087raw_atomic_long_fetch_or(long i, atomic_long_t *v)
846{1088{
847 return arch_atomic_fetch_or(i, v);1089#ifdef CONFIG_64BIT
1090 return raw_atomic64_fetch_or(i, v);
1091#else
1092 return raw_atomic_fetch_or(i, v);
1093#endif
848}1094}
849 1095
1096/**
1097 * raw_atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
1098 * @i: long value
1099 * @v: pointer to atomic_long_t
1100 *
1101 * Atomically updates @v to (@v | @i) with acquire ordering.
1102 *
1103 * Safe to use in noinstr code; prefer atomic_long_fetch_or_acquire() elsewhere.
1104 *
1105 * Return: The original value of @v.
1106 */
850static __always_inline long1107static __always_inline long
851arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)1108raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
852{1109{
853 return arch_atomic_fetch_or_acquire(i, v);1110#ifdef CONFIG_64BIT
1111 return raw_atomic64_fetch_or_acquire(i, v);
1112#else
1113 return raw_atomic_fetch_or_acquire(i, v);
1114#endif
854}1115}
855 1116
1117/**
1118 * raw_atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
1119 * @i: long value
1120 * @v: pointer to atomic_long_t
1121 *
1122 * Atomically updates @v to (@v | @i) with release ordering.
1123 *
1124 * Safe to use in noinstr code; prefer atomic_long_fetch_or_release() elsewhere.
1125 *
1126 * Return: The original value of @v.
1127 */
856static __always_inline long1128static __always_inline long
857arch_atomic_long_fetch_or_release(long i, atomic_long_t *v)1129raw_atomic_long_fetch_or_release(long i, atomic_long_t *v)
858{1130{
859 return arch_atomic_fetch_or_release(i, v);1131#ifdef CONFIG_64BIT
1132 return raw_atomic64_fetch_or_release(i, v);
1133#else
1134 return raw_atomic_fetch_or_release(i, v);
1135#endif
860}1136}
861 1137
1138/**
1139 * raw_atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1140 * @i: long value
1141 * @v: pointer to atomic_long_t
1142 *
1143 * Atomically updates @v to (@v | @i) with relaxed ordering.
1144 *
1145 * Safe to use in noinstr code; prefer atomic_long_fetch_or_relaxed() elsewhere.
1146 *
1147 * Return: The original value of @v.
1148 */
862static __always_inline long1149static __always_inline long
863arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)1150raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
864{1151{
865 return arch_atomic_fetch_or_relaxed(i, v);1152#ifdef CONFIG_64BIT
1153 return raw_atomic64_fetch_or_relaxed(i, v);
1154#else
1155 return raw_atomic_fetch_or_relaxed(i, v);
1156#endif
866}1157}
867 1158
1159/**
1160 * raw_atomic_long_xor() - atomic bitwise XOR with relaxed ordering
1161 * @i: long value
1162 * @v: pointer to atomic_long_t
1163 *
1164 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1165 *
1166 * Safe to use in noinstr code; prefer atomic_long_xor() elsewhere.
1167 *
1168 * Return: Nothing.
1169 */
868static __always_inline void1170static __always_inline void
869arch_atomic_long_xor(long i, atomic_long_t *v)1171raw_atomic_long_xor(long i, atomic_long_t *v)
870{1172{
871 arch_atomic_xor(i, v);1173#ifdef CONFIG_64BIT
1174 raw_atomic64_xor(i, v);
1175#else
1176 raw_atomic_xor(i, v);
1177#endif
872}1178}
873 1179
1180/**
1181 * raw_atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
1182 * @i: long value
1183 * @v: pointer to atomic_long_t
1184 *
1185 * Atomically updates @v to (@v ^ @i) with full ordering.
1186 *
1187 * Safe to use in noinstr code; prefer atomic_long_fetch_xor() elsewhere.
1188 *
1189 * Return: The original value of @v.
1190 */
874static __always_inline long1191static __always_inline long
875arch_atomic_long_fetch_xor(long i, atomic_long_t *v)1192raw_atomic_long_fetch_xor(long i, atomic_long_t *v)
876{1193{
877 return arch_atomic_fetch_xor(i, v);1194#ifdef CONFIG_64BIT
1195 return raw_atomic64_fetch_xor(i, v);
1196#else
1197 return raw_atomic_fetch_xor(i, v);
1198#endif
878}1199}
879 1200
1201/**
1202 * raw_atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1203 * @i: long value
1204 * @v: pointer to atomic_long_t
1205 *
1206 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1207 *
1208 * Safe to use in noinstr code; prefer atomic_long_fetch_xor_acquire() elsewhere.
1209 *
1210 * Return: The original value of @v.
1211 */
880static __always_inline long1212static __always_inline long
881arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)1213raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
882{1214{
883 return arch_atomic_fetch_xor_acquire(i, v);1215#ifdef CONFIG_64BIT
1216 return raw_atomic64_fetch_xor_acquire(i, v);
1217#else
1218 return raw_atomic_fetch_xor_acquire(i, v);
1219#endif
884}1220}
885 1221
1222/**
1223 * raw_atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
1224 * @i: long value
1225 * @v: pointer to atomic_long_t
1226 *
1227 * Atomically updates @v to (@v ^ @i) with release ordering.
1228 *
1229 * Safe to use in noinstr code; prefer atomic_long_fetch_xor_release() elsewhere.
1230 *
1231 * Return: The original value of @v.
1232 */
886static __always_inline long1233static __always_inline long
887arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v)1234raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
888{1235{
889 return arch_atomic_fetch_xor_release(i, v);1236#ifdef CONFIG_64BIT
1237 return raw_atomic64_fetch_xor_release(i, v);
1238#else
1239 return raw_atomic_fetch_xor_release(i, v);
1240#endif
890}1241}
891 1242
1243/**
1244 * raw_atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1245 * @i: long value
1246 * @v: pointer to atomic_long_t
1247 *
1248 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1249 *
1250 * Safe to use in noinstr code; prefer atomic_long_fetch_xor_relaxed() elsewhere.
1251 *
1252 * Return: The original value of @v.
1253 */
892static __always_inline long1254static __always_inline long
893arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)1255raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
894{1256{
895 return arch_atomic_fetch_xor_relaxed(i, v);1257#ifdef CONFIG_64BIT
1258 return raw_atomic64_fetch_xor_relaxed(i, v);
1259#else
1260 return raw_atomic_fetch_xor_relaxed(i, v);
1261#endif
896}1262}
897 1263
1264/**
1265 * raw_atomic_long_xchg() - atomic exchange with full ordering
1266 * @v: pointer to atomic_long_t
1267 * @new: long value to assign
1268 *
1269 * Atomically updates @v to @new with full ordering.
1270 *
1271 * Safe to use in noinstr code; prefer atomic_long_xchg() elsewhere.
1272 *
1273 * Return: The original value of @v.
1274 */
898static __always_inline long1275static __always_inline long
899arch_atomic_long_xchg(atomic_long_t *v, long i)1276raw_atomic_long_xchg(atomic_long_t *v, long new)
900{1277{
901 return arch_atomic_xchg(v, i);1278#ifdef CONFIG_64BIT
1279 return raw_atomic64_xchg(v, new);
1280#else
1281 return raw_atomic_xchg(v, new);
1282#endif
902}1283}
903 1284
1285/**
1286 * raw_atomic_long_xchg_acquire() - atomic exchange with acquire ordering
1287 * @v: pointer to atomic_long_t
1288 * @new: long value to assign
1289 *
1290 * Atomically updates @v to @new with acquire ordering.
1291 *
1292 * Safe to use in noinstr code; prefer atomic_long_xchg_acquire() elsewhere.
1293 *
1294 * Return: The original value of @v.
1295 */
904static __always_inline long1296static __always_inline long
905arch_atomic_long_xchg_acquire(atomic_long_t *v, long i)1297raw_atomic_long_xchg_acquire(atomic_long_t *v, long new)
906{1298{
907 return arch_atomic_xchg_acquire(v, i);1299#ifdef CONFIG_64BIT
1300 return raw_atomic64_xchg_acquire(v, new);
1301#else
1302 return raw_atomic_xchg_acquire(v, new);
1303#endif
908}1304}
909 1305
1306/**
1307 * raw_atomic_long_xchg_release() - atomic exchange with release ordering
1308 * @v: pointer to atomic_long_t
1309 * @new: long value to assign
1310 *
1311 * Atomically updates @v to @new with release ordering.
1312 *
1313 * Safe to use in noinstr code; prefer atomic_long_xchg_release() elsewhere.
1314 *
1315 * Return: The original value of @v.
1316 */
910static __always_inline long1317static __always_inline long
911arch_atomic_long_xchg_release(atomic_long_t *v, long i)1318raw_atomic_long_xchg_release(atomic_long_t *v, long new)
912{1319{
913 return arch_atomic_xchg_release(v, i);1320#ifdef CONFIG_64BIT
1321 return raw_atomic64_xchg_release(v, new);
1322#else
1323 return raw_atomic_xchg_release(v, new);
1324#endif
914}1325}
915 1326
1327/**
1328 * raw_atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
1329 * @v: pointer to atomic_long_t
1330 * @new: long value to assign
1331 *
1332 * Atomically updates @v to @new with relaxed ordering.
1333 *
1334 * Safe to use in noinstr code; prefer atomic_long_xchg_relaxed() elsewhere.
1335 *
1336 * Return: The original value of @v.
1337 */
916static __always_inline long1338static __always_inline long
917arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i)1339raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new)
918{1340{
919 return arch_atomic_xchg_relaxed(v, i);1341#ifdef CONFIG_64BIT
1342 return raw_atomic64_xchg_relaxed(v, new);
1343#else
1344 return raw_atomic_xchg_relaxed(v, new);
1345#endif
920}1346}
921 1347
1348/**
1349 * raw_atomic_long_cmpxchg() - atomic compare and exchange with full ordering
1350 * @v: pointer to atomic_long_t
1351 * @old: long value to compare with
1352 * @new: long value to assign
1353 *
1354 * If (@v == @old), atomically updates @v to @new with full ordering.
1355 *
1356 * Safe to use in noinstr code; prefer atomic_long_cmpxchg() elsewhere.
1357 *
1358 * Return: The original value of @v.
1359 */
922static __always_inline long1360static __always_inline long
923arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)1361raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
924{1362{
925 return arch_atomic_cmpxchg(v, old, new);1363#ifdef CONFIG_64BIT
1364 return raw_atomic64_cmpxchg(v, old, new);
1365#else
1366 return raw_atomic_cmpxchg(v, old, new);
1367#endif
926}1368}
927 1369
1370/**
1371 * raw_atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1372 * @v: pointer to atomic_long_t
1373 * @old: long value to compare with
1374 * @new: long value to assign
1375 *
1376 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1377 *
1378 * Safe to use in noinstr code; prefer atomic_long_cmpxchg_acquire() elsewhere.
1379 *
1380 * Return: The original value of @v.
1381 */
928static __always_inline long1382static __always_inline long
929arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)1383raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
930{1384{
931 return arch_atomic_cmpxchg_acquire(v, old, new);1385#ifdef CONFIG_64BIT
1386 return raw_atomic64_cmpxchg_acquire(v, old, new);
1387#else
1388 return raw_atomic_cmpxchg_acquire(v, old, new);
1389#endif
932}1390}
933 1391
1392/**
1393 * raw_atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
1394 * @v: pointer to atomic_long_t
1395 * @old: long value to compare with
1396 * @new: long value to assign
1397 *
1398 * If (@v == @old), atomically updates @v to @new with release ordering.
1399 *
1400 * Safe to use in noinstr code; prefer atomic_long_cmpxchg_release() elsewhere.
1401 *
1402 * Return: The original value of @v.
1403 */
934static __always_inline long1404static __always_inline long
935arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)1405raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
936{1406{
937 return arch_atomic_cmpxchg_release(v, old, new);1407#ifdef CONFIG_64BIT
1408 return raw_atomic64_cmpxchg_release(v, old, new);
1409#else
1410 return raw_atomic_cmpxchg_release(v, old, new);
1411#endif
938}1412}
939 1413
1414/**
1415 * raw_atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1416 * @v: pointer to atomic_long_t
1417 * @old: long value to compare with
1418 * @new: long value to assign
1419 *
1420 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1421 *
1422 * Safe to use in noinstr code; prefer atomic_long_cmpxchg_relaxed() elsewhere.
1423 *
1424 * Return: The original value of @v.
1425 */
940static __always_inline long1426static __always_inline long
941arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)1427raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
942{1428{
943 return arch_atomic_cmpxchg_relaxed(v, old, new);1429#ifdef CONFIG_64BIT
1430 return raw_atomic64_cmpxchg_relaxed(v, old, new);
1431#else
1432 return raw_atomic_cmpxchg_relaxed(v, old, new);
1433#endif
944}1434}
945 1435
1436/**
1437 * raw_atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
1438 * @v: pointer to atomic_long_t
1439 * @old: pointer to long value to compare with
1440 * @new: long value to assign
1441 *
1442 * If (@v == @old), atomically updates @v to @new with full ordering.
1443 * Otherwise, updates @old to the current value of @v.
1444 *
1445 * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg() elsewhere.
1446 *
1447 * Return: @true if the exchange occured, @false otherwise.
1448 */
946static __always_inline bool1449static __always_inline bool
947arch_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)1450raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
948{1451{
949 return arch_atomic_try_cmpxchg(v, (int *)old, new);1452#ifdef CONFIG_64BIT
1453 return raw_atomic64_try_cmpxchg(v, (s64 *)old, new);
1454#else
1455 return raw_atomic_try_cmpxchg(v, (int *)old, new);
1456#endif
950}1457}
951 1458
1459/**
1460 * raw_atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
1461 * @v: pointer to atomic_long_t
1462 * @old: pointer to long value to compare with
1463 * @new: long value to assign
1464 *
1465 * If (@v == @old), atomically updates @v to @new with acquire ordering.
1466 * Otherwise, updates @old to the current value of @v.
1467 *
1468 * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_acquire() elsewhere.
1469 *
1470 * Return: @true if the exchange occured, @false otherwise.
1471 */
952static __always_inline bool1472static __always_inline bool
953arch_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)1473raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
954{1474{
955 return arch_atomic_try_cmpxchg_acquire(v, (int *)old, new);1475#ifdef CONFIG_64BIT
1476 return raw_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
1477#else
1478 return raw_atomic_try_cmpxchg_acquire(v, (int *)old, new);
1479#endif
956}1480}
957 1481
1482/**
1483 * raw_atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
1484 * @v: pointer to atomic_long_t
1485 * @old: pointer to long value to compare with
1486 * @new: long value to assign
1487 *
1488 * If (@v == @old), atomically updates @v to @new with release ordering.
1489 * Otherwise, updates @old to the current value of @v.
1490 *
1491 * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_release() elsewhere.
1492 *
1493 * Return: @true if the exchange occured, @false otherwise.
1494 */
958static __always_inline bool1495static __always_inline bool
959arch_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)1496raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
960{1497{
961 return arch_atomic_try_cmpxchg_release(v, (int *)old, new);1498#ifdef CONFIG_64BIT
1499 return raw_atomic64_try_cmpxchg_release(v, (s64 *)old, new);
1500#else
1501 return raw_atomic_try_cmpxchg_release(v, (int *)old, new);
1502#endif
962}1503}
963 1504
1505/**
1506 * raw_atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
1507 * @v: pointer to atomic_long_t
1508 * @old: pointer to long value to compare with
1509 * @new: long value to assign
1510 *
1511 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
1512 * Otherwise, updates @old to the current value of @v.
1513 *
1514 * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_relaxed() elsewhere.
1515 *
1516 * Return: @true if the exchange occured, @false otherwise.
1517 */
964static __always_inline bool1518static __always_inline bool
965arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)1519raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
966{1520{
967 return arch_atomic_try_cmpxchg_relaxed(v, (int *)old, new);1521#ifdef CONFIG_64BIT
1522 return raw_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
1523#else
1524 return raw_atomic_try_cmpxchg_relaxed(v, (int *)old, new);
1525#endif
968}1526}
969 1527
1528/**
1529 * raw_atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
1530 * @i: long value to add
1531 * @v: pointer to atomic_long_t
1532 *
1533 * Atomically updates @v to (@v - @i) with full ordering.
1534 *
1535 * Safe to use in noinstr code; prefer atomic_long_sub_and_test() elsewhere.
1536 *
1537 * Return: @true if the resulting value of @v is zero, @false otherwise.
1538 */
970static __always_inline bool1539static __always_inline bool
971arch_atomic_long_sub_and_test(long i, atomic_long_t *v)1540raw_atomic_long_sub_and_test(long i, atomic_long_t *v)
972{1541{
973 return arch_atomic_sub_and_test(i, v);1542#ifdef CONFIG_64BIT
1543 return raw_atomic64_sub_and_test(i, v);
1544#else
1545 return raw_atomic_sub_and_test(i, v);
1546#endif
974}1547}
975 1548
1549/**
1550 * raw_atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
1551 * @v: pointer to atomic_long_t
1552 *
1553 * Atomically updates @v to (@v - 1) with full ordering.
1554 *
1555 * Safe to use in noinstr code; prefer atomic_long_dec_and_test() elsewhere.
1556 *
1557 * Return: @true if the resulting value of @v is zero, @false otherwise.
1558 */
976static __always_inline bool1559static __always_inline bool
977arch_atomic_long_dec_and_test(atomic_long_t *v)1560raw_atomic_long_dec_and_test(atomic_long_t *v)
978{1561{
979 return arch_atomic_dec_and_test(v);1562#ifdef CONFIG_64BIT
1563 return raw_atomic64_dec_and_test(v);
1564#else
1565 return raw_atomic_dec_and_test(v);
1566#endif
980}1567}
981 1568
1569/**
1570 * raw_atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
1571 * @v: pointer to atomic_long_t
1572 *
1573 * Atomically updates @v to (@v + 1) with full ordering.
1574 *
1575 * Safe to use in noinstr code; prefer atomic_long_inc_and_test() elsewhere.
1576 *
1577 * Return: @true if the resulting value of @v is zero, @false otherwise.
1578 */
982static __always_inline bool1579static __always_inline bool
983arch_atomic_long_inc_and_test(atomic_long_t *v)1580raw_atomic_long_inc_and_test(atomic_long_t *v)
984{1581{
985 return arch_atomic_inc_and_test(v);1582#ifdef CONFIG_64BIT
1583 return raw_atomic64_inc_and_test(v);
1584#else
1585 return raw_atomic_inc_and_test(v);
1586#endif
986}1587}
987 1588
1589/**
1590 * raw_atomic_long_add_negative() - atomic add and test if negative with full ordering
1591 * @i: long value to add
1592 * @v: pointer to atomic_long_t
1593 *
1594 * Atomically updates @v to (@v + @i) with full ordering.
1595 *
1596 * Safe to use in noinstr code; prefer atomic_long_add_negative() elsewhere.
1597 *
1598 * Return: @true if the resulting value of @v is negative, @false otherwise.
1599 */
988static __always_inline bool1600static __always_inline bool
989arch_atomic_long_add_negative(long i, atomic_long_t *v)1601raw_atomic_long_add_negative(long i, atomic_long_t *v)
990{1602{
991 return arch_atomic_add_negative(i, v);1603#ifdef CONFIG_64BIT
1604 return raw_atomic64_add_negative(i, v);
1605#else
1606 return raw_atomic_add_negative(i, v);
1607#endif
992}1608}
993 1609
1610/**
1611 * raw_atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
1612 * @i: long value to add
1613 * @v: pointer to atomic_long_t
1614 *
1615 * Atomically updates @v to (@v + @i) with acquire ordering.
1616 *
1617 * Safe to use in noinstr code; prefer atomic_long_add_negative_acquire() elsewhere.
1618 *
1619 * Return: @true if the resulting value of @v is negative, @false otherwise.
1620 */
994static __always_inline bool1621static __always_inline bool
995arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)1622raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
996{1623{
997 return arch_atomic_add_negative_acquire(i, v);1624#ifdef CONFIG_64BIT
1625 return raw_atomic64_add_negative_acquire(i, v);
1626#else
1627 return raw_atomic_add_negative_acquire(i, v);
1628#endif
998}1629}
999 1630
1631/**
1632 * raw_atomic_long_add_negative_release() - atomic add and test if negative with release ordering
1633 * @i: long value to add
1634 * @v: pointer to atomic_long_t
1635 *
1636 * Atomically updates @v to (@v + @i) with release ordering.
1637 *
1638 * Safe to use in noinstr code; prefer atomic_long_add_negative_release() elsewhere.
1639 *
1640 * Return: @true if the resulting value of @v is negative, @false otherwise.
1641 */
1000static __always_inline bool1642static __always_inline bool
1001arch_atomic_long_add_negative_release(long i, atomic_long_t *v)1643raw_atomic_long_add_negative_release(long i, atomic_long_t *v)
1002{1644{
1003 return arch_atomic_add_negative_release(i, v);1645#ifdef CONFIG_64BIT
1646 return raw_atomic64_add_negative_release(i, v);
1647#else
1648 return raw_atomic_add_negative_release(i, v);
1649#endif
1004}1650}
1005 1651
1652/**
1653 * raw_atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
1654 * @i: long value to add
1655 * @v: pointer to atomic_long_t
1656 *
1657 * Atomically updates @v to (@v + @i) with relaxed ordering.
1658 *
1659 * Safe to use in noinstr code; prefer atomic_long_add_negative_relaxed() elsewhere.
1660 *
1661 * Return: @true if the resulting value of @v is negative, @false otherwise.
1662 */
1006static __always_inline bool1663static __always_inline bool
1007arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)1664raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1008{1665{
1009 return arch_atomic_add_negative_relaxed(i, v);1666#ifdef CONFIG_64BIT
1667 return raw_atomic64_add_negative_relaxed(i, v);
1668#else
1669 return raw_atomic_add_negative_relaxed(i, v);
1670#endif
1010}1671}
1011 1672
1673/**
1674 * raw_atomic_long_fetch_add_unless() - atomic add unless value with full ordering
1675 * @v: pointer to atomic_long_t
1676 * @a: long value to add
1677 * @u: long value to compare with
1678 *
1679 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1680 *
1681 * Safe to use in noinstr code; prefer atomic_long_fetch_add_unless() elsewhere.
1682 *
1683 * Return: The original value of @v.
1684 */
1012static __always_inline long1685static __always_inline long
1013arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)1686raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1014{1687{
1015 return arch_atomic_fetch_add_unless(v, a, u);1688#ifdef CONFIG_64BIT
1689 return raw_atomic64_fetch_add_unless(v, a, u);
1690#else
1691 return raw_atomic_fetch_add_unless(v, a, u);
1692#endif
1016}1693}
1017 1694
1695/**
1696 * raw_atomic_long_add_unless() - atomic add unless value with full ordering
1697 * @v: pointer to atomic_long_t
1698 * @a: long value to add
1699 * @u: long value to compare with
1700 *
1701 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
1702 *
1703 * Safe to use in noinstr code; prefer atomic_long_add_unless() elsewhere.
1704 *
1705 * Return: @true if @v was updated, @false otherwise.
1706 */
1018static __always_inline bool1707static __always_inline bool
1019arch_atomic_long_add_unless(atomic_long_t *v, long a, long u)1708raw_atomic_long_add_unless(atomic_long_t *v, long a, long u)
1020{1709{
1021 return arch_atomic_add_unless(v, a, u);1710#ifdef CONFIG_64BIT
1711 return raw_atomic64_add_unless(v, a, u);
1712#else
1713 return raw_atomic_add_unless(v, a, u);
1714#endif
1022}1715}
1023 1716
1717/**
1718 * raw_atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
1719 * @v: pointer to atomic_long_t
1720 *
1721 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
1722 *
1723 * Safe to use in noinstr code; prefer atomic_long_inc_not_zero() elsewhere.
1724 *
1725 * Return: @true if @v was updated, @false otherwise.
1726 */
1024static __always_inline bool1727static __always_inline bool
1025arch_atomic_long_inc_not_zero(atomic_long_t *v)1728raw_atomic_long_inc_not_zero(atomic_long_t *v)
1026{1729{
1027 return arch_atomic_inc_not_zero(v);1730#ifdef CONFIG_64BIT
1731 return raw_atomic64_inc_not_zero(v);
1732#else
1733 return raw_atomic_inc_not_zero(v);
1734#endif
1028}1735}
1029 1736
1737/**
1738 * raw_atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
1739 * @v: pointer to atomic_long_t
1740 *
1741 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
1742 *
1743 * Safe to use in noinstr code; prefer atomic_long_inc_unless_negative() elsewhere.
1744 *
1745 * Return: @true if @v was updated, @false otherwise.
1746 */
1030static __always_inline bool1747static __always_inline bool
1031arch_atomic_long_inc_unless_negative(atomic_long_t *v)1748raw_atomic_long_inc_unless_negative(atomic_long_t *v)
1032{1749{
1033 return arch_atomic_inc_unless_negative(v);1750#ifdef CONFIG_64BIT
1751 return raw_atomic64_inc_unless_negative(v);
1752#else
1753 return raw_atomic_inc_unless_negative(v);
1754#endif
1034}1755}
1035 1756
1757/**
1758 * raw_atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
1759 * @v: pointer to atomic_long_t
1760 *
1761 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
1762 *
1763 * Safe to use in noinstr code; prefer atomic_long_dec_unless_positive() elsewhere.
1764 *
1765 * Return: @true if @v was updated, @false otherwise.
1766 */
1036static __always_inline bool1767static __always_inline bool
1037arch_atomic_long_dec_unless_positive(atomic_long_t *v)1768raw_atomic_long_dec_unless_positive(atomic_long_t *v)
1038{1769{
1039 return arch_atomic_dec_unless_positive(v);1770#ifdef CONFIG_64BIT
1771 return raw_atomic64_dec_unless_positive(v);
1772#else
1773 return raw_atomic_dec_unless_positive(v);
1774#endif
1040}1775}
1041 1776
1777/**
1778 * raw_atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
1779 * @v: pointer to atomic_long_t
1780 *
1781 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
1782 *
1783 * Safe to use in noinstr code; prefer atomic_long_dec_if_positive() elsewhere.
1784 *
1785 * Return: The old value of (@v - 1), regardless of whether @v was updated.
1786 */
1042static __always_inline long1787static __always_inline long
1043arch_atomic_long_dec_if_positive(atomic_long_t *v)1788raw_atomic_long_dec_if_positive(atomic_long_t *v)
1044{1789{
1045 return arch_atomic_dec_if_positive(v);1790#ifdef CONFIG_64BIT
1791 return raw_atomic64_dec_if_positive(v);
1792#else
1793 return raw_atomic_dec_if_positive(v);
1794#endif
1046}1795}
1047 1796
1048#endif /* CONFIG_64BIT */
1049#endif /* _LINUX_ATOMIC_LONG_H */1797#endif /* _LINUX_ATOMIC_LONG_H */
1050// a194c07d7d2f4b0e178d3c118c919775d5d65f501798// 4ef23f98c73cff96d239896175fd26b10b88899e
include/linux/context_tracking.h+2−2
@@ -119,7 +119,7 @@ extern void ct_idle_exit(void); · 118 unmodified lines
119 */119 */
120static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)120static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
121{121{
122 return !(arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX);122 return !(raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX);
123}123}
124 124
125/*125/*
@@ -128,7 +128,7 @@ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) · 2 unmodified lines
128 */128 */
129static __always_inline unsigned long ct_state_inc(int incby)129static __always_inline unsigned long ct_state_inc(int incby)
130{130{
131 return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));131 return raw_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));
132}132}
133 133
134static __always_inline bool warn_rcu_enter(void)134static __always_inline bool warn_rcu_enter(void)
include/linux/context_tracking_state.h+1−1
@@ -51,7 +51,7 @@ DECLARE_PER_CPU(struct context_tracking, context_tracking); · 50 unmodified lines
51#ifdef CONFIG_CONTEXT_TRACKING_USER51#ifdef CONFIG_CONTEXT_TRACKING_USER
52static __always_inline int __ct_state(void)52static __always_inline int __ct_state(void)
53{53{
54 return arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;54 return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;
55}55}
56#endif56#endif
57 57
include/linux/cpumask.h+1−1
@@ -1071,7 +1071,7 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu) · 1070 unmodified lines
1071 */1071 */
1072static __always_inline unsigned int num_online_cpus(void)1072static __always_inline unsigned int num_online_cpus(void)
1073{1073{
1074 return arch_atomic_read(&__num_online_cpus);1074 return raw_atomic_read(&__num_online_cpus);
1075}1075}
1076#define num_possible_cpus() cpumask_weight(cpu_possible_mask)1076#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
1077#define num_present_cpus() cpumask_weight(cpu_present_mask)1077#define num_present_cpus() cpumask_weight(cpu_present_mask)
include/linux/dmar.h+66−59
@@ -202,67 +202,74 @@ static inline void detect_intel_iommu(void) · 201 unmodified lines
202 202
203struct irte {203struct irte {
204 union {204 union {
205 /* Shared between remapped and posted mode*/
206 struct {205 struct {
207 __u64 present : 1, /* 0 */206 union {
208 fpd : 1, /* 1 */207 /* Shared between remapped and posted mode*/
209 __res0 : 6, /* 2 - 6 */208 struct {
210 avail : 4, /* 8 - 11 */209 __u64 present : 1, /* 0 */
211 __res1 : 3, /* 12 - 14 */210 fpd : 1, /* 1 */
212 pst : 1, /* 15 */211 __res0 : 6, /* 2 - 6 */
213 vector : 8, /* 16 - 23 */212 avail : 4, /* 8 - 11 */
214 __res2 : 40; /* 24 - 63 */213 __res1 : 3, /* 12 - 14 */
214 pst : 1, /* 15 */
215 vector : 8, /* 16 - 23 */
216 __res2 : 40; /* 24 - 63 */
217 };
218
219 /* Remapped mode */
220 struct {
221 __u64 r_present : 1, /* 0 */
222 r_fpd : 1, /* 1 */
223 dst_mode : 1, /* 2 */
224 redir_hint : 1, /* 3 */
225 trigger_mode : 1, /* 4 */
226 dlvry_mode : 3, /* 5 - 7 */
227 r_avail : 4, /* 8 - 11 */
228 r_res0 : 4, /* 12 - 15 */
229 r_vector : 8, /* 16 - 23 */
230 r_res1 : 8, /* 24 - 31 */
231 dest_id : 32; /* 32 - 63 */
232 };
233
234 /* Posted mode */
235 struct {
236 __u64 p_present : 1, /* 0 */
237 p_fpd : 1, /* 1 */
238 p_res0 : 6, /* 2 - 7 */
239 p_avail : 4, /* 8 - 11 */
240 p_res1 : 2, /* 12 - 13 */
241 p_urgent : 1, /* 14 */
242 p_pst : 1, /* 15 */
243 p_vector : 8, /* 16 - 23 */
244 p_res2 : 14, /* 24 - 37 */
245 pda_l : 26; /* 38 - 63 */
246 };
247 __u64 low;
248 };
249
250 union {
251 /* Shared between remapped and posted mode*/
252 struct {
253 __u64 sid : 16, /* 64 - 79 */
254 sq : 2, /* 80 - 81 */
255 svt : 2, /* 82 - 83 */
256 __res3 : 44; /* 84 - 127 */
257 };
258
259 /* Posted mode*/
260 struct {
261 __u64 p_sid : 16, /* 64 - 79 */
262 p_sq : 2, /* 80 - 81 */
263 p_svt : 2, /* 82 - 83 */
264 p_res3 : 12, /* 84 - 95 */
265 pda_h : 32; /* 96 - 127 */
266 };
267 __u64 high;
268 };
215 };269 };
216 270#ifdef CONFIG_IRQ_REMAP
217 /* Remapped mode */271 __u128 irte;
218 struct {272#endif
219 __u64 r_present : 1, /* 0 */
220 r_fpd : 1, /* 1 */
221 dst_mode : 1, /* 2 */
222 redir_hint : 1, /* 3 */
223 trigger_mode : 1, /* 4 */
224 dlvry_mode : 3, /* 5 - 7 */
225 r_avail : 4, /* 8 - 11 */
226 r_res0 : 4, /* 12 - 15 */
227 r_vector : 8, /* 16 - 23 */
228 r_res1 : 8, /* 24 - 31 */
229 dest_id : 32; /* 32 - 63 */
230 };
231
232 /* Posted mode */
233 struct {
234 __u64 p_present : 1, /* 0 */
235 p_fpd : 1, /* 1 */
236 p_res0 : 6, /* 2 - 7 */
237 p_avail : 4, /* 8 - 11 */
238 p_res1 : 2, /* 12 - 13 */
239 p_urgent : 1, /* 14 */
240 p_pst : 1, /* 15 */
241 p_vector : 8, /* 16 - 23 */
242 p_res2 : 14, /* 24 - 37 */
243 pda_l : 26; /* 38 - 63 */
244 };
245 __u64 low;
246 };
247
248 union {
249 /* Shared between remapped and posted mode*/
250 struct {
251 __u64 sid : 16, /* 64 - 79 */
252 sq : 2, /* 80 - 81 */
253 svt : 2, /* 82 - 83 */
254 __res3 : 44; /* 84 - 127 */
255 };
256
257 /* Posted mode*/
258 struct {
259 __u64 p_sid : 16, /* 64 - 79 */
260 p_sq : 2, /* 80 - 81 */
261 p_svt : 2, /* 82 - 83 */
262 p_res3 : 12, /* 84 - 95 */
263 pda_h : 32; /* 96 - 127 */
264 };
265 __u64 high;
266 };273 };
267};274};
268 275
include/linux/jump_label.h+1−1
@@ -257,7 +257,7 @@ extern enum jump_label_type jump_label_init_type(struct jump_entry *entry); · 256 unmodified lines
257 257
258static __always_inline int static_key_count(struct static_key *key)258static __always_inline int static_key_count(struct static_key *key)
259{259{
260 return arch_atomic_read(&key->enabled);260 return raw_atomic_read(&key->enabled);
261}261}
262 262
263static __always_inline void jump_label_init(void)263static __always_inline void jump_label_init(void)
include/linux/lockdep.h+8−0
@@ -447,6 +447,14 @@ extern int lockdep_is_held(const void *); · 446 unmodified lines
447 447
448#endif /* !LOCKDEP */448#endif /* !LOCKDEP */
449 449
450#ifdef CONFIG_PROVE_LOCKING
451void lockdep_set_lock_cmp_fn(struct lockdep_map *, lock_cmp_fn, lock_print_fn);
452
453#define lock_set_cmp_fn(lock, ...) lockdep_set_lock_cmp_fn(&(lock)->dep_map, __VA_ARGS__)
454#else
455#define lock_set_cmp_fn(lock, ...) do { } while (0)
456#endif
457
450enum xhlock_context_t {458enum xhlock_context_t {
451 XHLOCK_HARD,459 XHLOCK_HARD,
452 XHLOCK_SOFT,460 XHLOCK_SOFT,
include/linux/lockdep_types.h+8−0
@@ -85,6 +85,11 @@ struct lock_trace; · 84 unmodified lines
85 85
86#define LOCKSTAT_POINTS 486#define LOCKSTAT_POINTS 4
87 87
88struct lockdep_map;
89typedef int (*lock_cmp_fn)(const struct lockdep_map *a,
90 const struct lockdep_map *b);
91typedef void (*lock_print_fn)(const struct lockdep_map *map);
92
88/*93/*
89 * The lock-class itself. The order of the structure members matters.94 * The lock-class itself. The order of the structure members matters.
90 * reinit_class() zeroes the key member and all subsequent members.95 * reinit_class() zeroes the key member and all subsequent members.
@@ -110,6 +115,9 @@ struct lock_class { · 19 unmodified lines
110 struct list_head locks_after, locks_before;115 struct list_head locks_after, locks_before;
111 116
112 const struct lockdep_subclass_key *key;117 const struct lockdep_subclass_key *key;
118 lock_cmp_fn cmp_fn;
119 lock_print_fn print_fn;
120
113 unsigned int subclass;121 unsigned int subclass;
114 unsigned int dep_gen_id;122 unsigned int dep_gen_id;
115 123
include/linux/percpu-defs.h+13−32
@@ -343,31 +343,19 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } · 342 unmodified lines
343 pscr2_ret__; \343 pscr2_ret__; \
344})344})
345 345
346/*346#define __pcpu_size_call_return2bool(stem, variable, ...) \
347 * Special handling for cmpxchg_double. cmpxchg_double is passed two
348 * percpu variables. The first has to be aligned to a double word
349 * boundary and the second has to follow directly thereafter.
350 * We enforce this on all architectures even if they don't support
351 * a double cmpxchg instruction, since it's a cheap requirement, and it
352 * avoids breaking the requirement for architectures with the instruction.
353 */
354#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
355({ \347({ \
356 bool pdcrb_ret__; \348 bool pscr2_ret__; \
357 __verify_pcpu_ptr(&(pcp1)); \349 __verify_pcpu_ptr(&(variable)); \
358 BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \350 switch(sizeof(variable)) { \
359 VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \351 case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
360 VM_BUG_ON((unsigned long)(&(pcp2)) != \352 case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
361 (unsigned long)(&(pcp1)) + sizeof(pcp1)); \353 case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
362 switch(sizeof(pcp1)) { \354 case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
363 case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
364 case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
365 case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
366 case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
367 default: \355 default: \
368 __bad_size_call_parameter(); break; \356 __bad_size_call_parameter(); break; \
369 } \357 } \
370 pdcrb_ret__; \358 pscr2_ret__; \
371})359})
372 360
373#define __pcpu_size_call(stem, variable, ...) \361#define __pcpu_size_call(stem, variable, ...) \
@@ -426,9 +414,8 @@ do { \ · 52 unmodified lines
426#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)414#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
427#define raw_cpu_cmpxchg(pcp, oval, nval) \415#define raw_cpu_cmpxchg(pcp, oval, nval) \
428 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)416 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
429#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \417#define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \
430 __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)418 __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)
431
432#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))419#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
433#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)420#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
434#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)421#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
@@ -488,11 +475,6 @@ do { \ · 53 unmodified lines
488 raw_cpu_cmpxchg(pcp, oval, nval); \475 raw_cpu_cmpxchg(pcp, oval, nval); \
489})476})
490 477
491#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
492({ __this_cpu_preempt_check("cmpxchg_double"); \
493 raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
494})
495
496#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))478#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
497#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)479#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
498#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)480#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
@@ -513,9 +495,8 @@ do { \ · 14 unmodified lines
513#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)495#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
514#define this_cpu_cmpxchg(pcp, oval, nval) \496#define this_cpu_cmpxchg(pcp, oval, nval) \
515 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)497 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
516#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \498#define this_cpu_try_cmpxchg(pcp, ovalp, nval) \
517 __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)499 __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)
518
519#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))500#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
520#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)501#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
521#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)502#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
include/linux/slub_def.h+9−3
@@ -39,7 +39,8 @@ enum stat_item { · 38 unmodified lines
39 CPU_PARTIAL_FREE, /* Refill cpu partial on free */39 CPU_PARTIAL_FREE, /* Refill cpu partial on free */
40 CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */40 CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */
41 CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */41 CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */
42 NR_SLUB_STAT_ITEMS };42 NR_SLUB_STAT_ITEMS
43};
43 44
44#ifndef CONFIG_SLUB_TINY45#ifndef CONFIG_SLUB_TINY
45/*46/*
@@ -47,8 +48,13 @@ enum stat_item { · 1 unmodified line
47 * with this_cpu_cmpxchg_double() alignment requirements.48 * with this_cpu_cmpxchg_double() alignment requirements.
48 */49 */
49struct kmem_cache_cpu {50struct kmem_cache_cpu {
50 void **freelist; /* Pointer to next available object */51 union {
51 unsigned long tid; /* Globally unique transaction id */52 struct {
53 void **freelist; /* Pointer to next available object */
54 unsigned long tid; /* Globally unique transaction id */
55 };
56 freelist_aba_t freelist_tid;
57 };
52 struct slab *slab; /* The slab from which we are allocating */58 struct slab *slab; /* The slab from which we are allocating */
53#ifdef CONFIG_SLUB_CPU_PARTIAL59#ifdef CONFIG_SLUB_CPU_PARTIAL
54 struct slab *partial; /* Partially allocated frozen slabs */60 struct slab *partial; /* Partially allocated frozen slabs */
include/linux/types.h+5−0
@@ -10,6 +10,11 @@ · 9 unmodified lines
10#define DECLARE_BITMAP(name,bits) \10#define DECLARE_BITMAP(name,bits) \
11 unsigned long name[BITS_TO_LONGS(bits)]11 unsigned long name[BITS_TO_LONGS(bits)]
12 12
13#ifdef __SIZEOF_INT128__
14typedef __s128 s128;
15typedef __u128 u128;
16#endif
17
13typedef u32 __kernel_dev_t;18typedef u32 __kernel_dev_t;
14 19
15typedef __kernel_fd_set fd_set;20typedef __kernel_fd_set fd_set;
include/uapi/linux/types.h+4−0
@@ -13,6 +13,10 @@ · 12 unmodified lines
13 13
14#include <linux/posix_types.h>14#include <linux/posix_types.h>
15 15
16#ifdef __SIZEOF_INT128__
17typedef __signed__ __int128 __s128 __attribute__((aligned(16)));
18typedef unsigned __int128 __u128 __attribute__((aligned(16)));
19#endif
16 20
17/*21/*
18 * Below are truly Linux-specific types that should never collide with22 * Below are truly Linux-specific types that should never collide with
kernel/context_tracking.c+6−6
@@ -510,7 +510,7 @@ void noinstr __ct_user_enter(enum ctx_state state) · 509 unmodified lines
510 * In this we case we don't care about any concurrency/ordering.510 * In this we case we don't care about any concurrency/ordering.
511 */511 */
512 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE))512 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE))
513 arch_atomic_set(&ct->state, state);513 raw_atomic_set(&ct->state, state);
514 } else {514 } else {
515 /*515 /*
516 * Even if context tracking is disabled on this CPU, because it's outside516 * Even if context tracking is disabled on this CPU, because it's outside
@@ -527,7 +527,7 @@ void noinstr __ct_user_enter(enum ctx_state state) · 10 unmodified lines
527 */527 */
528 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) {528 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) {
529 /* Tracking for vtime only, no concurrent RCU EQS accounting */529 /* Tracking for vtime only, no concurrent RCU EQS accounting */
530 arch_atomic_set(&ct->state, state);530 raw_atomic_set(&ct->state, state);
531 } else {531 } else {
532 /*532 /*
533 * Tracking for vtime and RCU EQS. Make sure we don't race533 * Tracking for vtime and RCU EQS. Make sure we don't race
@@ -535,7 +535,7 @@ void noinstr __ct_user_enter(enum ctx_state state) · 1 unmodified line
535 * RCU only requires RCU_DYNTICKS_IDX increments to be fully535 * RCU only requires RCU_DYNTICKS_IDX increments to be fully
536 * ordered.536 * ordered.
537 */537 */
538 arch_atomic_add(state, &ct->state);538 raw_atomic_add(state, &ct->state);
539 }539 }
540 }540 }
541 }541 }
@@ -630,12 +630,12 @@ void noinstr __ct_user_exit(enum ctx_state state) · 88 unmodified lines
630 * In this we case we don't care about any concurrency/ordering.630 * In this we case we don't care about any concurrency/ordering.
631 */631 */
632 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE))632 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE))
633 arch_atomic_set(&ct->state, CONTEXT_KERNEL);633 raw_atomic_set(&ct->state, CONTEXT_KERNEL);
634 634
635 } else {635 } else {
636 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) {636 if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) {
637 /* Tracking for vtime only, no concurrent RCU EQS accounting */637 /* Tracking for vtime only, no concurrent RCU EQS accounting */
638 arch_atomic_set(&ct->state, CONTEXT_KERNEL);638 raw_atomic_set(&ct->state, CONTEXT_KERNEL);
639 } else {639 } else {
640 /*640 /*
641 * Tracking for vtime and RCU EQS. Make sure we don't race641 * Tracking for vtime and RCU EQS. Make sure we don't race
@@ -643,7 +643,7 @@ void noinstr __ct_user_exit(enum ctx_state state) · 1 unmodified line
643 * RCU only requires RCU_DYNTICKS_IDX increments to be fully643 * RCU only requires RCU_DYNTICKS_IDX increments to be fully
644 * ordered.644 * ordered.
645 */645 */
646 arch_atomic_sub(state, &ct->state);646 raw_atomic_sub(state, &ct->state);
647 }647 }
648 }648 }
649 }649 }
kernel/locking/lockdep.c+87−31
@@ -709,7 +709,7 @@ void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS]) · 708 unmodified lines
709 usage[i] = '\0';709 usage[i] = '\0';
710}710}
711 711
712static void __print_lock_name(struct lock_class *class)712static void __print_lock_name(struct held_lock *hlock, struct lock_class *class)
713{713{
714 char str[KSYM_NAME_LEN];714 char str[KSYM_NAME_LEN];
715 const char *name;715 const char *name;
@@ -724,17 +724,19 @@ static void __print_lock_name(struct lock_class *class) · 8 unmodified lines
724 printk(KERN_CONT "#%d", class->name_version);724 printk(KERN_CONT "#%d", class->name_version);
725 if (class->subclass)725 if (class->subclass)
726 printk(KERN_CONT "/%d", class->subclass);726 printk(KERN_CONT "/%d", class->subclass);
727 if (hlock && class->print_fn)
728 class->print_fn(hlock->instance);
727 }729 }
728}730}
729 731
730static void print_lock_name(struct lock_class *class)732static void print_lock_name(struct held_lock *hlock, struct lock_class *class)
731{733{
732 char usage[LOCK_USAGE_CHARS];734 char usage[LOCK_USAGE_CHARS];
733 735
734 get_usage_chars(class, usage);736 get_usage_chars(class, usage);
735 737
736 printk(KERN_CONT " (");738 printk(KERN_CONT " (");
737 __print_lock_name(class);739 __print_lock_name(hlock, class);
738 printk(KERN_CONT "){%s}-{%d:%d}", usage,740 printk(KERN_CONT "){%s}-{%d:%d}", usage,
739 class->wait_type_outer ?: class->wait_type_inner,741 class->wait_type_outer ?: class->wait_type_inner,
740 class->wait_type_inner);742 class->wait_type_inner);
@@ -772,7 +774,7 @@ static void print_lock(struct held_lock *hlock) · 31 unmodified lines
772 }774 }
773 775
774 printk(KERN_CONT "%px", hlock->instance);776 printk(KERN_CONT "%px", hlock->instance);
775 print_lock_name(lock);777 print_lock_name(hlock, lock);
776 printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);778 printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);
777}779}
778 780
@@ -1868,7 +1870,7 @@ print_circular_bug_entry(struct lock_list *target, int depth) · 1089 unmodified lines
1868 if (debug_locks_silent)1870 if (debug_locks_silent)
1869 return;1871 return;
1870 printk("\n-> #%u", depth);1872 printk("\n-> #%u", depth);
1871 print_lock_name(target->class);1873 print_lock_name(NULL, target->class);
1872 printk(KERN_CONT ":\n");1874 printk(KERN_CONT ":\n");
1873 print_lock_trace(target->trace, 6);1875 print_lock_trace(target->trace, 6);
1874}1876}
@@ -1899,11 +1901,11 @@ print_circular_lock_scenario(struct held_lock *src, · 24 unmodified lines
1899 */1901 */
1900 if (parent != source) {1902 if (parent != source) {
1901 printk("Chain exists of:\n ");1903 printk("Chain exists of:\n ");
1902 __print_lock_name(source);1904 __print_lock_name(src, source);
1903 printk(KERN_CONT " --> ");1905 printk(KERN_CONT " --> ");
1904 __print_lock_name(parent);1906 __print_lock_name(NULL, parent);
1905 printk(KERN_CONT " --> ");1907 printk(KERN_CONT " --> ");
1906 __print_lock_name(target);1908 __print_lock_name(tgt, target);
1907 printk(KERN_CONT "\n\n");1909 printk(KERN_CONT "\n\n");
1908 }1910 }
1909 1911
@@ -1914,13 +1916,13 @@ print_circular_lock_scenario(struct held_lock *src, · 4 unmodified lines
1914 printk(" rlock(");1916 printk(" rlock(");
1915 else1917 else
1916 printk(" lock(");1918 printk(" lock(");
1917 __print_lock_name(target);1919 __print_lock_name(tgt, target);
1918 printk(KERN_CONT ");\n");1920 printk(KERN_CONT ");\n");
1919 printk(" lock(");1921 printk(" lock(");
1920 __print_lock_name(parent);1922 __print_lock_name(NULL, parent);
1921 printk(KERN_CONT ");\n");1923 printk(KERN_CONT ");\n");
1922 printk(" lock(");1924 printk(" lock(");
1923 __print_lock_name(target);1925 __print_lock_name(tgt, target);
1924 printk(KERN_CONT ");\n");1926 printk(KERN_CONT ");\n");
1925 if (src_read != 0)1927 if (src_read != 0)
1926 printk(" rlock(");1928 printk(" rlock(");
@@ -1928,7 +1930,7 @@ print_circular_lock_scenario(struct held_lock *src, · 1 unmodified line
1928 printk(" sync(");1930 printk(" sync(");
1929 else1931 else
1930 printk(" lock(");1932 printk(" lock(");
1931 __print_lock_name(source);1933 __print_lock_name(src, source);
1932 printk(KERN_CONT ");\n");1934 printk(KERN_CONT ");\n");
1933 printk("\n *** DEADLOCK ***\n\n");1935 printk("\n *** DEADLOCK ***\n\n");
1934}1936}
@@ -2154,6 +2156,8 @@ check_path(struct held_lock *target, struct lock_list *src_entry, · 219 unmodified lines
2154 return ret;2156 return ret;
2155}2157}
2156 2158
2159static void print_deadlock_bug(struct task_struct *, struct held_lock *, struct held_lock *);
2160
2157/*2161/*
2158 * Prove that the dependency graph starting at <src> can not2162 * Prove that the dependency graph starting at <src> can not
2159 * lead to <target>. If it can, there is a circle when adding2163 * lead to <target>. If it can, there is a circle when adding
@@ -2185,7 +2189,10 @@ check_noncircular(struct held_lock *src, struct held_lock *target, · 25 unmodified lines
2185 *trace = save_trace();2189 *trace = save_trace();
2186 }2190 }
2187 2191
2188 print_circular_bug(&src_entry, target_entry, src, target);2192 if (src->class_idx == target->class_idx)
2193 print_deadlock_bug(current, src, target);
2194 else
2195 print_circular_bug(&src_entry, target_entry, src, target);
2189 }2196 }
2190 2197
2191 return ret;2198 return ret;
@@ -2346,7 +2353,7 @@ static void print_lock_class_header(struct lock_class *class, int depth) · 154 unmodified lines
2346 int bit;2353 int bit;
2347 2354
2348 printk("%*s->", depth, "");2355 printk("%*s->", depth, "");
2349 print_lock_name(class);2356 print_lock_name(NULL, class);
2350#ifdef CONFIG_DEBUG_LOCKDEP2357#ifdef CONFIG_DEBUG_LOCKDEP
2351 printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));2358 printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));
2352#endif2359#endif
@@ -2528,11 +2535,11 @@ print_irq_lock_scenario(struct lock_list *safe_entry, · 175 unmodified lines
2528 */2535 */
2529 if (middle_class != unsafe_class) {2536 if (middle_class != unsafe_class) {
2530 printk("Chain exists of:\n ");2537 printk("Chain exists of:\n ");
2531 __print_lock_name(safe_class);2538 __print_lock_name(NULL, safe_class);
2532 printk(KERN_CONT " --> ");2539 printk(KERN_CONT " --> ");
2533 __print_lock_name(middle_class);2540 __print_lock_name(NULL, middle_class);
2534 printk(KERN_CONT " --> ");2541 printk(KERN_CONT " --> ");
2535 __print_lock_name(unsafe_class);2542 __print_lock_name(NULL, unsafe_class);
2536 printk(KERN_CONT "\n\n");2543 printk(KERN_CONT "\n\n");
2537 }2544 }
2538 2545
@@ -2540,18 +2547,18 @@ print_irq_lock_scenario(struct lock_list *safe_entry, · 1 unmodified line
2540 printk(" CPU0 CPU1\n");2547 printk(" CPU0 CPU1\n");
2541 printk(" ---- ----\n");2548 printk(" ---- ----\n");
2542 printk(" lock(");2549 printk(" lock(");
2543 __print_lock_name(unsafe_class);2550 __print_lock_name(NULL, unsafe_class);
2544 printk(KERN_CONT ");\n");2551 printk(KERN_CONT ");\n");
2545 printk(" local_irq_disable();\n");2552 printk(" local_irq_disable();\n");
2546 printk(" lock(");2553 printk(" lock(");
2547 __print_lock_name(safe_class);2554 __print_lock_name(NULL, safe_class);
2548 printk(KERN_CONT ");\n");2555 printk(KERN_CONT ");\n");
2549 printk(" lock(");2556 printk(" lock(");
2550 __print_lock_name(middle_class);2557 __print_lock_name(NULL, middle_class);
2551 printk(KERN_CONT ");\n");2558 printk(KERN_CONT ");\n");
2552 printk(" <Interrupt>\n");2559 printk(" <Interrupt>\n");
2553 printk(" lock(");2560 printk(" lock(");
2554 __print_lock_name(safe_class);2561 __print_lock_name(NULL, safe_class);
2555 printk(KERN_CONT ");\n");2562 printk(KERN_CONT ");\n");
2556 printk("\n *** DEADLOCK ***\n\n");2563 printk("\n *** DEADLOCK ***\n\n");
2557}2564}
@@ -2588,20 +2595,20 @@ print_bad_irq_dependency(struct task_struct *curr, · 30 unmodified lines
2588 pr_warn("\nand this task is already holding:\n");2595 pr_warn("\nand this task is already holding:\n");
2589 print_lock(prev);2596 print_lock(prev);
2590 pr_warn("which would create a new lock dependency:\n");2597 pr_warn("which would create a new lock dependency:\n");
2591 print_lock_name(hlock_class(prev));2598 print_lock_name(prev, hlock_class(prev));
2592 pr_cont(" ->");2599 pr_cont(" ->");
2593 print_lock_name(hlock_class(next));2600 print_lock_name(next, hlock_class(next));
2594 pr_cont("\n");2601 pr_cont("\n");
2595 2602
2596 pr_warn("\nbut this new dependency connects a %s-irq-safe lock:\n",2603 pr_warn("\nbut this new dependency connects a %s-irq-safe lock:\n",
2597 irqclass);2604 irqclass);
2598 print_lock_name(backwards_entry->class);2605 print_lock_name(NULL, backwards_entry->class);
2599 pr_warn("\n... which became %s-irq-safe at:\n", irqclass);2606 pr_warn("\n... which became %s-irq-safe at:\n", irqclass);
2600 2607
2601 print_lock_trace(backwards_entry->class->usage_traces[bit1], 1);2608 print_lock_trace(backwards_entry->class->usage_traces[bit1], 1);
2602 2609
2603 pr_warn("\nto a %s-irq-unsafe lock:\n", irqclass);2610 pr_warn("\nto a %s-irq-unsafe lock:\n", irqclass);
2604 print_lock_name(forwards_entry->class);2611 print_lock_name(NULL, forwards_entry->class);
2605 pr_warn("\n... which became %s-irq-unsafe at:\n", irqclass);2612 pr_warn("\n... which became %s-irq-unsafe at:\n", irqclass);
2606 pr_warn("...");2613 pr_warn("...");
2607 2614
@@ -2971,10 +2978,10 @@ print_deadlock_scenario(struct held_lock *nxt, struct held_lock *prv) · 363 unmodified lines
2971 printk(" CPU0\n");2978 printk(" CPU0\n");
2972 printk(" ----\n");2979 printk(" ----\n");
2973 printk(" lock(");2980 printk(" lock(");
2974 __print_lock_name(prev);2981 __print_lock_name(prv, prev);
2975 printk(KERN_CONT ");\n");2982 printk(KERN_CONT ");\n");
2976 printk(" lock(");2983 printk(" lock(");
2977 __print_lock_name(next);2984 __print_lock_name(nxt, next);
2978 printk(KERN_CONT ");\n");2985 printk(KERN_CONT ");\n");
2979 printk("\n *** DEADLOCK ***\n\n");2986 printk("\n *** DEADLOCK ***\n\n");
2980 printk(" May be due to missing lock nesting notation\n\n");2987 printk(" May be due to missing lock nesting notation\n\n");
@@ -2984,6 +2991,8 @@ static void · 3 unmodified lines
2984print_deadlock_bug(struct task_struct *curr, struct held_lock *prev,2991print_deadlock_bug(struct task_struct *curr, struct held_lock *prev,
2985 struct held_lock *next)2992 struct held_lock *next)
2986{2993{
2994 struct lock_class *class = hlock_class(prev);
2995
2987 if (!debug_locks_off_graph_unlock() || debug_locks_silent)2996 if (!debug_locks_off_graph_unlock() || debug_locks_silent)
2988 return;2997 return;
2989 2998
@@ -2998,6 +3007,11 @@ print_deadlock_bug(struct task_struct *curr, struct held_lock *prev, · 8 unmodified lines
2998 pr_warn("\nbut task is already holding lock:\n");3007 pr_warn("\nbut task is already holding lock:\n");
2999 print_lock(prev);3008 print_lock(prev);
3000 3009
3010 if (class->cmp_fn) {
3011 pr_warn("and the lock comparison function returns %i:\n",
3012 class->cmp_fn(prev->instance, next->instance));
3013 }
3014
3001 pr_warn("\nother info that might help us debug this:\n");3015 pr_warn("\nother info that might help us debug this:\n");
3002 print_deadlock_scenario(next, prev);3016 print_deadlock_scenario(next, prev);
3003 lockdep_print_held_locks(curr);3017 lockdep_print_held_locks(curr);
@@ -3019,6 +3033,7 @@ print_deadlock_bug(struct task_struct *curr, struct held_lock *prev, · 15 unmodified lines
3019static int3033static int
3020check_deadlock(struct task_struct *curr, struct held_lock *next)3034check_deadlock(struct task_struct *curr, struct held_lock *next)
3021{3035{
3036 struct lock_class *class;
3022 struct held_lock *prev;3037 struct held_lock *prev;
3023 struct held_lock *nest = NULL;3038 struct held_lock *nest = NULL;
3024 int i;3039 int i;
@@ -3039,6 +3054,12 @@ check_deadlock(struct task_struct *curr, struct held_lock *next) · 14 unmodified lines
3039 if ((next->read == 2) && prev->read)3054 if ((next->read == 2) && prev->read)
3040 continue;3055 continue;
3041 3056
3057 class = hlock_class(prev);
3058
3059 if (class->cmp_fn &&
3060 class->cmp_fn(prev->instance, next->instance) < 0)
3061 continue;
3062
3042 /*3063 /*
3043 * We're holding the nest_lock, which serializes this lock's3064 * We're holding the nest_lock, which serializes this lock's
3044 * nesting behaviour.3065 * nesting behaviour.
@@ -3100,6 +3121,14 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev, · 55 unmodified lines
3100 return 2;3121 return 2;
3101 }3122 }
3102 3123
3124 if (prev->class_idx == next->class_idx) {
3125 struct lock_class *class = hlock_class(prev);
3126
3127 if (class->cmp_fn &&
3128 class->cmp_fn(prev->instance, next->instance) < 0)
3129 return 2;
3130 }
3131
3103 /*3132 /*
3104 * Prove that the new <prev> -> <next> dependency would not3133 * Prove that the new <prev> -> <next> dependency would not
3105 * create a circular dependency in the graph. (We do this by3134 * create a circular dependency in the graph. (We do this by
@@ -3576,7 +3605,7 @@ static void print_chain_keys_chain(struct lock_chain *chain) · 470 unmodified lines
3576 hlock_id = chain_hlocks[chain->base + i];3605 hlock_id = chain_hlocks[chain->base + i];
3577 chain_key = print_chain_key_iteration(hlock_id, chain_key);3606 chain_key = print_chain_key_iteration(hlock_id, chain_key);
3578 3607
3579 print_lock_name(lock_classes + chain_hlock_class_idx(hlock_id));3608 print_lock_name(NULL, lock_classes + chain_hlock_class_idx(hlock_id));
3580 printk("\n");3609 printk("\n");
3581 }3610 }
3582}3611}
@@ -3933,11 +3962,11 @@ static void print_usage_bug_scenario(struct held_lock *lock) · 350 unmodified lines
3933 printk(" CPU0\n");3962 printk(" CPU0\n");
3934 printk(" ----\n");3963 printk(" ----\n");
3935 printk(" lock(");3964 printk(" lock(");
3936 __print_lock_name(class);3965 __print_lock_name(lock, class);
3937 printk(KERN_CONT ");\n");3966 printk(KERN_CONT ");\n");
3938 printk(" <Interrupt>\n");3967 printk(" <Interrupt>\n");
3939 printk(" lock(");3968 printk(" lock(");
3940 __print_lock_name(class);3969 __print_lock_name(lock, class);
3941 printk(KERN_CONT ");\n");3970 printk(KERN_CONT ");\n");
3942 printk("\n *** DEADLOCK ***\n\n");3971 printk("\n *** DEADLOCK ***\n\n");
3943}3972}
@@ -4023,7 +4052,7 @@ print_irq_inversion_bug(struct task_struct *curr, · 79 unmodified lines
4023 pr_warn("but this lock took another, %s-unsafe lock in the past:\n", irqclass);4052 pr_warn("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
4024 else4053 else
4025 pr_warn("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);4054 pr_warn("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
4026 print_lock_name(other->class);4055 print_lock_name(NULL, other->class);
4027 pr_warn("\n\nand interrupts could create inverse lock ordering between them.\n\n");4056 pr_warn("\n\nand interrupts could create inverse lock ordering between them.\n\n");
4028 4057
4029 pr_warn("\nother info that might help us debug this:\n");4058 pr_warn("\nother info that might help us debug this:\n");
@@ -4896,6 +4925,33 @@ EXPORT_SYMBOL_GPL(lockdep_init_map_type); · 866 unmodified lines
4896struct lock_class_key __lockdep_no_validate__;4925struct lock_class_key __lockdep_no_validate__;
4897EXPORT_SYMBOL_GPL(__lockdep_no_validate__);4926EXPORT_SYMBOL_GPL(__lockdep_no_validate__);
4898 4927
4928#ifdef CONFIG_PROVE_LOCKING
4929void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn,
4930 lock_print_fn print_fn)
4931{
4932 struct lock_class *class = lock->class_cache[0];
4933 unsigned long flags;
4934
4935 raw_local_irq_save(flags);
4936 lockdep_recursion_inc();
4937
4938 if (!class)
4939 class = register_lock_class(lock, 0, 0);
4940
4941 if (class) {
4942 WARN_ON(class->cmp_fn && class->cmp_fn != cmp_fn);
4943 WARN_ON(class->print_fn && class->print_fn != print_fn);
4944
4945 class->cmp_fn = cmp_fn;
4946 class->print_fn = print_fn;
4947 }
4948
4949 lockdep_recursion_finish();
4950 raw_local_irq_restore(flags);
4951}
4952EXPORT_SYMBOL_GPL(lockdep_set_lock_cmp_fn);
4953#endif
4954
4899static void4955static void
4900print_lock_nested_lock_not_held(struct task_struct *curr,4956print_lock_nested_lock_not_held(struct task_struct *curr,
4901 struct held_lock *hlock)4957 struct held_lock *hlock)
kernel/sched/clock.c+1−1
@@ -287,7 +287,7 @@ again: · 286 unmodified lines
287 clock = wrap_max(clock, min_clock);287 clock = wrap_max(clock, min_clock);
288 clock = wrap_min(clock, max_clock);288 clock = wrap_min(clock, max_clock);
289 289
290 if (!arch_try_cmpxchg64(&scd->clock, &old_clock, clock))290 if (!raw_try_cmpxchg64(&scd->clock, &old_clock, clock))
291 goto again;291 goto again;
292 292
293 return clock;293 return clock;
lib/crypto/curve25519-hacl64.c+0−2
@@ -14,8 +14,6 @@ · 13 unmodified lines
14#include <crypto/curve25519.h>14#include <crypto/curve25519.h>
15#include <linux/string.h>15#include <linux/string.h>
16 16
17typedef __uint128_t u128;
18
19static __always_inline u64 u64_eq_mask(u64 a, u64 b)17static __always_inline u64 u64_eq_mask(u64 a, u64 b)
20{18{
21 u64 x = a ^ b;19 u64 x = a ^ b;
lib/crypto/poly1305-donna64.c+0−2
@@ -10,8 +10,6 @@ · 9 unmodified lines
10#include <asm/unaligned.h>10#include <asm/unaligned.h>
11#include <crypto/internal/poly1305.h>11#include <crypto/internal/poly1305.h>
12 12
13typedef __uint128_t u128;
14
15void poly1305_core_setkey(struct poly1305_core_key *key,13void poly1305_core_setkey(struct poly1305_core_key *key,
16 const u8 raw_key[POLY1305_BLOCK_SIZE])14 const u8 raw_key[POLY1305_BLOCK_SIZE])
17{15{
mm/slab.h+46−7
@@ -6,6 +6,38 @@ · 5 unmodified lines
6 */6 */
7void __init kmem_cache_init(void);7void __init kmem_cache_init(void);
8 8
9#ifdef CONFIG_64BIT
10# ifdef system_has_cmpxchg128
11# define system_has_freelist_aba() system_has_cmpxchg128()
12# define try_cmpxchg_freelist try_cmpxchg128
13# endif
14#define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg128
15typedef u128 freelist_full_t;
16#else /* CONFIG_64BIT */
17# ifdef system_has_cmpxchg64
18# define system_has_freelist_aba() system_has_cmpxchg64()
19# define try_cmpxchg_freelist try_cmpxchg64
20# endif
21#define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg64
22typedef u64 freelist_full_t;
23#endif /* CONFIG_64BIT */
24
25#if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
26#undef system_has_freelist_aba
27#endif
28
29/*
30 * Freelist pointer and counter to cmpxchg together, avoids the typical ABA
31 * problems with cmpxchg of just a pointer.
32 */
33typedef union {
34 struct {
35 void *freelist;
36 unsigned long counter;
37 };
38 freelist_full_t full;
39} freelist_aba_t;
40
9/* Reuses the bits in struct page */41/* Reuses the bits in struct page */
10struct slab {42struct slab {
11 unsigned long __page_flags;43 unsigned long __page_flags;
@@ -38,14 +70,21 @@ struct slab { · 26 unmodified lines
38#endif70#endif
39 };71 };
40 /* Double-word boundary */72 /* Double-word boundary */
41 void *freelist; /* first free object */
42 union {73 union {
43 unsigned long counters;
44 struct {74 struct {
45 unsigned inuse:16;75 void *freelist; /* first free object */
46 unsigned objects:15;76 union {
47 unsigned frozen:1;77 unsigned long counters;
78 struct {
79 unsigned inuse:16;
80 unsigned objects:15;
81 unsigned frozen:1;
82 };
83 };
48 };84 };
85#ifdef system_has_freelist_aba
86 freelist_aba_t freelist_counter;
87#endif
49 };88 };
50 };89 };
51 struct rcu_head rcu_head;90 struct rcu_head rcu_head;
@@ -72,8 +111,8 @@ SLAB_MATCH(memcg_data, memcg_data); · 20 unmodified lines
72#endif111#endif
73#undef SLAB_MATCH112#undef SLAB_MATCH
74static_assert(sizeof(struct slab) <= sizeof(struct page));113static_assert(sizeof(struct slab) <= sizeof(struct page));
75#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && defined(CONFIG_SLUB)114#if defined(system_has_freelist_aba) && defined(CONFIG_SLUB)
76static_assert(IS_ALIGNED(offsetof(struct slab, freelist), 2*sizeof(void *)));115static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t)));
77#endif116#endif
78 117
79/**118/**
mm/slub.c+82−57
@@ -292,7 +292,12 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) · 291 unmodified lines
292/* Poison object */292/* Poison object */
293#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)293#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)
294/* Use cmpxchg_double */294/* Use cmpxchg_double */
295
296#ifdef system_has_freelist_aba
295#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)297#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
298#else
299#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0U)
300#endif
296 301
297/*302/*
298 * Tracking user of a slab.303 * Tracking user of a slab.
@@ -512,6 +517,40 @@ static __always_inline void slab_unlock(struct slab *slab) · 213 unmodified lines
512 __bit_spin_unlock(PG_locked, &page->flags);517 __bit_spin_unlock(PG_locked, &page->flags);
513}518}
514 519
520static inline bool
521__update_freelist_fast(struct slab *slab,
522 void *freelist_old, unsigned long counters_old,
523 void *freelist_new, unsigned long counters_new)
524{
525#ifdef system_has_freelist_aba
526 freelist_aba_t old = { .freelist = freelist_old, .counter = counters_old };
527 freelist_aba_t new = { .freelist = freelist_new, .counter = counters_new };
528
529 return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full);
530#else
531 return false;
532#endif
533}
534
535static inline bool
536__update_freelist_slow(struct slab *slab,
537 void *freelist_old, unsigned long counters_old,
538 void *freelist_new, unsigned long counters_new)
539{
540 bool ret = false;
541
542 slab_lock(slab);
543 if (slab->freelist == freelist_old &&
544 slab->counters == counters_old) {
545 slab->freelist = freelist_new;
546 slab->counters = counters_new;
547 ret = true;
548 }
549 slab_unlock(slab);
550
551 return ret;
552}
553
515/*554/*
516 * Interrupts must be disabled (for the fallback code to work right), typically555 * Interrupts must be disabled (for the fallback code to work right), typically
517 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is556 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is
@@ -519,33 +558,25 @@ static __always_inline void slab_unlock(struct slab *slab) · 1 unmodified line
519 * allocation/ free operation in hardirq context. Therefore nothing can558 * allocation/ free operation in hardirq context. Therefore nothing can
520 * interrupt the operation.559 * interrupt the operation.
521 */560 */
522static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,561static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
523 void *freelist_old, unsigned long counters_old,562 void *freelist_old, unsigned long counters_old,
524 void *freelist_new, unsigned long counters_new,563 void *freelist_new, unsigned long counters_new,
525 const char *n)564 const char *n)
526{565{
566 bool ret;
567
527 if (USE_LOCKLESS_FAST_PATH())568 if (USE_LOCKLESS_FAST_PATH())
528 lockdep_assert_irqs_disabled();569 lockdep_assert_irqs_disabled();
529#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \570
530 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
531 if (s->flags & __CMPXCHG_DOUBLE) {571 if (s->flags & __CMPXCHG_DOUBLE) {
532 if (cmpxchg_double(&slab->freelist, &slab->counters,572 ret = __update_freelist_fast(slab, freelist_old, counters_old,
533 freelist_old, counters_old,573 freelist_new, counters_new);
534 freelist_new, counters_new))574 } else {
535 return true;575 ret = __update_freelist_slow(slab, freelist_old, counters_old,
536 } else576 freelist_new, counters_new);
537#endif
538 {
539 slab_lock(slab);
540 if (slab->freelist == freelist_old &&
541 slab->counters == counters_old) {
542 slab->freelist = freelist_new;
543 slab->counters = counters_new;
544 slab_unlock(slab);
545 return true;
546 }
547 slab_unlock(slab);
548 }577 }
578 if (likely(ret))
579 return true;
549 580
550 cpu_relax();581 cpu_relax();
551 stat(s, CMPXCHG_DOUBLE_FAIL);582 stat(s, CMPXCHG_DOUBLE_FAIL);
@@ -557,36 +588,26 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab · 5 unmodified lines
557 return false;588 return false;
558}589}
559 590
560static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,591static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
561 void *freelist_old, unsigned long counters_old,592 void *freelist_old, unsigned long counters_old,
562 void *freelist_new, unsigned long counters_new,593 void *freelist_new, unsigned long counters_new,
563 const char *n)594 const char *n)
564{595{
565#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \596 bool ret;
566 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)597
567 if (s->flags & __CMPXCHG_DOUBLE) {598 if (s->flags & __CMPXCHG_DOUBLE) {
568 if (cmpxchg_double(&slab->freelist, &slab->counters,599 ret = __update_freelist_fast(slab, freelist_old, counters_old,
569 freelist_old, counters_old,600 freelist_new, counters_new);
570 freelist_new, counters_new))601 } else {
571 return true;
572 } else
573#endif
574 {
575 unsigned long flags;602 unsigned long flags;
576 603
577 local_irq_save(flags);604 local_irq_save(flags);
578 slab_lock(slab);605 ret = __update_freelist_slow(slab, freelist_old, counters_old,
579 if (slab->freelist == freelist_old &&606 freelist_new, counters_new);
580 slab->counters == counters_old) {
581 slab->freelist = freelist_new;
582 slab->counters = counters_new;
583 slab_unlock(slab);
584 local_irq_restore(flags);
585 return true;
586 }
587 slab_unlock(slab);
588 local_irq_restore(flags);607 local_irq_restore(flags);
589 }608 }
609 if (likely(ret))
610 return true;
590 611
591 cpu_relax();612 cpu_relax();
592 stat(s, CMPXCHG_DOUBLE_FAIL);613 stat(s, CMPXCHG_DOUBLE_FAIL);
@@ -2228,7 +2249,7 @@ static inline void *acquire_slab(struct kmem_cache *s, · 1635 unmodified lines
2228 VM_BUG_ON(new.frozen);2249 VM_BUG_ON(new.frozen);
2229 new.frozen = 1;2250 new.frozen = 1;
2230 2251
2231 if (!__cmpxchg_double_slab(s, slab,2252 if (!__slab_update_freelist(s, slab,
2232 freelist, counters,2253 freelist, counters,
2233 new.freelist, new.counters,2254 new.freelist, new.counters,
2234 "acquire_slab"))2255 "acquire_slab"))
@@ -2554,7 +2575,7 @@ redo: · 319 unmodified lines
2554 }2575 }
2555 2576
2556 2577
2557 if (!cmpxchg_double_slab(s, slab,2578 if (!slab_update_freelist(s, slab,
2558 old.freelist, old.counters,2579 old.freelist, old.counters,
2559 new.freelist, new.counters,2580 new.freelist, new.counters,
2560 "unfreezing slab")) {2581 "unfreezing slab")) {
@@ -2611,7 +2632,7 @@ static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab) · 50 unmodified lines
2611 2632
2612 new.frozen = 0;2633 new.frozen = 0;
2613 2634
2614 } while (!__cmpxchg_double_slab(s, slab,2635 } while (!__slab_update_freelist(s, slab,
2615 old.freelist, old.counters,2636 old.freelist, old.counters,
2616 new.freelist, new.counters,2637 new.freelist, new.counters,
2617 "unfreezing slab"));2638 "unfreezing slab"));
@@ -3008,6 +3029,18 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) · 390 unmodified lines
3008}3029}
3009 3030
3010#ifndef CONFIG_SLUB_TINY3031#ifndef CONFIG_SLUB_TINY
3032static inline bool
3033__update_cpu_freelist_fast(struct kmem_cache *s,
3034 void *freelist_old, void *freelist_new,
3035 unsigned long tid)
3036{
3037 freelist_aba_t old = { .freelist = freelist_old, .counter = tid };
3038 freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };
3039
3040 return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
3041 &old.full, new.full);
3042}
3043
3011/*3044/*
3012 * Check the slab->freelist and either transfer the freelist to the3045 * Check the slab->freelist and either transfer the freelist to the
3013 * per cpu freelist or deactivate the slab.3046 * per cpu freelist or deactivate the slab.
@@ -3034,7 +3067,7 @@ static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) · 20 unmodified lines
3034 new.inuse = slab->objects;3067 new.inuse = slab->objects;
3035 new.frozen = freelist != NULL;3068 new.frozen = freelist != NULL;
3036 3069
3037 } while (!__cmpxchg_double_slab(s, slab,3070 } while (!__slab_update_freelist(s, slab,
3038 freelist, counters,3071 freelist, counters,
3039 NULL, new.counters,3072 NULL, new.counters,
3040 "get_freelist"));3073 "get_freelist"));
@@ -3359,11 +3392,7 @@ redo: · 318 unmodified lines
3359 * against code executing on this cpu *not* from access by3392 * against code executing on this cpu *not* from access by
3360 * other cpus.3393 * other cpus.
3361 */3394 */
3362 if (unlikely(!this_cpu_cmpxchg_double(3395 if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) {
3363 s->cpu_slab->freelist, s->cpu_slab->tid,
3364 object, tid,
3365 next_object, next_tid(tid)))) {
3366
3367 note_cmpxchg_failure("slab_alloc", s, tid);3396 note_cmpxchg_failure("slab_alloc", s, tid);
3368 goto redo;3397 goto redo;
3369 }3398 }
@@ -3631,7 +3660,7 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, · 261 unmodified lines
3631 }3660 }
3632 }3661 }
3633 3662
3634 } while (!cmpxchg_double_slab(s, slab,3663 } while (!slab_update_freelist(s, slab,
3635 prior, counters,3664 prior, counters,
3636 head, new.counters,3665 head, new.counters,
3637 "__slab_free"));3666 "__slab_free"));
@@ -3736,11 +3765,7 @@ redo: · 98 unmodified lines
3736 3765
3737 set_freepointer(s, tail_obj, freelist);3766 set_freepointer(s, tail_obj, freelist);
3738 3767
3739 if (unlikely(!this_cpu_cmpxchg_double(3768 if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {
3740 s->cpu_slab->freelist, s->cpu_slab->tid,
3741 freelist, tid,
3742 head, next_tid(tid)))) {
3743
3744 note_cmpxchg_failure("slab_free", s, tid);3769 note_cmpxchg_failure("slab_free", s, tid);
3745 goto redo;3770 goto redo;
3746 }3771 }
@@ -4505,11 +4530,11 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) · 758 unmodified lines
4505 }4530 }
4506 }4531 }
4507 4532
4508#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \4533#ifdef system_has_freelist_aba
4509 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)4534 if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) {
4510 if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
4511 /* Enable fast mode */4535 /* Enable fast mode */
4512 s->flags |= __CMPXCHG_DOUBLE;4536 s->flags |= __CMPXCHG_DOUBLE;
4537 }
4513#endif4538#endif
4514 4539
4515 /*4540 /*
scripts/atomic/atomic-tbl.sh+108−4
@@ -36,9 +36,16 @@ meta_has_relaxed() · 35 unmodified lines
36 meta_in "$1" "BFIR"36 meta_in "$1" "BFIR"
37}37}
38 38
39#find_fallback_template(pfx, name, sfx, order)39#meta_is_implicitly_relaxed(meta)
40find_fallback_template()40meta_is_implicitly_relaxed()
41{41{
42 meta_in "$1" "vls"
43}
44
45#find_template(tmpltype, pfx, name, sfx, order)
46find_template()
47{
48 local tmpltype="$1"; shift
42 local pfx="$1"; shift49 local pfx="$1"; shift
43 local name="$1"; shift50 local name="$1"; shift
44 local sfx="$1"; shift51 local sfx="$1"; shift
@@ -52,8 +59,8 @@ find_fallback_template() · 7 unmodified lines
52 #59 #
53 # Start at the most specific, and fall back to the most general. Once60 # Start at the most specific, and fall back to the most general. Once
54 # we find a specific fallback, don't bother looking for more.61 # we find a specific fallback, don't bother looking for more.
55 for base in "${pfx}${name}${sfx}${order}" "${name}"; do62 for base in "${pfx}${name}${sfx}${order}" "${pfx}${name}${sfx}" "${name}"; do
56 file="${ATOMICDIR}/fallbacks/${base}"63 file="${ATOMICDIR}/${tmpltype}/${base}"
57 64
58 if [ -f "${file}" ]; then65 if [ -f "${file}" ]; then
59 printf "${file}"66 printf "${file}"
@@ -62,6 +69,18 @@ find_fallback_template() · 2 unmodified lines
62 done69 done
63}70}
64 71
72#find_fallback_template(pfx, name, sfx, order)
73find_fallback_template()
74{
75 find_template "fallbacks" "$@"
76}
77
78#find_kerneldoc_template(pfx, name, sfx, order)
79find_kerneldoc_template()
80{
81 find_template "kerneldoc" "$@"
82}
83
65#gen_ret_type(meta, int)84#gen_ret_type(meta, int)
66gen_ret_type() {85gen_ret_type() {
67 local meta="$1"; shift86 local meta="$1"; shift
@@ -142,6 +161,91 @@ gen_args() · 74 unmodified lines
142 done161 done
143}162}
144 163
164#gen_desc_return(meta)
165gen_desc_return()
166{
167 local meta="$1"; shift
168
169 case "${meta}" in
170 [v])
171 printf "Return: Nothing."
172 ;;
173 [Ff])
174 printf "Return: The original value of @v."
175 ;;
176 [R])
177 printf "Return: The updated value of @v."
178 ;;
179 [l])
180 printf "Return: The value of @v."
181 ;;
182 esac
183}
184
185#gen_template_kerneldoc(template, class, meta, pfx, name, sfx, order, atomic, int, args...)
186gen_template_kerneldoc()
187{
188 local template="$1"; shift
189 local class="$1"; shift
190 local meta="$1"; shift
191 local pfx="$1"; shift
192 local name="$1"; shift
193 local sfx="$1"; shift
194 local order="$1"; shift
195 local atomic="$1"; shift
196 local int="$1"; shift
197
198 local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
199
200 local ret="$(gen_ret_type "${meta}" "${int}")"
201 local retstmt="$(gen_ret_stmt "${meta}")"
202 local params="$(gen_params "${int}" "${atomic}" "$@")"
203 local args="$(gen_args "$@")"
204 local desc_order=""
205 local desc_instrumentation=""
206 local desc_return=""
207
208 if [ ! -z "${order}" ]; then
209 desc_order="${order##_}"
210 elif meta_is_implicitly_relaxed "${meta}"; then
211 desc_order="relaxed"
212 else
213 desc_order="full"
214 fi
215
216 if [ -z "${class}" ]; then
217 desc_noinstr="Unsafe to use in noinstr code; use raw_${atomicname}() there."
218 else
219 desc_noinstr="Safe to use in noinstr code; prefer ${atomicname}() elsewhere."
220 fi
221
222 desc_return="$(gen_desc_return "${meta}")"
223
224 . ${template}
225}
226
227#gen_kerneldoc(class, meta, pfx, name, sfx, order, atomic, int, args...)
228gen_kerneldoc()
229{
230 local class="$1"; shift
231 local meta="$1"; shift
232 local pfx="$1"; shift
233 local name="$1"; shift
234 local sfx="$1"; shift
235 local order="$1"; shift
236
237 local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
238
239 local tmpl="$(find_kerneldoc_template "${pfx}" "${name}" "${sfx}" "${order}")"
240 if [ -z "${tmpl}" ]; then
241 printf "/*\n"
242 printf " * No kerneldoc available for ${class}${atomicname}\n"
243 printf " */\n"
244 else
245 gen_template_kerneldoc "${tmpl}" "${class}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
246 fi
247}
248
145#gen_proto_order_variants(meta, pfx, name, sfx, ...)249#gen_proto_order_variants(meta, pfx, name, sfx, ...)
146gen_proto_order_variants()250gen_proto_order_variants()
147{251{
scripts/atomic/atomics.tbl+1−1
@@ -27,7 +27,7 @@ and vF i v · 26 unmodified lines
27andnot vF i v27andnot vF i v
28or vF i v28or vF i v
29xor vF i v29xor vF i v
30xchg I v i30xchg I v i:new
31cmpxchg I v i:old i:new31cmpxchg I v i:old i:new
32try_cmpxchg B v p:old i:new32try_cmpxchg B v p:old i:new
33sub_and_test b i v33sub_and_test b i v
scripts/atomic/fallbacks/acquire+0−4
@@ -1,9 +1,5 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}
3arch_${atomic}_${pfx}${name}${sfx}_acquire(${params})
4{
5 ${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});2 ${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
6 __atomic_acquire_fence();3 __atomic_acquire_fence();
7 return ret;4 return ret;
8}
9EOF5EOF
scripts/atomic/fallbacks/add_negative+1−13
@@ -1,15 +1,3 @@
1cat <<EOF1cat <<EOF
2/**2 return raw_${atomic}_add_return${order}(i, v) < 0;
3 * arch_${atomic}_add_negative${order} - Add and test if negative
4 * @i: integer value to add
5 * @v: pointer of type ${atomic}_t
6 *
7 * Atomically adds @i to @v and returns true if the result is negative,
8 * or false when the result is greater than or equal to zero.
9 */
10static __always_inline bool
11arch_${atomic}_add_negative${order}(${int} i, ${atomic}_t *v)
12{
13 return arch_${atomic}_add_return${order}(i, v) < 0;
14}
15EOF3EOF
scripts/atomic/fallbacks/add_unless+1−14
@@ -1,16 +1,3 @@
1cat << EOF1cat << EOF
2/**2 return raw_${atomic}_fetch_add_unless(v, a, u) != u;
3 * arch_${atomic}_add_unless - add unless the number is already a given value
4 * @v: pointer of type ${atomic}_t
5 * @a: the amount to add to v...
6 * @u: ...unless v is equal to u.
7 *
8 * Atomically adds @a to @v, if @v was not already @u.
9 * Returns true if the addition was done.
10 */
11static __always_inline bool
12arch_${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13{
14 return arch_${atomic}_fetch_add_unless(v, a, u) != u;
15}
16EOF3EOF
scripts/atomic/fallbacks/andnot+1−5
@@ -1,7 +1,3 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}2 ${retstmt}raw_${atomic}_${pfx}and${sfx}${order}(~i, v);
3arch_${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
4{
5 ${retstmt}arch_${atomic}_${pfx}and${sfx}${order}(~i, v);
6}
7EOF3EOF
scripts/atomic/fallbacks/cmpxchg+3−0
@@ -0,0 +1,3 @@
1cat <<EOF
2 return raw_cmpxchg${order}(&v->counter, old, new);
3EOF
scripts/atomic/fallbacks/dec+1−5
@@ -1,7 +1,3 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}2 ${retstmt}raw_${atomic}_${pfx}sub${sfx}${order}(1, v);
3arch_${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
4{
5 ${retstmt}arch_${atomic}_${pfx}sub${sfx}${order}(1, v);
6}
7EOF3EOF
scripts/atomic/fallbacks/dec_and_test+1−13
@@ -1,15 +1,3 @@
1cat <<EOF1cat <<EOF
2/**2 return raw_${atomic}_dec_return(v) == 0;
3 * arch_${atomic}_dec_and_test - decrement and test
4 * @v: pointer of type ${atomic}_t
5 *
6 * Atomically decrements @v by 1 and
7 * returns true if the result is 0, or false for all other
8 * cases.
9 */
10static __always_inline bool
11arch_${atomic}_dec_and_test(${atomic}_t *v)
12{
13 return arch_${atomic}_dec_return(v) == 0;
14}
15EOF3EOF
scripts/atomic/fallbacks/dec_if_positive+2−6
@@ -1,15 +1,11 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}2 ${int} dec, c = raw_${atomic}_read(v);
3arch_${atomic}_dec_if_positive(${atomic}_t *v)
4{
5 ${int} dec, c = arch_${atomic}_read(v);
6 3
7 do {4 do {
8 dec = c - 1;5 dec = c - 1;
9 if (unlikely(dec < 0))6 if (unlikely(dec < 0))
10 break;7 break;
11 } while (!arch_${atomic}_try_cmpxchg(v, &c, dec));8 } while (!raw_${atomic}_try_cmpxchg(v, &c, dec));
12 9
13 return dec;10 return dec;
14}
15EOF11EOF
scripts/atomic/fallbacks/dec_unless_positive+2−6
@@ -1,14 +1,10 @@
1cat <<EOF1cat <<EOF
2static __always_inline bool2 ${int} c = raw_${atomic}_read(v);
3arch_${atomic}_dec_unless_positive(${atomic}_t *v)
4{
5 ${int} c = arch_${atomic}_read(v);
6 3
7 do {4 do {
8 if (unlikely(c > 0))5 if (unlikely(c > 0))
9 return false;6 return false;
10 } while (!arch_${atomic}_try_cmpxchg(v, &c, c - 1));7 } while (!raw_${atomic}_try_cmpxchg(v, &c, c - 1));
11 8
12 return true;9 return true;
13}
14EOF10EOF
scripts/atomic/fallbacks/fence+0−4
@@ -1,11 +1,7 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}
3arch_${atomic}_${pfx}${name}${sfx}(${params})
4{
5 ${ret} ret;2 ${ret} ret;
6 __atomic_pre_full_fence();3 __atomic_pre_full_fence();
7 ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});4 ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
8 __atomic_post_full_fence();5 __atomic_post_full_fence();
9 return ret;6 return ret;
10}
11EOF7EOF
scripts/atomic/fallbacks/fetch_add_unless+2−15
@@ -1,23 +1,10 @@
1cat << EOF1cat << EOF
2/**2 ${int} c = raw_${atomic}_read(v);
3 * arch_${atomic}_fetch_add_unless - add unless the number is already a given value
4 * @v: pointer of type ${atomic}_t
5 * @a: the amount to add to v...
6 * @u: ...unless v is equal to u.
7 *
8 * Atomically adds @a to @v, so long as @v was not already @u.
9 * Returns original value of @v
10 */
11static __always_inline ${int}
12arch_${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13{
14 ${int} c = arch_${atomic}_read(v);
15 3
16 do {4 do {
17 if (unlikely(c == u))5 if (unlikely(c == u))
18 break;6 break;
19 } while (!arch_${atomic}_try_cmpxchg(v, &c, c + a));7 } while (!raw_${atomic}_try_cmpxchg(v, &c, c + a));
20 8
21 return c;9 return c;
22}
23EOF10EOF
scripts/atomic/fallbacks/inc+1−5
@@ -1,7 +1,3 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}2 ${retstmt}raw_${atomic}_${pfx}add${sfx}${order}(1, v);
3arch_${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
4{
5 ${retstmt}arch_${atomic}_${pfx}add${sfx}${order}(1, v);
6}
7EOF3EOF
scripts/atomic/fallbacks/inc_and_test+1−13
@@ -1,15 +1,3 @@
1cat <<EOF1cat <<EOF
2/**2 return raw_${atomic}_inc_return(v) == 0;
3 * arch_${atomic}_inc_and_test - increment and test
4 * @v: pointer of type ${atomic}_t
5 *
6 * Atomically increments @v by 1
7 * and returns true if the result is zero, or false for all
8 * other cases.
9 */
10static __always_inline bool
11arch_${atomic}_inc_and_test(${atomic}_t *v)
12{
13 return arch_${atomic}_inc_return(v) == 0;
14}
15EOF3EOF
scripts/atomic/fallbacks/inc_not_zero+1−12
@@ -1,14 +1,3 @@
1cat <<EOF1cat <<EOF
2/**2 return raw_${atomic}_add_unless(v, 1, 0);
3 * arch_${atomic}_inc_not_zero - increment unless the number is zero
4 * @v: pointer of type ${atomic}_t
5 *
6 * Atomically increments @v by 1, if @v is non-zero.
7 * Returns true if the increment was done.
8 */
9static __always_inline bool
10arch_${atomic}_inc_not_zero(${atomic}_t *v)
11{
12 return arch_${atomic}_add_unless(v, 1, 0);
13}
14EOF3EOF
scripts/atomic/fallbacks/inc_unless_negative+2−6
@@ -1,14 +1,10 @@
1cat <<EOF1cat <<EOF
2static __always_inline bool2 ${int} c = raw_${atomic}_read(v);
3arch_${atomic}_inc_unless_negative(${atomic}_t *v)
4{
5 ${int} c = arch_${atomic}_read(v);
6 3
7 do {4 do {
8 if (unlikely(c < 0))5 if (unlikely(c < 0))
9 return false;6 return false;
10 } while (!arch_${atomic}_try_cmpxchg(v, &c, c + 1));7 } while (!raw_${atomic}_try_cmpxchg(v, &c, c + 1));
11 8
12 return true;9 return true;
13}
14EOF10EOF
scripts/atomic/fallbacks/read_acquire+1−5
@@ -1,16 +1,12 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}
3arch_${atomic}_read_acquire(const ${atomic}_t *v)
4{
5 ${int} ret;2 ${int} ret;
6 3
7 if (__native_word(${atomic}_t)) {4 if (__native_word(${atomic}_t)) {
8 ret = smp_load_acquire(&(v)->counter);5 ret = smp_load_acquire(&(v)->counter);
9 } else {6 } else {
10 ret = arch_${atomic}_read(v);7 ret = raw_${atomic}_read(v);
11 __atomic_acquire_fence();8 __atomic_acquire_fence();
12 }9 }
13 10
14 return ret;11 return ret;
15}
16EOF12EOF
scripts/atomic/fallbacks/release+0−4
@@ -1,8 +1,4 @@
1cat <<EOF1cat <<EOF
2static __always_inline ${ret}
3arch_${atomic}_${pfx}${name}${sfx}_release(${params})
4{
5 __atomic_release_fence();2 __atomic_release_fence();
6 ${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});3 ${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
7}
8EOF4EOF
scripts/atomic/fallbacks/set_release+1−5
@@ -1,12 +1,8 @@
1cat <<EOF1cat <<EOF
2static __always_inline void
3arch_${atomic}_set_release(${atomic}_t *v, ${int} i)
4{
5 if (__native_word(${atomic}_t)) {2 if (__native_word(${atomic}_t)) {
6 smp_store_release(&(v)->counter, i);3 smp_store_release(&(v)->counter, i);
7 } else {4 } else {
8 __atomic_release_fence();5 __atomic_release_fence();
9 arch_${atomic}_set(v, i);6 raw_${atomic}_set(v, i);
10 }7 }
11}
12EOF8EOF
scripts/atomic/fallbacks/sub_and_test+1−14
@@ -1,16 +1,3 @@
1cat <<EOF1cat <<EOF
2/**2 return raw_${atomic}_sub_return(i, v) == 0;
3 * arch_${atomic}_sub_and_test - subtract value from variable and test result
4 * @i: integer value to subtract
5 * @v: pointer of type ${atomic}_t
6 *
7 * Atomically subtracts @i from @v and returns
8 * true if the result is zero, or false for all
9 * other cases.
10 */
11static __always_inline bool
12arch_${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
13{
14 return arch_${atomic}_sub_return(i, v) == 0;
15}
16EOF3EOF
scripts/atomic/fallbacks/try_cmpxchg+1−5
@@ -1,11 +1,7 @@
1cat <<EOF1cat <<EOF
2static __always_inline bool
3arch_${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
4{
5 ${int} r, o = *old;2 ${int} r, o = *old;
6 r = arch_${atomic}_cmpxchg${order}(v, o, new);3 r = raw_${atomic}_cmpxchg${order}(v, o, new);
7 if (unlikely(r != o))4 if (unlikely(r != o))
8 *old = r;5 *old = r;
9 return likely(r == o);6 return likely(r == o);
10}
11EOF7EOF
scripts/atomic/fallbacks/xchg+3−0
@@ -0,0 +1,3 @@
1cat <<EOF
2 return raw_xchg${order}(&v->counter, new);
3EOF
scripts/atomic/gen-atomic-fallback.sh+175−91
@@ -17,23 +17,16 @@ gen_template_fallback() · 16 unmodified lines
17 local atomic="$1"; shift17 local atomic="$1"; shift
18 local int="$1"; shift18 local int="$1"; shift
19 19
20 local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}"
21
22 local ret="$(gen_ret_type "${meta}" "${int}")"20 local ret="$(gen_ret_type "${meta}" "${int}")"
23 local retstmt="$(gen_ret_stmt "${meta}")"21 local retstmt="$(gen_ret_stmt "${meta}")"
24 local params="$(gen_params "${int}" "${atomic}" "$@")"22 local params="$(gen_params "${int}" "${atomic}" "$@")"
25 local args="$(gen_args "$@")"23 local args="$(gen_args "$@")"
26 24
27 if [ ! -z "${template}" ]; then25 . ${template}
28 printf "#ifndef ${atomicname}\n"
29 . ${template}
30 printf "#define ${atomicname} ${atomicname}\n"
31 printf "#endif\n\n"
32 fi
33}26}
34 27
35#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)28#gen_order_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
36gen_proto_fallback()29gen_order_fallback()
37{30{
38 local meta="$1"; shift31 local meta="$1"; shift
39 local pfx="$1"; shift32 local pfx="$1"; shift
@@ -41,87 +34,124 @@ gen_proto_fallback() · 1 unmodified line
41 local sfx="$1"; shift34 local sfx="$1"; shift
42 local order="$1"; shift35 local order="$1"; shift
43 36
44 local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"37 local tmpl_order=${order#_}
38 local tmpl="${ATOMICDIR}/fallbacks/${tmpl_order:-fence}"
45 gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"39 gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
46}40}
47 41
48#gen_basic_fallbacks(basename)42#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
49gen_basic_fallbacks()43gen_proto_fallback()
50{
51 local basename="$1"; shift
52cat << EOF
53#define ${basename}_acquire ${basename}
54#define ${basename}_release ${basename}
55#define ${basename}_relaxed ${basename}
56EOF
57}
58
59gen_proto_order_variant()
60{44{
61 local meta="$1"; shift45 local meta="$1"; shift
62 local pfx="$1"; shift46 local pfx="$1"; shift
63 local name="$1"; shift47 local name="$1"; shift
64 local sfx="$1"; shift48 local sfx="$1"; shift
65 local order="$1"; shift49 local order="$1"; shift
66 local atomic="$1"
67 50
68 local basename="arch_${atomic}_${pfx}${name}${sfx}"51 local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
69 52 gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
70 printf "#define ${basename}${order} ${basename}${order}\n"
71}53}
72 54
73#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)55#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, args...)
74gen_proto_order_variants()56gen_proto_order_variant()
75{57{
76 local meta="$1"; shift58 local meta="$1"; shift
77 local pfx="$1"; shift59 local pfx="$1"; shift
78 local name="$1"; shift60 local name="$1"; shift
79 local sfx="$1"; shift61 local sfx="$1"; shift
80 local atomic="$1"62 local order="$1"; shift
63 local atomic="$1"; shift
64 local int="$1"; shift
81 65
82 local basename="arch_${atomic}_${pfx}${name}${sfx}"66 local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
67 local basename="${atomic}_${pfx}${name}${sfx}"
83 68
84 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"69 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
85 70
86 # If we don't have relaxed atomics, then we don't bother with ordering fallbacks71 local ret="$(gen_ret_type "${meta}" "${int}")"
87 # read_acquire and set_release need to be templated, though72 local retstmt="$(gen_ret_stmt "${meta}")"
88 if ! meta_has_relaxed "${meta}"; then73 local params="$(gen_params "${int}" "${atomic}" "$@")"
89 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"74 local args="$(gen_args "$@")"
90 75
91 if meta_has_acquire "${meta}"; then76 gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
92 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"77
93 fi78 printf "static __always_inline ${ret}\n"
79 printf "raw_${atomicname}(${params})\n"
80 printf "{\n"
81
82 # Where there is no possible fallback, this order variant is mandatory
83 # and must be provided by arch code. Add a comment to the header to
84 # make this obvious.
85 #
86 # Ideally we'd error on a missing definition, but arch code might
87 # define this order variant as a C function without a preprocessor
88 # symbol.
89 if [ -z ${template} ] && [ -z "${order}" ] && ! meta_has_relaxed "${meta}"; then
90 printf "\t${retstmt}arch_${atomicname}(${args});\n"
91 printf "}\n\n"
92 return
93 fi
94 94
95 if meta_has_release "${meta}"; then95 printf "#if defined(arch_${atomicname})\n"
96 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"96 printf "\t${retstmt}arch_${atomicname}(${args});\n"
97 fi
98 97
99 return98 # Allow FULL/ACQUIRE/RELEASE ops to be defined in terms of RELAXED ops
99 if [ "${order}" != "_relaxed" ] && meta_has_relaxed "${meta}"; then
100 printf "#elif defined(arch_${basename}_relaxed)\n"
101 gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
100 fi102 fi
101 103
102 printf "#ifndef ${basename}_relaxed\n"104 # Allow ACQUIRE/RELEASE/RELAXED ops to be defined in terms of FULL ops
105 if [ ! -z "${order}" ]; then
106 printf "#elif defined(arch_${basename})\n"
107 printf "\t${retstmt}arch_${basename}(${args});\n"
108 fi
103 109
110 printf "#else\n"
104 if [ ! -z "${template}" ]; then111 if [ ! -z "${template}" ]; then
105 printf "#ifdef ${basename}\n"112 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
113 else
114 printf "#error \"Unable to define raw_${atomicname}\"\n"
106 fi115 fi
107 116
108 gen_basic_fallbacks "${basename}"117 printf "#endif\n"
118 printf "}\n\n"
119}
109 120
110 if [ ! -z "${template}" ]; then121
111 printf "#endif /* ${basename} */\n\n"122#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
112 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"123gen_proto_order_variants()
113 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"124{
114 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"125 local meta="$1"; shift
115 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"126 local pfx="$1"; shift
127 local name="$1"; shift
128 local sfx="$1"; shift
129 local atomic="$1"
130
131 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
132
133 if meta_has_acquire "${meta}"; then
134 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
116 fi135 fi
117 136
118 printf "#else /* ${basename}_relaxed */\n\n"137 if meta_has_release "${meta}"; then
138 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
139 fi
119 140
120 gen_template_fallback "${ATOMICDIR}/fallbacks/acquire" "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"141 if meta_has_relaxed "${meta}"; then
121 gen_template_fallback "${ATOMICDIR}/fallbacks/release" "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"142 gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
122 gen_template_fallback "${ATOMICDIR}/fallbacks/fence" "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"143 fi
144}
123 145
124 printf "#endif /* ${basename}_relaxed */\n\n"146#gen_basic_fallbacks(basename)
147gen_basic_fallbacks()
148{
149 local basename="$1"; shift
150cat << EOF
151#define raw_${basename}_acquire arch_${basename}
152#define raw_${basename}_release arch_${basename}
153#define raw_${basename}_relaxed arch_${basename}
154EOF
125}155}
126 156
127gen_order_fallbacks()157gen_order_fallbacks()
@@ -130,36 +160,65 @@ gen_order_fallbacks() · 2 unmodified lines
130 160
131cat <<EOF161cat <<EOF
132 162
133#ifndef ${xchg}_acquire163#define raw_${xchg}_relaxed arch_${xchg}_relaxed
134#define ${xchg}_acquire(...) \\164
135 __atomic_op_acquire(${xchg}, __VA_ARGS__)165#ifdef arch_${xchg}_acquire
166#define raw_${xchg}_acquire arch_${xchg}_acquire
167#else
168#define raw_${xchg}_acquire(...) \\
169 __atomic_op_acquire(arch_${xchg}, __VA_ARGS__)
136#endif170#endif
137 171
138#ifndef ${xchg}_release172#ifdef arch_${xchg}_release
139#define ${xchg}_release(...) \\173#define raw_${xchg}_release arch_${xchg}_release
140 __atomic_op_release(${xchg}, __VA_ARGS__)174#else
175#define raw_${xchg}_release(...) \\
176 __atomic_op_release(arch_${xchg}, __VA_ARGS__)
141#endif177#endif
142 178
143#ifndef ${xchg}179#ifdef arch_${xchg}
144#define ${xchg}(...) \\180#define raw_${xchg} arch_${xchg}
145 __atomic_op_fence(${xchg}, __VA_ARGS__)181#else
182#define raw_${xchg}(...) \\
183 __atomic_op_fence(arch_${xchg}, __VA_ARGS__)
146#endif184#endif
147 185
148EOF186EOF
149}187}
150 188
151gen_xchg_fallbacks()189gen_xchg_order_fallback()
152{190{
153 local xchg="$1"; shift191 local xchg="$1"; shift
154 printf "#ifndef ${xchg}_relaxed\n"192 local order="$1"; shift
193 local forder="${order:-_fence}"
155 194
156 gen_basic_fallbacks ${xchg}195 printf "#if defined(arch_${xchg}${order})\n"
196 printf "#define raw_${xchg}${order} arch_${xchg}${order}\n"
157 197
158 printf "#else /* ${xchg}_relaxed */\n"198 if [ "${order}" != "_relaxed" ]; then
199 printf "#elif defined(arch_${xchg}_relaxed)\n"
200 printf "#define raw_${xchg}${order}(...) \\\\\n"
201 printf " __atomic_op${forder}(arch_${xchg}, __VA_ARGS__)\n"
202 fi
203
204 if [ ! -z "${order}" ]; then
205 printf "#elif defined(arch_${xchg})\n"
206 printf "#define raw_${xchg}${order} arch_${xchg}\n"
207 fi
159 208
160 gen_order_fallbacks ${xchg}209 printf "#else\n"
210 printf "extern void raw_${xchg}${order}_not_implemented(void);\n"
211 printf "#define raw_${xchg}${order}(...) raw_${xchg}${order}_not_implemented()\n"
212 printf "#endif\n\n"
213}
214
215gen_xchg_fallbacks()
216{
217 local xchg="$1"; shift
161 218
162 printf "#endif /* ${xchg}_relaxed */\n\n"219 for order in "" "_acquire" "_release" "_relaxed"; do
220 gen_xchg_order_fallback "${xchg}" "${order}"
221 done
163}222}
164 223
165gen_try_cmpxchg_fallback()224gen_try_cmpxchg_fallback()
@@ -168,40 +227,61 @@ gen_try_cmpxchg_fallback() · 2 unmodified lines
168 local order="$1"; shift;227 local order="$1"; shift;
169 228
170cat <<EOF229cat <<EOF
171#ifndef arch_try_${cmpxchg}${order}230#define raw_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
172#define arch_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
173({ \\231({ \\
174 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\232 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
175 ___r = arch_${cmpxchg}${order}((_ptr), ___o, (_new)); \\233 ___r = raw_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
176 if (unlikely(___r != ___o)) \\234 if (unlikely(___r != ___o)) \\
177 *___op = ___r; \\235 *___op = ___r; \\
178 likely(___r == ___o); \\236 likely(___r == ___o); \\
179})237})
180#endif /* arch_try_${cmpxchg}${order} */
181
182EOF238EOF
183}239}
184 240
185gen_try_cmpxchg_fallbacks()241gen_try_cmpxchg_order_fallback()
186{242{
187 local cmpxchg="$1"; shift;243 local cmpxchg="$1"; shift
244 local order="$1"; shift
245 local forder="${order:-_fence}"
188 246
189 printf "#ifndef arch_try_${cmpxchg}_relaxed\n"247 printf "#if defined(arch_try_${cmpxchg}${order})\n"
190 printf "#ifdef arch_try_${cmpxchg}\n"248 printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}${order}\n"
249
250 if [ "${order}" != "_relaxed" ]; then
251 printf "#elif defined(arch_try_${cmpxchg}_relaxed)\n"
252 printf "#define raw_try_${cmpxchg}${order}(...) \\\\\n"
253 printf " __atomic_op${forder}(arch_try_${cmpxchg}, __VA_ARGS__)\n"
254 fi
191 255
192 gen_basic_fallbacks "arch_try_${cmpxchg}"256 if [ ! -z "${order}" ]; then
257 printf "#elif defined(arch_try_${cmpxchg})\n"
258 printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}\n"
259 fi
193 260
194 printf "#endif /* arch_try_${cmpxchg} */\n\n"261 printf "#else\n"
262 gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
263 printf "#endif\n\n"
264}
265
266gen_try_cmpxchg_fallbacks()
267{
268 local cmpxchg="$1"; shift;
195 269
196 for order in "" "_acquire" "_release" "_relaxed"; do270 for order in "" "_acquire" "_release" "_relaxed"; do
197 gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"271 gen_try_cmpxchg_order_fallback "${cmpxchg}" "${order}"
198 done272 done
273}
199 274
200 printf "#else /* arch_try_${cmpxchg}_relaxed */\n"275gen_cmpxchg_local_fallbacks()
201 276{
202 gen_order_fallbacks "arch_try_${cmpxchg}"277 local cmpxchg="$1"; shift
203 278
204 printf "#endif /* arch_try_${cmpxchg}_relaxed */\n\n"279 printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
280 printf "#ifdef arch_try_${cmpxchg}\n"
281 printf "#define raw_try_${cmpxchg} arch_try_${cmpxchg}\n"
282 printf "#else\n"
283 gen_try_cmpxchg_fallback "${cmpxchg}" ""
284 printf "#endif\n\n"
205}285}
206 286
207cat << EOF287cat << EOF
@@ -217,16 +297,20 @@ cat << EOF · 9 unmodified lines
217 297
218EOF298EOF
219 299
220for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do300for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128"; do
221 gen_xchg_fallbacks "${xchg}"301 gen_xchg_fallbacks "${xchg}"
222done302done
223 303
224for cmpxchg in "cmpxchg" "cmpxchg64"; do304for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
225 gen_try_cmpxchg_fallbacks "${cmpxchg}"305 gen_try_cmpxchg_fallbacks "${cmpxchg}"
226done306done
227 307
228for cmpxchg in "cmpxchg_local" "cmpxchg64_local"; do308for cmpxchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local"; do
229 gen_try_cmpxchg_fallback "${cmpxchg}" ""309 gen_cmpxchg_local_fallbacks "${cmpxchg}" ""
310done
311
312for cmpxchg in "sync_cmpxchg"; do
313 printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
230done314done
231 315
232grep '^[a-z]' "$1" | while read name meta args; do316grep '^[a-z]' "$1" | while read name meta args; do
scripts/atomic/gen-atomic-instrumented.sh+16−26
@@ -68,12 +68,14 @@ gen_proto_order_variant() · 67 unmodified lines
68 local args="$(gen_args "$@")"68 local args="$(gen_args "$@")"
69 local retstmt="$(gen_ret_stmt "${meta}")"69 local retstmt="$(gen_ret_stmt "${meta}")"
70 70
71 gen_kerneldoc "" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
72
71cat <<EOF73cat <<EOF
72static __always_inline ${ret}74static __always_inline ${ret}
73${atomicname}(${params})75${atomicname}(${params})
74{76{
75${checks}77${checks}
76 ${retstmt}arch_${atomicname}(${args});78 ${retstmt}raw_${atomicname}(${args});
77}79}
78EOF80EOF
79 81
@@ -84,7 +86,6 @@ gen_xchg() · 4 unmodified lines
84{86{
85 local xchg="$1"; shift87 local xchg="$1"; shift
86 local order="$1"; shift88 local order="$1"; shift
87 local mult="$1"; shift
88 89
89 kcsan_barrier=""90 kcsan_barrier=""
90 if [ "${xchg%_local}" = "${xchg}" ]; then91 if [ "${xchg%_local}" = "${xchg}" ]; then
@@ -104,9 +105,9 @@ cat <<EOF · 13 unmodified lines
104EOF105EOF
105[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"106[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
106cat <<EOF107cat <<EOF
107 instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\108 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
108 instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\109 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\
109 arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\110 raw_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
110})111})
111EOF112EOF
112 113
@@ -119,8 +120,8 @@ cat <<EOF · 6 unmodified lines
119EOF120EOF
120[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"121[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
121cat <<EOF122cat <<EOF
122 instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\123 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
123 arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\124 raw_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\
124})125})
125EOF126EOF
126 127
@@ -134,15 +135,10 @@ cat << EOF · 7 unmodified lines
134// DO NOT MODIFY THIS FILE DIRECTLY135// DO NOT MODIFY THIS FILE DIRECTLY
135 136
136/*137/*
137 * This file provides wrappers with KASAN instrumentation for atomic operations.138 * This file provoides atomic operations with explicit instrumentation (e.g.
138 * To use this functionality an arch's atomic.h file needs to define all139 * KASAN, KCSAN), which should be used unless it is necessary to avoid
139 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include140 * instrumentation. Where it is necessary to aovid instrumenation, the
140 * this file at the end. This file provides atomic_read() that forwards to141 * raw_atomic*() operations should be used.
141 * arch_atomic_read() for actual atomic operation.
142 * Note: if an arch atomic operation is implemented by means of other atomic
143 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
144 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
145 * double instrumentation.
146 */142 */
147#ifndef _LINUX_ATOMIC_INSTRUMENTED_H143#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
148#define _LINUX_ATOMIC_INSTRUMENTED_H144#define _LINUX_ATOMIC_INSTRUMENTED_H
@@ -166,24 +162,18 @@ grep '^[a-z]' "$1" | while read name meta args; do · 17 unmodified lines
166done162done
167 163
168 164
169for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do165for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
170 for order in "" "_acquire" "_release" "_relaxed"; do166 for order in "" "_acquire" "_release" "_relaxed"; do
171 gen_xchg "${xchg}" "${order}" ""167 gen_xchg "${xchg}" "${order}"
172 printf "\n"168 printf "\n"
173 done169 done
174done170done
175 171
176for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do172for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do
177 gen_xchg "${xchg}" "" ""173 gen_xchg "${xchg}" ""
178 printf "\n"174 printf "\n"
179done175done
180 176
181gen_xchg "cmpxchg_double" "" "2 * "
182
183printf "\n\n"
184
185gen_xchg "cmpxchg_double_local" "" "2 * "
186
187cat <<EOF177cat <<EOF
188 178
189#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */179#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
scripts/atomic/gen-atomic-long.sh+18−20
@@ -32,24 +32,34 @@ gen_args_cast() · 31 unmodified lines
32 done32 done
33}33}
34 34
35#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...)35#gen_proto_order_variant(meta, pfx, name, sfx, order, arg...)
36gen_proto_order_variant()36gen_proto_order_variant()
37{37{
38 local meta="$1"; shift38 local meta="$1"; shift
39 local name="$1$2$3$4"; shift; shift; shift; shift39 local pfx="$1"; shift
40 local atomic="$1"; shift40 local name="$1"; shift
41 local int="$1"; shift41 local sfx="$1"; shift
42 local order="$1"; shift
43
44 local atomicname="${pfx}${name}${sfx}${order}"
42 45
43 local ret="$(gen_ret_type "${meta}" "long")"46 local ret="$(gen_ret_type "${meta}" "long")"
44 local params="$(gen_params "long" "atomic_long" "$@")"47 local params="$(gen_params "long" "atomic_long" "$@")"
45 local argscast="$(gen_args_cast "${int}" "${atomic}" "$@")"48 local argscast_32="$(gen_args_cast "int" "atomic" "$@")"
49 local argscast_64="$(gen_args_cast "s64" "atomic64" "$@")"
46 local retstmt="$(gen_ret_stmt "${meta}")"50 local retstmt="$(gen_ret_stmt "${meta}")"
47 51
52 gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "atomic_long" "long" "$@"
53
48cat <<EOF54cat <<EOF
49static __always_inline ${ret}55static __always_inline ${ret}
50arch_atomic_long_${name}(${params})56raw_atomic_long_${atomicname}(${params})
51{57{
52 ${retstmt}arch_${atomic}_${name}(${argscast});58#ifdef CONFIG_64BIT
59 ${retstmt}raw_atomic64_${atomicname}(${argscast_64});
60#else
61 ${retstmt}raw_atomic_${atomicname}(${argscast_32});
62#endif
53}63}
54 64
55EOF65EOF
@@ -79,24 +89,12 @@ typedef atomic_t atomic_long_t; · 23 unmodified lines
79#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed89#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
80#endif90#endif
81 91
82#ifdef CONFIG_64BIT
83
84EOF
85
86grep '^[a-z]' "$1" | while read name meta args; do
87 gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
88done
89
90cat <<EOF
91#else /* CONFIG_64BIT */
92
93EOF92EOF
94 93
95grep '^[a-z]' "$1" | while read name meta args; do94grep '^[a-z]' "$1" | while read name meta args; do
96 gen_proto "${meta}" "${name}" "atomic" "int" ${args}95 gen_proto "${meta}" "${name}" ${args}
97done96done
98 97
99cat <<EOF98cat <<EOF
100#endif /* CONFIG_64BIT */
101#endif /* _LINUX_ATOMIC_LONG_H */99#endif /* _LINUX_ATOMIC_LONG_H */
102EOF100EOF
scripts/atomic/kerneldoc/add+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic add with ${desc_order} ordering
4 * @i: ${int} value to add
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v + @i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * ${desc_return}
12 */
13EOF
scripts/atomic/kerneldoc/add_negative+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic add and test if negative with ${desc_order} ordering
4 * @i: ${int} value to add
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v + @i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * Return: @true if the resulting value of @v is negative, @false otherwise.
12 */
13EOF
scripts/atomic/kerneldoc/add_unless+18−0
@@ -0,0 +1,18 @@
1if [ -z "${pfx}" ]; then
2 desc_return="Return: @true if @v was updated, @false otherwise."
3fi
4
5cat <<EOF
6/**
7 * ${class}${atomicname}() - atomic add unless value with ${desc_order} ordering
8 * @v: pointer to ${atomic}_t
9 * @a: ${int} value to add
10 * @u: ${int} value to compare with
11 *
12 * If (@v != @u), atomically updates @v to (@v + @a) with ${desc_order} ordering.
13 *
14 * ${desc_noinstr}
15 *
16 * ${desc_return}
17 */
18EOF
scripts/atomic/kerneldoc/and+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic bitwise AND with ${desc_order} ordering
4 * @i: ${int} value
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v & @i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * ${desc_return}
12 */
13EOF
scripts/atomic/kerneldoc/andnot+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic bitwise AND NOT with ${desc_order} ordering
4 * @i: ${int} value
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v & ~@i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * ${desc_return}
12 */
13EOF
scripts/atomic/kerneldoc/cmpxchg+14−0
@@ -0,0 +1,14 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 * @old: ${int} value to compare with
6 * @new: ${int} value to assign
7 *
8 * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
9 *
10 * ${desc_noinstr}
11 *
12 * Return: The original value of @v.
13 */
14EOF
scripts/atomic/kerneldoc/dec+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic decrement with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * Atomically updates @v to (@v - 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * ${desc_return}
11 */
12EOF
scripts/atomic/kerneldoc/dec_and_test+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic decrement and test if zero with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * Atomically updates @v to (@v - 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * Return: @true if the resulting value of @v is zero, @false otherwise.
11 */
12EOF
scripts/atomic/kerneldoc/dec_if_positive+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic decrement if positive with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * If (@v > 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * Return: The old value of (@v - 1), regardless of whether @v was updated.
11 */
12EOF
scripts/atomic/kerneldoc/dec_unless_positive+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic decrement unless positive with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * If (@v <= 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * Return: @true if @v was updated, @false otherwise.
11 */
12EOF
scripts/atomic/kerneldoc/inc+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic increment with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * Atomically updates @v to (@v + 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * ${desc_return}
11 */
12EOF
scripts/atomic/kerneldoc/inc_and_test+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic increment and test if zero with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * Atomically updates @v to (@v + 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * Return: @true if the resulting value of @v is zero, @false otherwise.
11 */
12EOF
scripts/atomic/kerneldoc/inc_not_zero+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic increment unless zero with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * If (@v != 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * Return: @true if @v was updated, @false otherwise.
11 */
12EOF
scripts/atomic/kerneldoc/inc_unless_negative+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic increment unless negative with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * If (@v >= 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * Return: @true if @v was updated, @false otherwise.
11 */
12EOF
scripts/atomic/kerneldoc/or+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic bitwise OR with ${desc_order} ordering
4 * @i: ${int} value
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v | @i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * ${desc_return}
12 */
13EOF
scripts/atomic/kerneldoc/read+12−0
@@ -0,0 +1,12 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic load with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 *
6 * Atomically loads the value of @v with ${desc_order} ordering.
7 *
8 * ${desc_noinstr}
9 *
10 * Return: The value loaded from @v.
11 */
12EOF
scripts/atomic/kerneldoc/set+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic set with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 * @i: ${int} value to assign
6 *
7 * Atomically sets @v to @i with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * Return: Nothing.
12 */
13EOF
scripts/atomic/kerneldoc/sub+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic subtract with ${desc_order} ordering
4 * @i: ${int} value to subtract
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v - @i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * ${desc_return}
12 */
13EOF
scripts/atomic/kerneldoc/sub_and_test+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic subtract and test if zero with ${desc_order} ordering
4 * @i: ${int} value to add
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v - @i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * Return: @true if the resulting value of @v is zero, @false otherwise.
12 */
13EOF
scripts/atomic/kerneldoc/try_cmpxchg+15−0
@@ -0,0 +1,15 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 * @old: pointer to ${int} value to compare with
6 * @new: ${int} value to assign
7 *
8 * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
9 * Otherwise, updates @old to the current value of @v.
10 *
11 * ${desc_noinstr}
12 *
13 * Return: @true if the exchange occured, @false otherwise.
14 */
15EOF
scripts/atomic/kerneldoc/xchg+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic exchange with ${desc_order} ordering
4 * @v: pointer to ${atomic}_t
5 * @new: ${int} value to assign
6 *
7 * Atomically updates @v to @new with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * Return: The original value of @v.
12 */
13EOF
scripts/atomic/kerneldoc/xor+13−0
@@ -0,0 +1,13 @@
1cat <<EOF
2/**
3 * ${class}${atomicname}() - atomic bitwise XOR with ${desc_order} ordering
4 * @i: ${int} value
5 * @v: pointer to ${atomic}_t
6 *
7 * Atomically updates @v to (@v ^ @i) with ${desc_order} ordering.
8 *
9 * ${desc_noinstr}
10 *
11 * ${desc_return}
12 */
13EOF
scripts/kernel-doc+1−1
@@ -64,7 +64,7 @@ my $type_constant = '\b``([^\`]+)``\b'; · 63 unmodified lines
64my $type_constant2 = '\%([-_\w]+)';64my $type_constant2 = '\%([-_\w]+)';
65my $type_func = '(\w+)\(\)';65my $type_func = '(\w+)\(\)';
66my $type_param = '\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)';66my $type_param = '\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)';
67my $type_param_ref = '([\!]?)\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)';67my $type_param_ref = '([\!~]?)\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)';
68my $type_fp_param = '\@(\w+)\(\)'; # Special RST handling for func ptr params68my $type_fp_param = '\@(\w+)\(\)'; # Special RST handling for func ptr params
69my $type_fp_param2 = '\@(\w+->\S+)\(\)'; # Special RST handling for structs with func ptr params69my $type_fp_param2 = '\@(\w+->\S+)\(\)'; # Special RST handling for structs with func ptr params
70my $type_env = '(\$\w+)';70my $type_env = '(\$\w+)';
scripts/min-tool-version.sh+5−1
@@ -17,7 +17,11 @@ binutils) · 16 unmodified lines
17 echo 2.25.017 echo 2.25.0
18 ;;18 ;;
19gcc)19gcc)
20 echo 5.1.020 if [ "$SRCARCH" = parisc ]; then
21 echo 11.0.0
22 else
23 echo 5.1.0
24 fi
21 ;;25 ;;
22llvm)26llvm)
23 if [ "$SRCARCH" = s390 ]; then27 if [ "$SRCARCH" = s390 ]; then