Merge tag 'locking-core-2023-06-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar: - Introduce cmpxchg128() -- aka. the demise of cmpxchg_double() The cmpxchg128() family of functions is basically & functionally the same as cmpxchg_double(), but with a saner interface. Instead of a 6-parameter horror that forced u128 - u64/u64-halves layout details on the interface and exposed users to complexity, fragility & bugs, use a natural 3-parameter interface with u128 types. - Restructure the generated atomic headers, and add kerneldoc comments for all of the generic atomic{,64,_long}_t operations. The generated definitions are much cleaner now, and come with documentation. - Implement lock_set_cmp_fn() on lockdep, for defining an ordering when taking multiple locks of the same type. This gets rid of one use of lockdep_set_novalidate_class() in the bcache code. - Fix raw_cpu_generic_try_cmpxchg() bug due to an unintended variable shadowing generating garbage code on Clang on certain ARM builds. * tag 'locking-core-2023-06-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (43 commits) locking/atomic: scripts: fix ${atomic}_dec_if_positive() kerneldoc percpu: Fix self-assignment of __old in raw_cpu_generic_try_cmpxchg() locking/atomic: treewide: delete arch_atomic_*() kerneldoc locking/atomic: docs: Add atomic operations to the driver basic API documentation locking/atomic: scripts: generate kerneldoc comments docs: scripts: kernel-doc: accept bitwise negation like ~@var locking/atomic: scripts: simplify raw_atomic*() definitions locking/atomic: scripts: simplify raw_atomic_long*() definitions locking/atomic: scripts: split pfx/name/sfx/order locking/atomic: scripts: restructure fallback ifdeffery locking/atomic: scripts: build raw_atomic_long*() directly locking/atomic: treewide: use raw_atomic*_<op>() locking/atomic: scripts: add trivial raw_atomic*_<op>() locking/atomic: scripts: factor out order template generation locking/atomic: scripts: remove leftover "${mult}" locking/atomic: scripts: remove bogus order parameter locking/atomic: xtensa: add preprocessor symbols locking/atomic: x86: add preprocessor symbols locking/atomic: sparc: add preprocessor symbols locking/atomic: sh: add preprocessor symbols ...

Linus Torvalds committed 3 years ago

Commit bc6cb4d2 parents ed3b792 b33eb50136 files · +9922 −4161Browse files

Unified · Split

136 files

Documentation/core-api/this_cpu_ops.rst +0 −2
Documentation/driver-api/basics.rst +7 −1
arch/alpha/include/asm/atomic.h +0 −35
arch/arc/include/asm/atomic-spinlock.h +9 −0
arch/arc/include/asm/atomic.h +0 −24
arch/arc/include/asm/atomic64-arcv2.h +2 −17
arch/arm/include/asm/assembler.h +17 −0
arch/arm/include/asm/atomic.h +11 −4
arch/arm/include/asm/sync_bitops.h +25 −4
arch/arm/lib/bitops.h +11 −3
arch/arm/lib/testchangebit.S +4 −0
arch/arm/lib/testclearbit.S +4 −0
arch/arm/lib/testsetbit.S +4 −0
arch/arm64/include/asm/atomic.h +0 −28
arch/arm64/include/asm/atomic_ll_sc.h +32 −24
arch/arm64/include/asm/atomic_lse.h +17 −22
arch/arm64/include/asm/cmpxchg.h +14 −34
arch/arm64/include/asm/percpu.h +20 −10
arch/csky/include/asm/atomic.h +0 −35
arch/hexagon/include/asm/atomic.h +9 −60
arch/ia64/include/asm/atomic.h +0 −7
arch/loongarch/include/asm/atomic.h +0 −56
arch/m68k/include/asm/atomic.h +12 −6
arch/mips/include/asm/atomic.h +0 −11
arch/openrisc/include/asm/atomic.h +0 −3
arch/parisc/include/asm/atomic.h +18 −9
arch/powerpc/include/asm/atomic.h +0 −24
arch/powerpc/kernel/smp.c +6 −6
arch/riscv/include/asm/atomic.h +0 −72
arch/s390/include/asm/cmpxchg.h +6 −26
arch/s390/include/asm/cpu_mf.h +1 −1
arch/s390/include/asm/percpu.h +16 −18
arch/s390/kernel/perf_cpum_sf.c +3 −13
arch/sh/include/asm/atomic-grb.h +9 −0
arch/sh/include/asm/atomic-irq.h +9 −0
arch/sh/include/asm/atomic-llsc.h +9 −0
arch/sh/include/asm/atomic.h +0 −3
arch/sparc/include/asm/atomic_32.h +16 −2
arch/sparc/include/asm/atomic_64.h +18 −11
arch/x86/include/asm/atomic.h +0 −87
arch/x86/include/asm/atomic64_32.h +0 −76
arch/x86/include/asm/atomic64_64.h +0 −81
arch/x86/include/asm/cmpxchg.h +0 −25
arch/x86/include/asm/cmpxchg_32.h +1 −1
arch/x86/include/asm/cmpxchg_64.h +66 −1
arch/x86/include/asm/percpu.h +61 −41
arch/x86/kernel/alternative.c +2 −2
arch/x86/kernel/cpu/mce/core.c +8 −8
arch/x86/kernel/nmi.c +1 −1
arch/x86/kernel/pvclock.c +2 −2
arch/x86/kvm/x86.c +1 −1
arch/x86/lib/Makefile +2 −1
arch/x86/lib/cmpxchg16b_emu.S +25 −18
arch/x86/lib/cmpxchg8b_emu.S +53 −14
arch/xtensa/include/asm/atomic.h +9 −3
drivers/iommu/amd/amd_iommu_types.h +7 −2
drivers/iommu/amd/iommu.c +4 −6
drivers/iommu/intel/irq_remapping.c +2 −6
drivers/md/bcache/btree.c +22 −1
drivers/md/bcache/btree.h +2 −2
include/asm-generic/atomic.h +0 −3
include/asm-generic/bitops/atomic.h +6 −6
include/asm-generic/bitops/lock.h +4 −4
include/asm-generic/percpu.h +152 −49
include/crypto/b128ops.h +3 −11
include/linux/atomic/atomic-arch-fallback.h +3576 −1587
include/linux/atomic/atomic-instrumented.h +3194 −361
include/linux/atomic/atomic-long.h +1435 −687
include/linux/context_tracking.h +2 −2
include/linux/context_tracking_state.h +1 −1
include/linux/cpumask.h +1 −1
include/linux/dmar.h +66 −59
include/linux/jump_label.h +1 −1
include/linux/lockdep.h +8 −0
include/linux/lockdep_types.h +8 −0
include/linux/percpu-defs.h +13 −32
include/linux/slub_def.h +9 −3
include/linux/types.h +5 −0
include/uapi/linux/types.h +4 −0
kernel/context_tracking.c +6 −6
kernel/locking/lockdep.c +87 −31
kernel/sched/clock.c +1 −1
lib/crypto/curve25519-hacl64.c +0 −2
lib/crypto/poly1305-donna64.c +0 −2
mm/slab.h +46 −7
mm/slub.c +82 −57
scripts/atomic/atomic-tbl.sh +108 −4
scripts/atomic/atomics.tbl +1 −1
scripts/atomic/fallbacks/acquire +0 −4
scripts/atomic/fallbacks/add_negative +1 −13
scripts/atomic/fallbacks/add_unless +1 −14
scripts/atomic/fallbacks/andnot +1 −5
scripts/atomic/fallbacks/cmpxchg +3 −0
scripts/atomic/fallbacks/dec +1 −5
scripts/atomic/fallbacks/dec_and_test +1 −13
scripts/atomic/fallbacks/dec_if_positive +2 −6
scripts/atomic/fallbacks/dec_unless_positive +2 −6
scripts/atomic/fallbacks/fence +0 −4
scripts/atomic/fallbacks/fetch_add_unless +2 −15
scripts/atomic/fallbacks/inc +1 −5
scripts/atomic/fallbacks/inc_and_test +1 −13
scripts/atomic/fallbacks/inc_not_zero +1 −12
scripts/atomic/fallbacks/inc_unless_negative +2 −6
scripts/atomic/fallbacks/read_acquire +1 −5
scripts/atomic/fallbacks/release +0 −4
scripts/atomic/fallbacks/set_release +1 −5
scripts/atomic/fallbacks/sub_and_test +1 −14
scripts/atomic/fallbacks/try_cmpxchg +1 −5
scripts/atomic/fallbacks/xchg +3 −0
scripts/atomic/gen-atomic-fallback.sh +175 −91
scripts/atomic/gen-atomic-instrumented.sh +16 −26
scripts/atomic/gen-atomic-long.sh +18 −20
scripts/atomic/kerneldoc/add +13 −0
scripts/atomic/kerneldoc/add_negative +13 −0
scripts/atomic/kerneldoc/add_unless +18 −0
scripts/atomic/kerneldoc/and +13 −0
scripts/atomic/kerneldoc/andnot +13 −0
scripts/atomic/kerneldoc/cmpxchg +14 −0
scripts/atomic/kerneldoc/dec +12 −0
scripts/atomic/kerneldoc/dec_and_test +12 −0
scripts/atomic/kerneldoc/dec_if_positive +12 −0
scripts/atomic/kerneldoc/dec_unless_positive +12 −0
scripts/atomic/kerneldoc/inc +12 −0
scripts/atomic/kerneldoc/inc_and_test +12 −0
scripts/atomic/kerneldoc/inc_not_zero +12 −0
scripts/atomic/kerneldoc/inc_unless_negative +12 −0
scripts/atomic/kerneldoc/or +13 −0
scripts/atomic/kerneldoc/read +12 −0
scripts/atomic/kerneldoc/set +13 −0
scripts/atomic/kerneldoc/sub +13 −0
scripts/atomic/kerneldoc/sub_and_test +13 −0
scripts/atomic/kerneldoc/try_cmpxchg +15 −0
scripts/atomic/kerneldoc/xchg +13 −0
scripts/atomic/kerneldoc/xor +13 −0
scripts/kernel-doc +1 −1
scripts/min-tool-version.sh +5 −1

Documentation/core-api/this_cpu_ops.rst+0−2

@@ -53,7 +53,6 @@ preemption and interrupts:: · 52 unmodified lines
53	53	this_cpu_add_return(pcp, val)
54	54	this_cpu_xchg(pcp, nval)
55	55	this_cpu_cmpxchg(pcp, oval, nval)
56		− this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
57	56	this_cpu_sub(pcp, val)
58	57	this_cpu_inc(pcp)
59	58	this_cpu_dec(pcp)
@@ -242,7 +241,6 @@ safe:: · 182 unmodified lines
242	241	__this_cpu_add_return(pcp, val)
243	242	__this_cpu_xchg(pcp, nval)
244	243	__this_cpu_cmpxchg(pcp, oval, nval)
245		− __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
246	244	__this_cpu_sub(pcp, val)
247	245	__this_cpu_inc(pcp)
248	246	__this_cpu_dec(pcp)

Documentation/driver-api/basics.rst+7−1

@@ -84,7 +84,13 @@ Reference counting · 83 unmodified lines
84	84	Atomics
85	85	-------
86	86
87		−.. kernel-doc:: arch/x86/include/asm/atomic.h
	87	+.. kernel-doc:: include/linux/atomic/atomic-instrumented.h
	88	+ :internal:
	89	+
	90	+.. kernel-doc:: include/linux/atomic/atomic-arch-fallback.h
	91	+ :internal:
	92	+
	93	+.. kernel-doc:: include/linux/atomic/atomic-long.h
88	94	:internal:
89	95
90	96	Kernel objects manipulation

arch/alpha/include/asm/atomic.h+0−35

@@ -200,25 +200,6 @@ ATOMIC_OPS(xor, xor) · 199 unmodified lines
200	200	#undef ATOMIC_OP_RETURN
201	201	#undef ATOMIC_OP
202	202
203		−#define arch_atomic64_cmpxchg(v, old, new) \
204		− (arch_cmpxchg(&((v)->counter), old, new))
205		−#define arch_atomic64_xchg(v, new) \
206		− (arch_xchg(&((v)->counter), new))
207		−
208		−#define arch_atomic_cmpxchg(v, old, new) \
209		− (arch_cmpxchg(&((v)->counter), old, new))
210		−#define arch_atomic_xchg(v, new) \
211		− (arch_xchg(&((v)->counter), new))
212		−
213		−/**
214		− * arch_atomic_fetch_add_unless - add unless the number is a given value
215		− * @v: pointer of type atomic_t
216		− * @a: the amount to add to v...
217		− * @u: ...unless v is equal to u.
218		− *
219		− * Atomically adds @a to @v, so long as it was not @u.
220		− * Returns the old value of @v.
221		− */
222	203	static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
223	204	{
224	205	int c, new, old;
@@ -242,15 +223,6 @@ static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) · 17 unmodified lines
242	223	}
243	224	#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
244	225
245		−/**
246		− * arch_atomic64_fetch_add_unless - add unless the number is a given value
247		− * @v: pointer of type atomic64_t
248		− * @a: the amount to add to v...
249		− * @u: ...unless v is equal to u.
250		− *
251		− * Atomically adds @a to @v, so long as it was not @u.
252		− * Returns the old value of @v.
253		− */
254	226	static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
255	227	{
256	228	s64 c, new, old;
@@ -274,13 +246,6 @@ static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u · 17 unmodified lines
274	246	}
275	247	#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
276	248
277		−/*
278		− * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
279		− * @v: pointer of type atomic_t
280		− *
281		− * The function returns the old value of *v minus 1, even if
282		− * the atomic variable, v, was not decremented.
283		− */
284	249	static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
285	250	{
286	251	s64 old, tmp;

arch/arc/include/asm/atomic-spinlock.h+9−0

@@ -81,6 +81,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 80 unmodified lines
81	81	ATOMIC_OPS(add, +=, add)
82	82	ATOMIC_OPS(sub, -=, sub)
83	83
	84	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	85	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	86	+#define arch_atomic_add_return arch_atomic_add_return
	87	+#define arch_atomic_sub_return arch_atomic_sub_return
	88	+
84	89	#undef ATOMIC_OPS
85	90	#define ATOMIC_OPS(op, c_op, asm_op) \
86	91	ATOMIC_OP(op, c_op, asm_op) \
@@ -92,7 +97,11 @@ ATOMIC_OPS(or, \|=, or) · 5 unmodified lines
92	97	ATOMIC_OPS(xor, ^=, xor)
93	98
94	99	#define arch_atomic_andnot arch_atomic_andnot
	100	+
	101	+#define arch_atomic_fetch_and arch_atomic_fetch_and
95	102	#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
	103	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	104	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
96	105
97	106	#undef ATOMIC_OPS
98	107	#undef ATOMIC_FETCH_OP

arch/arc/include/asm/atomic.h+0−24

@@ -22,30 +22,6 @@ · 21 unmodified lines
22	22	#include <asm/atomic-spinlock.h>
23	23	#endif
24	24
25		−#define arch_atomic_cmpxchg(v, o, n) \
26		−({ \
27		− arch_cmpxchg(&((v)->counter), (o), (n)); \
28		−})
29		−
30		−#ifdef arch_cmpxchg_relaxed
31		−#define arch_atomic_cmpxchg_relaxed(v, o, n) \
32		−({ \
33		− arch_cmpxchg_relaxed(&((v)->counter), (o), (n)); \
34		−})
35		−#endif
36		−
37		−#define arch_atomic_xchg(v, n) \
38		−({ \
39		− arch_xchg(&((v)->counter), (n)); \
40		−})
41		−
42		−#ifdef arch_xchg_relaxed
43		−#define arch_atomic_xchg_relaxed(v, n) \
44		−({ \
45		− arch_xchg_relaxed(&((v)->counter), (n)); \
46		−})
47		−#endif
48		−
49	25	/*
50	26	* 64-bit atomics
51	27	*/

arch/arc/include/asm/atomic64-arcv2.h+2−17

@@ -159,6 +159,7 @@ arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) · 158 unmodified lines
159	159
160	160	return prev;
161	161	}
	162	+#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
162	163
163	164	static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
164	165	{
@@ -179,14 +180,7 @@ static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) · 14 unmodified lines
179	180
180	181	return prev;
181	182	}
182		−
183		−/**
184		− * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
185		− * @v: pointer of type atomic64_t
186		− *
187		− * The function returns the old value of *v minus 1, even if
188		− * the atomic variable, v, was not decremented.
189		− */
	183	+#define arch_atomic64_xchg arch_atomic64_xchg
190	184
191	185	static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
192	186	{
@@ -212,15 +206,6 @@ static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) · 19 unmodified lines
212	206	}
213	207	#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
214	208
215		−/**
216		− * arch_atomic64_fetch_add_unless - add unless the number is a given value
217		− * @v: pointer of type atomic64_t
218		− * @a: the amount to add to v...
219		− * @u: ...unless v is equal to u.
220		− *
221		− * Atomically adds @a to @v, if it was not @u.
222		− * Returns the old value of @v
223		− */
224	209	static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
225	210	{
226	211	s64 old, temp;

arch/arm/include/asm/assembler.h+17−0

@@ -394,6 +394,23 @@ ALT_UP_B(.L0_\@) · 393 unmodified lines
394	394	#endif
395	395	.endm
396	396
	397	+/*
	398	+ * Raw SMP data memory barrier
	399	+ */
	400	+ .macro __smp_dmb mode
	401	+#if __LINUX_ARM_ARCH__ >= 7
	402	+ .ifeqs "\mode","arm"
	403	+ dmb ish
	404	+ .else
	405	+ W(dmb) ish
	406	+ .endif
	407	+#elif __LINUX_ARM_ARCH__ == 6
	408	+ mcr p15, 0, r0, c7, c10, 5 @ dmb
	409	+#else
	410	+ .error "Incompatible SMP platform"
	411	+#endif
	412	+ .endm
	413	+
397	414	#if defined(CONFIG_CPU_V7M)
398	415	/*
399	416	* setmode is used to assert to be in svc mode during boot. For v7-M

arch/arm/include/asm/atomic.h+11−4

@@ -197,6 +197,16 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 196 unmodified lines
197	197	return val; \
198	198	}
199	199
	200	+#define arch_atomic_add_return arch_atomic_add_return
	201	+#define arch_atomic_sub_return arch_atomic_sub_return
	202	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	203	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	204	+
	205	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	206	+#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
	207	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	208	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	209	+
200	210	static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
201	211	{
202	212	int ret;
@@ -210,8 +220,7 @@ static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) · 7 unmodified lines
210	220
211	221	return ret;
212	222	}
213		−
214		−#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
	223	+#define arch_atomic_cmpxchg arch_atomic_cmpxchg
215	224
216	225	#endif /* __LINUX_ARM_ARCH__ */
217	226
@@ -240,8 +249,6 @@ ATOMIC_OPS(xor, ^=, eor) · 22 unmodified lines
240	249	#undef ATOMIC_OP_RETURN
241	250	#undef ATOMIC_OP
242	251
243		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
244		−
245	252	#ifndef CONFIG_GENERIC_ATOMIC64
246	253	typedef struct {
247	254	s64 counter;

arch/arm/include/asm/sync_bitops.h+25−4

@@ -14,14 +14,35 @@ · 13 unmodified lines
14	14	* ops which are SMP safe even on a UP kernel.
15	15	*/
16	16
	17	+/*
	18	+ * Unordered
	19	+ */
	20	+
17	21	#define sync_set_bit(nr, p) _set_bit(nr, p)
18	22	#define sync_clear_bit(nr, p) _clear_bit(nr, p)
19	23	#define sync_change_bit(nr, p) _change_bit(nr, p)
20		−#define sync_test_and_set_bit(nr, p) _test_and_set_bit(nr, p)
21		−#define sync_test_and_clear_bit(nr, p) _test_and_clear_bit(nr, p)
22		−#define sync_test_and_change_bit(nr, p) _test_and_change_bit(nr, p)
23	24	#define sync_test_bit(nr, addr) test_bit(nr, addr)
24		−#define arch_sync_cmpxchg arch_cmpxchg
25	25
	26	+/*
	27	+ * Fully ordered
	28	+ */
	29	+
	30	+int _sync_test_and_set_bit(int nr, volatile unsigned long * p);
	31	+#define sync_test_and_set_bit(nr, p) _sync_test_and_set_bit(nr, p)
	32	+
	33	+int _sync_test_and_clear_bit(int nr, volatile unsigned long * p);
	34	+#define sync_test_and_clear_bit(nr, p) _sync_test_and_clear_bit(nr, p)
	35	+
	36	+int _sync_test_and_change_bit(int nr, volatile unsigned long * p);
	37	+#define sync_test_and_change_bit(nr, p) _sync_test_and_change_bit(nr, p)
	38	+
	39	+#define arch_sync_cmpxchg(ptr, old, new) \
	40	+({ \
	41	+ __typeof__(*(ptr)) __ret; \
	42	+ __smp_mb__before_atomic(); \
	43	+ __ret = arch_cmpxchg_relaxed((ptr), (old), (new)); \
	44	+ __smp_mb__after_atomic(); \
	45	+ __ret; \
	46	+})
26	47
27	48	#endif

arch/arm/lib/bitops.h+11−3

@@ -28,7 +28,7 @@ UNWIND( .fnend ) · 27 unmodified lines
28	28	ENDPROC(\name )
29	29	.endm
30	30
31		− .macro testop, name, instr, store
	31	+ .macro __testop, name, instr, store, barrier
32	32	ENTRY( \name )
33	33	UNWIND( .fnstart )
34	34	ands ip, r1, #3
@@ -38,7 +38,7 @@ UNWIND( .fnstart ) · 3 unmodified lines
38	38	mov r0, r0, lsr #5
39	39	add r1, r1, r0, lsl #2 @ Get word offset
40	40	mov r3, r2, lsl r3 @ create mask
41		− smp_dmb
	41	+ \barrier
42	42	#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
43	43	.arch_extension mp
44	44	ALT_SMP(W(pldw) [r1])
@@ -50,13 +50,21 @@ UNWIND( .fnstart ) · 5 unmodified lines
50	50	strex ip, r2, [r1]
51	51	cmp ip, #0
52	52	bne 1b
53		− smp_dmb
	53	+ \barrier
54	54	cmp r0, #0
55	55	movne r0, #1
56	56	2: bx lr
57	57	UNWIND( .fnend )
58	58	ENDPROC(\name )
59	59	.endm
	60	+
	61	+ .macro testop, name, instr, store
	62	+ __testop \name, \instr, \store, smp_dmb
	63	+ .endm
	64	+
	65	+ .macro sync_testop, name, instr, store
	66	+ __testop \name, \instr, \store, __smp_dmb
	67	+ .endm
60	68	#else
61	69	.macro bitop, name, instr
62	70	ENTRY( \name )

arch/arm/lib/testchangebit.S+4−0

@@ -10,3 +10,7 @@ · 9 unmodified lines
10	10	.text
11	11
12	12	testop _test_and_change_bit, eor, str
	13	+
	14	+#if __LINUX_ARM_ARCH__ >= 6
	15	+sync_testop _sync_test_and_change_bit, eor, str
	16	+#endif

arch/arm/lib/testclearbit.S+4−0

@@ -10,3 +10,7 @@ · 9 unmodified lines
10	10	.text
11	11
12	12	testop _test_and_clear_bit, bicne, strne
	13	+
	14	+#if __LINUX_ARM_ARCH__ >= 6
	15	+sync_testop _sync_test_and_clear_bit, bicne, strne
	16	+#endif

arch/arm/lib/testsetbit.S+4−0

@@ -10,3 +10,7 @@ · 9 unmodified lines
10	10	.text
11	11
12	12	testop _test_and_set_bit, orreq, streq
	13	+
	14	+#if __LINUX_ARM_ARCH__ >= 6
	15	+sync_testop _sync_test_and_set_bit, orreq, streq
	16	+#endif

arch/arm64/include/asm/atomic.h+0−28

@@ -142,24 +142,6 @@ static __always_inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 141 unmodified lines
142	142	#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
143	143	#define arch_atomic_fetch_xor arch_atomic_fetch_xor
144	144
145		−#define arch_atomic_xchg_relaxed(v, new) \
146		− arch_xchg_relaxed(&((v)->counter), (new))
147		−#define arch_atomic_xchg_acquire(v, new) \
148		− arch_xchg_acquire(&((v)->counter), (new))
149		−#define arch_atomic_xchg_release(v, new) \
150		− arch_xchg_release(&((v)->counter), (new))
151		−#define arch_atomic_xchg(v, new) \
152		− arch_xchg(&((v)->counter), (new))
153		−
154		−#define arch_atomic_cmpxchg_relaxed(v, old, new) \
155		− arch_cmpxchg_relaxed(&((v)->counter), (old), (new))
156		−#define arch_atomic_cmpxchg_acquire(v, old, new) \
157		− arch_cmpxchg_acquire(&((v)->counter), (old), (new))
158		−#define arch_atomic_cmpxchg_release(v, old, new) \
159		− arch_cmpxchg_release(&((v)->counter), (old), (new))
160		−#define arch_atomic_cmpxchg(v, old, new) \
161		− arch_cmpxchg(&((v)->counter), (old), (new))
162		−
163	145	#define arch_atomic_andnot arch_atomic_andnot
164	146
165	147	/*
@@ -209,16 +191,6 @@ static __always_inline long arch_atomic64_dec_if_positive(atomic64_t *v) · 43 unmodified lines
209	191	#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
210	192	#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
211	193
212		−#define arch_atomic64_xchg_relaxed arch_atomic_xchg_relaxed
213		−#define arch_atomic64_xchg_acquire arch_atomic_xchg_acquire
214		−#define arch_atomic64_xchg_release arch_atomic_xchg_release
215		−#define arch_atomic64_xchg arch_atomic_xchg
216		−
217		−#define arch_atomic64_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed
218		−#define arch_atomic64_cmpxchg_acquire arch_atomic_cmpxchg_acquire
219		−#define arch_atomic64_cmpxchg_release arch_atomic_cmpxchg_release
220		−#define arch_atomic64_cmpxchg arch_atomic_cmpxchg
221		−
222	194	#define arch_atomic64_andnot arch_atomic64_andnot
223	195
224	196	#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive

arch/arm64/include/asm/atomic_ll_sc.h+32−24

@@ -294,38 +294,46 @@ __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) · 293 unmodified lines
294	294
295	295	#undef __CMPXCHG_CASE
296	296
297		−#define __CMPXCHG_DBL(name, mb, rel, cl) \
298		−static __always_inline long \
299		−__ll_sc__cmpxchg_double##name(unsigned long old1, \
300		− unsigned long old2, \
301		− unsigned long new1, \
302		− unsigned long new2, \
303		− volatile void *ptr) \
	297	+union __u128_halves {
	298	+ u128 full;
	299	+ struct {
	300	+ u64 low, high;
	301	+ };
	302	+};
	303	+
	304	+#define __CMPXCHG128(name, mb, rel, cl...) \
	305	+static __always_inline u128 \
	306	+__ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
304	307	{ \
305		− unsigned long tmp, ret; \
	308	+ union __u128_halves r, o = { .full = (old) }, \
	309	+ n = { .full = (new) }; \
	310	+ unsigned int tmp; \
306	311	\
307		− asm volatile("// __cmpxchg_double" #name "\n" \
308		− " prfm pstl1strm, %2\n" \
309		− "1: ldxp %0, %1, %2\n" \
310		− " eor %0, %0, %3\n" \
311		− " eor %1, %1, %4\n" \
312		− " orr %1, %0, %1\n" \
313		− " cbnz %1, 2f\n" \
314		− " st" #rel "xp %w0, %5, %6, %2\n" \
315		− " cbnz %w0, 1b\n" \
	312	+ asm volatile("// __cmpxchg128" #name "\n" \
	313	+ " prfm pstl1strm, %[v]\n" \
	314	+ "1: ldxp %[rl], %[rh], %[v]\n" \
	315	+ " cmp %[rl], %[ol]\n" \
	316	+ " ccmp %[rh], %[oh], 0, eq\n" \
	317	+ " b.ne 2f\n" \
	318	+ " st" #rel "xp %w[tmp], %[nl], %[nh], %[v]\n" \
	319	+ " cbnz %w[tmp], 1b\n" \
316	320	" " #mb "\n" \
317	321	"2:" \
318		− : "=&r" (tmp), "=&r" (ret), "+Q" ((__uint128_t )ptr) \
319		− : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \
320		− : cl); \
	322	+ : [v] "+Q" ((u128 )ptr), \
	323	+ [rl] "=&r" (r.low), [rh] "=&r" (r.high), \
	324	+ [tmp] "=&r" (tmp) \
	325	+ : [ol] "r" (o.low), [oh] "r" (o.high), \
	326	+ [nl] "r" (n.low), [nh] "r" (n.high) \
	327	+ : "cc", ##cl); \
321	328	\
322		− return ret; \
	329	+ return r.full; \
323	330	}
324	331
325		−__CMPXCHG_DBL( , , , )
326		−__CMPXCHG_DBL(_mb, dmb ish, l, "memory")
	332	+__CMPXCHG128( , , )
	333	+__CMPXCHG128(_mb, dmb ish, l, "memory")
	334	+
	335	+#undef __CMPXCHG128
327	336
328		−#undef __CMPXCHG_DBL
329	337	#undef K
330	338
331	339	#endif /* __ASM_ATOMIC_LL_SC_H */

arch/arm64/include/asm/atomic_lse.h+17−22

@@ -281,40 +281,35 @@ __CMPXCHG_CASE(x, , mb_, 64, al, "memory") · 280 unmodified lines
281	281
282	282	#undef __CMPXCHG_CASE
283	283
284		−#define __CMPXCHG_DBL(name, mb, cl...) \
285		−static __always_inline long \
286		−__lse__cmpxchg_double##name(unsigned long old1, \
287		− unsigned long old2, \
288		− unsigned long new1, \
289		− unsigned long new2, \
290		− volatile void *ptr) \
	284	+#define __CMPXCHG128(name, mb, cl...) \
	285	+static __always_inline u128 \
	286	+__lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
291	287	{ \
292		− unsigned long oldval1 = old1; \
293		− unsigned long oldval2 = old2; \
294		− register unsigned long x0 asm ("x0") = old1; \
295		− register unsigned long x1 asm ("x1") = old2; \
296		− register unsigned long x2 asm ("x2") = new1; \
297		− register unsigned long x3 asm ("x3") = new2; \
	288	+ union __u128_halves r, o = { .full = (old) }, \
	289	+ n = { .full = (new) }; \
	290	+ register unsigned long x0 asm ("x0") = o.low; \
	291	+ register unsigned long x1 asm ("x1") = o.high; \
	292	+ register unsigned long x2 asm ("x2") = n.low; \
	293	+ register unsigned long x3 asm ("x3") = n.high; \
298	294	register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
299	295	\
300	296	asm volatile( \
301	297	__LSE_PREAMBLE \
302	298	" casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
303		− " eor %[old1], %[old1], %[oldval1]\n" \
304		− " eor %[old2], %[old2], %[oldval2]\n" \
305		− " orr %[old1], %[old1], %[old2]" \
306	299	: [old1] "+&r" (x0), [old2] "+&r" (x1), \
307		− [v] "+Q" ((__uint128_t )ptr) \
	300	+ [v] "+Q" ((u128 )ptr) \
308	301	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
309		− [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
	302	+ [oldval1] "r" (o.low), [oldval2] "r" (o.high) \
310	303	: cl); \
311	304	\
312		− return x0; \
	305	+ r.low = x0; r.high = x1; \
	306	+ \
	307	+ return r.full; \
313	308	}
314	309
315		−__CMPXCHG_DBL( , )
316		−__CMPXCHG_DBL(_mb, al, "memory")
	310	+__CMPXCHG128( , )
	311	+__CMPXCHG128(_mb, al, "memory")
317	312
318		−#undef __CMPXCHG_DBL
	313	+#undef __CMPXCHG128
319	314
320	315	#endif /* __ASM_ATOMIC_LSE_H */

arch/arm64/include/asm/cmpxchg.h+14−34

@@ -130,21 +130,18 @@ __CMPXCHG_CASE(mb_, 64) · 129 unmodified lines
130	130
131	131	#undef __CMPXCHG_CASE
132	132
133		−#define __CMPXCHG_DBL(name) \
134		−static inline long __cmpxchg_double##name(unsigned long old1, \
135		− unsigned long old2, \
136		− unsigned long new1, \
137		− unsigned long new2, \
138		− volatile void *ptr) \
	133	+#define __CMPXCHG128(name) \
	134	+static inline u128 __cmpxchg128##name(volatile u128 *ptr, \
	135	+ u128 old, u128 new) \
139	136	{ \
140		− return __lse_ll_sc_body(_cmpxchg_double##name, \
141		− old1, old2, new1, new2, ptr); \
	137	+ return __lse_ll_sc_body(_cmpxchg128##name, \
	138	+ ptr, old, new); \
142	139	}
143	140
144		−__CMPXCHG_DBL( )
145		−__CMPXCHG_DBL(_mb)
	141	+__CMPXCHG128( )
	142	+__CMPXCHG128(_mb)
146	143
147		−#undef __CMPXCHG_DBL
	144	+#undef __CMPXCHG128
148	145
149	146	#define __CMPXCHG_GEN(sfx) \
150	147	static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
@@ -198,34 +195,17 @@ __CMPXCHG_GEN(_mb) · 47 unmodified lines
198	195	#define arch_cmpxchg64 arch_cmpxchg
199	196	#define arch_cmpxchg64_local arch_cmpxchg_local
200	197
201		−/* cmpxchg_double */
202		−#define system_has_cmpxchg_double() 1
203		−
204		−#define __cmpxchg_double_check(ptr1, ptr2) \
205		−({ \
206		− if (sizeof(*(ptr1)) != 8) \
207		− BUILD_BUG(); \
208		− VM_BUG_ON((unsigned long )(ptr2) - (unsigned long )(ptr1) != 1); \
209		−})
	198	+/* cmpxchg128 */
	199	+#define system_has_cmpxchg128() 1
210	200
211		−#define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
	201	+#define arch_cmpxchg128(ptr, o, n) \
212	202	({ \
213		− int __ret; \
214		− __cmpxchg_double_check(ptr1, ptr2); \
215		− __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
216		− (unsigned long)(n1), (unsigned long)(n2), \
217		− ptr1); \
218		− __ret; \
	203	+ __cmpxchg128_mb((ptr), (o), (n)); \
219	204	})
220	205
221		−#define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
	206	+#define arch_cmpxchg128_local(ptr, o, n) \
222	207	({ \
223		− int __ret; \
224		− __cmpxchg_double_check(ptr1, ptr2); \
225		− __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
226		− (unsigned long)(n1), (unsigned long)(n2), \
227		− ptr1); \
228		− __ret; \
	208	+ __cmpxchg128((ptr), (o), (n)); \
229	209	})
230	210
231	211	#define __CMPWAIT_CASE(w, sfx, sz) \

arch/arm64/include/asm/percpu.h+20−10

@@ -140,17 +140,11 @@ PERCPU_RET_OP(add, add, ldadd) · 139 unmodified lines
140	140	* re-enabling preemption for preemptible kernels, but doing that in a way
141	141	* which builds inside a module would mean messing directly with the preempt
142	142	* count. If you do this, peterz and tglx will hunt you down.
	143	+ *
	144	+ * Not to mention it'll break the actual preemption model for missing a
	145	+ * preemption point when TIF_NEED_RESCHED gets set while preemption is
	146	+ * disabled.
143	147	*/
144		−#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
145		−({ \
146		− int __ret; \
147		− preempt_disable_notrace(); \
148		− __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
149		− raw_cpu_ptr(&(ptr2)), \
150		− o1, o2, n1, n2); \
151		− preempt_enable_notrace(); \
152		− __ret; \
153		−})
154	148
155	149	#define _pcp_protect(op, pcp, ...) \
156	150	({ \
@@ -240,6 +234,22 @@ PERCPU_RET_OP(add, add, ldadd) · 83 unmodified lines
240	234	#define this_cpu_cmpxchg_8(pcp, o, n) \
241	235	_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
242	236
	237	+#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
	238	+
	239	+#define this_cpu_cmpxchg128(pcp, o, n) \
	240	+({ \
	241	+ typedef typeof(pcp) pcp_op_T__; \
	242	+ u128 old__, new__, ret__; \
	243	+ pcp_op_T__ *ptr__; \
	244	+ old__ = o; \
	245	+ new__ = n; \
	246	+ preempt_disable_notrace(); \
	247	+ ptr__ = raw_cpu_ptr(&(pcp)); \
	248	+ ret__ = cmpxchg128_local((void *)ptr__, old__, new__); \
	249	+ preempt_enable_notrace(); \
	250	+ ret__; \
	251	+})
	252	+
243	253	#ifdef __KVM_NVHE_HYPERVISOR__
244	254	extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
245	255	#define __per_cpu_offset

arch/csky/include/asm/atomic.h+0−35

@@ -195,41 +195,6 @@ arch_atomic_dec_if_positive(atomic_t *v) · 194 unmodified lines
195	195	}
196	196	#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
197	197
198		−#define ATOMIC_OP() \
199		−static __always_inline \
200		−int arch_atomic_xchg_relaxed(atomic_t *v, int n) \
201		−{ \
202		− return __xchg_relaxed(n, &(v->counter), 4); \
203		−} \
204		−static __always_inline \
205		−int arch_atomic_cmpxchg_relaxed(atomic_t *v, int o, int n) \
206		−{ \
207		− return __cmpxchg_relaxed(&(v->counter), o, n, 4); \
208		−} \
209		−static __always_inline \
210		−int arch_atomic_cmpxchg_acquire(atomic_t *v, int o, int n) \
211		−{ \
212		− return __cmpxchg_acquire(&(v->counter), o, n, 4); \
213		−} \
214		−static __always_inline \
215		−int arch_atomic_cmpxchg(atomic_t *v, int o, int n) \
216		−{ \
217		− return __cmpxchg(&(v->counter), o, n, 4); \
218		−}
219		−
220		−#define ATOMIC_OPS() \
221		− ATOMIC_OP()
222		−
223		−ATOMIC_OPS()
224		−
225		−#define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed
226		−#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed
227		−#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
228		−#define arch_atomic_cmpxchg arch_atomic_cmpxchg
229		−
230		−#undef ATOMIC_OPS
231		−#undef ATOMIC_OP
232		−
233	198	#else
234	199	#include <asm-generic/atomic.h>
235	200	#endif

arch/hexagon/include/asm/atomic.h+9−60

@@ -28,58 +28,8 @@ static inline void arch_atomic_set(atomic_t *v, int new) · 27 unmodified lines
28	28
29	29	#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i))
30	30
31		−/**
32		− * arch_atomic_read - reads a word, atomically
33		− * @v: pointer to atomic value
34		− *
35		− * Assumes all word reads on our architecture are atomic.
36		− */
37	31	#define arch_atomic_read(v) READ_ONCE((v)->counter)
38	32
39		−/**
40		− * arch_atomic_xchg - atomic
41		− * @v: pointer to memory to change
42		− * @new: new value (technically passed in a register -- see xchg)
43		− */
44		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
45		−
46		−
47		−/**
48		− * arch_atomic_cmpxchg - atomic compare-and-exchange values
49		− * @v: pointer to value to change
50		− * @old: desired old value to match
51		− * @new: new value to put in
52		− *
53		− * Parameters are then pointer, value-in-register, value-in-register,
54		− * and the output is the old value.
55		− *
56		− * Apparently this is complicated for archs that don't support
57		− * the memw_locked like we do (or it's broken or whatever).
58		− *
59		− * Kind of the lynchpin of the rest of the generically defined routines.
60		− * Remember V2 had that bug with dotnew predicate set by memw_locked.
61		− *
62		− * "old" is "expected" old val, __oldval is actual old value
63		− */
64		−static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
65		−{
66		− int __oldval;
67		−
68		− asm volatile(
69		− "1: %0 = memw_locked(%1);\n"
70		− " { P0 = cmp.eq(%0,%2);\n"
71		− " if (!P0.new) jump:nt 2f; }\n"
72		− " memw_locked(%1,P0) = %3;\n"
73		− " if (!P0) jump 1b;\n"
74		− "2:\n"
75		− : "=&r" (__oldval)
76		− : "r" (&v->counter), "r" (old), "r" (new)
77		− : "memory", "p0"
78		− );
79		−
80		− return __oldval;
81		−}
82		−
83	33	#define ATOMIC_OP(op) \
84	34	static inline void arch_atomic_##op(int i, atomic_t *v) \
85	35	{ \
@@ -135,6 +85,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 49 unmodified lines
135	85	ATOMIC_OPS(add)
136	86	ATOMIC_OPS(sub)
137	87
	88	+#define arch_atomic_add_return arch_atomic_add_return
	89	+#define arch_atomic_sub_return arch_atomic_sub_return
	90	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	91	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	92	+
138	93	#undef ATOMIC_OPS
139	94	#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
140	95
@@ -142,21 +97,15 @@ ATOMIC_OPS(and) · 1 unmodified line
142	97	ATOMIC_OPS(or)
143	98	ATOMIC_OPS(xor)
144	99
	100	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	101	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	102	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	103	+
145	104	#undef ATOMIC_OPS
146	105	#undef ATOMIC_FETCH_OP
147	106	#undef ATOMIC_OP_RETURN
148	107	#undef ATOMIC_OP
149	108
150		−/**
151		− * arch_atomic_fetch_add_unless - add unless the number is a given value
152		− * @v: pointer to value
153		− * @a: amount to add
154		− * @u: unless value is equal to u
155		− *
156		− * Returns old value.
157		− *
158		− */
159		−
160	109	static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
161	110	{
162	111	int __oldval;

arch/ia64/include/asm/atomic.h+0−7

@@ -207,13 +207,6 @@ ATOMIC64_FETCH_OP(xor, ^) · 206 unmodified lines
207	207	#undef ATOMIC64_FETCH_OP
208	208	#undef ATOMIC64_OP
209	209
210		−#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), old, new))
211		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
212		−
213		−#define arch_atomic64_cmpxchg(v, old, new) \
214		− (arch_cmpxchg(&((v)->counter), old, new))
215		−#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new))
216		−
217	210	#define arch_atomic_add(i,v) (void)arch_atomic_add_return((i), (v))
218	211	#define arch_atomic_sub(i,v) (void)arch_atomic_sub_return((i), (v))
219	212

arch/loongarch/include/asm/atomic.h+0−56

@@ -29,21 +29,7 @@ · 28 unmodified lines
29	29
30	30	#define ATOMIC_INIT(i) { (i) }
31	31
32		−/*
33		− * arch_atomic_read - read atomic variable
34		− * @v: pointer of type atomic_t
35		− *
36		− * Atomically reads the value of @v.
37		− */
38	32	#define arch_atomic_read(v) READ_ONCE((v)->counter)
39		−
40		−/*
41		− * arch_atomic_set - set atomic variable
42		− * @v: pointer of type atomic_t
43		− * @i: required value
44		− *
45		− * Atomically sets the value of @v to @i.
46		− */
47	33	#define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i))
48	34
49	35	#define ATOMIC_OP(op, I, asm_op) \
@@ -139,14 +125,6 @@ static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) · 89 unmodified lines
139	125	}
140	126	#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
141	127
142		−/*
143		− * arch_atomic_sub_if_positive - conditionally subtract integer from atomic variable
144		− * @i: integer value to subtract
145		− * @v: pointer of type atomic_t
146		− *
147		− * Atomically test @v and subtract @i if @v is greater or equal than @i.
148		− * The function returns the old value of @v minus @i.
149		− */
150	128	static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
151	129	{
152	130	int result;
@@ -181,31 +159,13 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v) · 28 unmodified lines
181	159	return result;
182	160	}
183	161
184		−#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
185		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
186		−
187		−/*
188		− * arch_atomic_dec_if_positive - decrement by 1 if old value positive
189		− * @v: pointer of type atomic_t
190		− */
191	162	#define arch_atomic_dec_if_positive(v) arch_atomic_sub_if_positive(1, v)
192	163
193	164	#ifdef CONFIG_64BIT
194	165
195	166	#define ATOMIC64_INIT(i) { (i) }
196	167
197		−/*
198		− * arch_atomic64_read - read atomic variable
199		− * @v: pointer of type atomic64_t
200		− *
201		− */
202	168	#define arch_atomic64_read(v) READ_ONCE((v)->counter)
203		−
204		−/*
205		− * arch_atomic64_set - set atomic variable
206		− * @v: pointer of type atomic64_t
207		− * @i: required value
208		− */
209	169	#define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
210	170
211	171	#define ATOMIC64_OP(op, I, asm_op) \
@@ -300,14 +260,6 @@ static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u) · 88 unmodified lines
300	260	}
301	261	#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
302	262
303		−/*
304		− * arch_atomic64_sub_if_positive - conditionally subtract integer from atomic variable
305		− * @i: integer value to subtract
306		− * @v: pointer of type atomic64_t
307		− *
308		− * Atomically test @v and subtract @i if @v is greater or equal than @i.
309		− * The function returns the old value of @v minus @i.
310		− */
311	263	static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
312	264	{
313	265	long result;
@@ -342,14 +294,6 @@ static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v) · 28 unmodified lines
342	294	return result;
343	295	}
344	296
345		−#define arch_atomic64_cmpxchg(v, o, n) \
346		− ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
347		−#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
348		−
349		−/*
350		− * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
351		− * @v: pointer of type atomic64_t
352		− */
353	297	#define arch_atomic64_dec_if_positive(v) arch_atomic64_sub_if_positive(1, v)
354	298
355	299	#endif /* CONFIG_64BIT */

arch/m68k/include/asm/atomic.h+12−6

@@ -106,6 +106,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \ · 105 unmodified lines
106	106	ATOMIC_OPS(add, +=, add)
107	107	ATOMIC_OPS(sub, -=, sub)
108	108
	109	+#define arch_atomic_add_return arch_atomic_add_return
	110	+#define arch_atomic_sub_return arch_atomic_sub_return
	111	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	112	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	113	+
109	114	#undef ATOMIC_OPS
110	115	#define ATOMIC_OPS(op, c_op, asm_op) \
111	116	ATOMIC_OP(op, c_op, asm_op) \
@@ -115,6 +120,10 @@ ATOMIC_OPS(and, &=, and) · 3 unmodified lines
115	120	ATOMIC_OPS(or, \|=, or)
116	121	ATOMIC_OPS(xor, ^=, eor)
117	122
	123	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	124	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	125	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	126	+
118	127	#undef ATOMIC_OPS
119	128	#undef ATOMIC_FETCH_OP
120	129	#undef ATOMIC_OP_RETURN
@@ -158,12 +167,7 @@ static inline int arch_atomic_inc_and_test(atomic_t *v) · 37 unmodified lines
158	167	}
159	168	#define arch_atomic_inc_and_test arch_atomic_inc_and_test
160	169
161		−#ifdef CONFIG_RMW_INSNS
162		−
163		−#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n)))
164		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
165		−
166		−#else /* !CONFIG_RMW_INSNS */
	170	+#ifndef CONFIG_RMW_INSNS
167	171
168	172	static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
169	173	{
@@ -177,6 +181,7 @@ static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) · 7 unmodified lines
177	181	local_irq_restore(flags);
178	182	return prev;
179	183	}
	184	+#define arch_atomic_cmpxchg arch_atomic_cmpxchg
180	185
181	186	static inline int arch_atomic_xchg(atomic_t *v, int new)
182	187	{
@@ -189,6 +194,7 @@ static inline int arch_atomic_xchg(atomic_t *v, int new) · 6 unmodified lines
189	194	local_irq_restore(flags);
190	195	return prev;
191	196	}
	197	+#define arch_atomic_xchg arch_atomic_xchg
192	198
193	199	#endif /* !CONFIG_RMW_INSNS */
194	200

arch/mips/include/asm/atomic.h+0−11

@@ -33,17 +33,6 @@ static __always_inline void arch_##pfx##_set(pfx##_t *v, type i) \ · 32 unmodified lines
33	33	{ \
34	34	WRITE_ONCE(v->counter, i); \
35	35	} \
36		− \
37		−static __always_inline type \
38		−arch_##pfx##_cmpxchg(pfx##_t *v, type o, type n) \
39		−{ \
40		− return arch_cmpxchg(&v->counter, o, n); \
41		−} \
42		− \
43		−static __always_inline type arch_##pfx##_xchg(pfx##_t *v, type n) \
44		−{ \
45		− return arch_xchg(&v->counter, n); \
46		−}
47	36
48	37	ATOMIC_OPS(atomic, int)
49	38

arch/openrisc/include/asm/atomic.h+0−3

@@ -130,7 +130,4 @@ static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) · 129 unmodified lines
130	130
131	131	#include <asm/cmpxchg.h>
132	132
133		−#define arch_atomic_xchg(ptr, v) (arch_xchg(&(ptr)->counter, (v)))
134		−#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), (old), (new)))
135		−
136	133	#endif /* __ASM_OPENRISC_ATOMIC_H */

arch/parisc/include/asm/atomic.h+18−9

@@ -73,10 +73,6 @@ static __inline__ int arch_atomic_read(const atomic_t *v) · 72 unmodified lines
73	73	return READ_ONCE((v)->counter);
74	74	}
75	75
76		−/* exported interface */
77		−#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
78		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
79		−
80	76	#define ATOMIC_OP(op, c_op) \
81	77	static __inline__ void arch_atomic_##op(int i, atomic_t *v) \
82	78	{ \
@@ -122,6 +118,11 @@ static __inline__ int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 39 unmodified lines
122	118	ATOMIC_OPS(add, +=)
123	119	ATOMIC_OPS(sub, -=)
124	120
	121	+#define arch_atomic_add_return arch_atomic_add_return
	122	+#define arch_atomic_sub_return arch_atomic_sub_return
	123	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	124	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	125	+
125	126	#undef ATOMIC_OPS
126	127	#define ATOMIC_OPS(op, c_op) \
127	128	ATOMIC_OP(op, c_op) \
@@ -131,6 +132,10 @@ ATOMIC_OPS(and, &=) · 3 unmodified lines
131	132	ATOMIC_OPS(or, \|=)
132	133	ATOMIC_OPS(xor, ^=)
133	134
	135	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	136	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	137	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	138	+
134	139	#undef ATOMIC_OPS
135	140	#undef ATOMIC_FETCH_OP
136	141	#undef ATOMIC_OP_RETURN
@@ -185,6 +190,11 @@ static __inline__ s64 arch_atomic64_fetch_##op(s64 i, atomic64_t *v) \ · 48 unmodified lines
185	190	ATOMIC64_OPS(add, +=)
186	191	ATOMIC64_OPS(sub, -=)
187	192
	193	+#define arch_atomic64_add_return arch_atomic64_add_return
	194	+#define arch_atomic64_sub_return arch_atomic64_sub_return
	195	+#define arch_atomic64_fetch_add arch_atomic64_fetch_add
	196	+#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
	197	+
188	198	#undef ATOMIC64_OPS
189	199	#define ATOMIC64_OPS(op, c_op) \
190	200	ATOMIC64_OP(op, c_op) \
@@ -194,6 +204,10 @@ ATOMIC64_OPS(and, &=) · 3 unmodified lines
194	204	ATOMIC64_OPS(or, \|=)
195	205	ATOMIC64_OPS(xor, ^=)
196	206
	207	+#define arch_atomic64_fetch_and arch_atomic64_fetch_and
	208	+#define arch_atomic64_fetch_or arch_atomic64_fetch_or
	209	+#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
	210	+
197	211	#undef ATOMIC64_OPS
198	212	#undef ATOMIC64_FETCH_OP
199	213	#undef ATOMIC64_OP_RETURN
@@ -218,11 +232,6 @@ arch_atomic64_read(const atomic64_t *v) · 18 unmodified lines
218	232	return READ_ONCE((v)->counter);
219	233	}
220	234
221		−/* exported interface */
222		−#define arch_atomic64_cmpxchg(v, o, n) \
223		− ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
224		−#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new))
225		−
226	235	#endif /* !CONFIG_64BIT */
227	236
228	237

arch/powerpc/include/asm/atomic.h+0−24

@@ -126,18 +126,6 @@ ATOMIC_OPS(xor, xor, "", K) · 125 unmodified lines
126	126	#undef ATOMIC_OP_RETURN_RELAXED
127	127	#undef ATOMIC_OP
128	128
129		−#define arch_atomic_cmpxchg(v, o, n) \
130		− (arch_cmpxchg(&((v)->counter), (o), (n)))
131		−#define arch_atomic_cmpxchg_relaxed(v, o, n) \
132		− arch_cmpxchg_relaxed(&((v)->counter), (o), (n))
133		−#define arch_atomic_cmpxchg_acquire(v, o, n) \
134		− arch_cmpxchg_acquire(&((v)->counter), (o), (n))
135		−
136		−#define arch_atomic_xchg(v, new) \
137		− (arch_xchg(&((v)->counter), new))
138		−#define arch_atomic_xchg_relaxed(v, new) \
139		− arch_xchg_relaxed(&((v)->counter), (new))
140		−
141	129	/**
142	130	* atomic_fetch_add_unless - add unless the number is a given value
143	131	* @v: pointer of type atomic_t
@@ -396,18 +384,6 @@ static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v) · 252 unmodified lines
396	384	}
397	385	#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
398	386
399		−#define arch_atomic64_cmpxchg(v, o, n) \
400		− (arch_cmpxchg(&((v)->counter), (o), (n)))
401		−#define arch_atomic64_cmpxchg_relaxed(v, o, n) \
402		− arch_cmpxchg_relaxed(&((v)->counter), (o), (n))
403		−#define arch_atomic64_cmpxchg_acquire(v, o, n) \
404		− arch_cmpxchg_acquire(&((v)->counter), (o), (n))
405		−
406		−#define arch_atomic64_xchg(v, new) \
407		− (arch_xchg(&((v)->counter), new))
408		−#define arch_atomic64_xchg_relaxed(v, new) \
409		− arch_xchg_relaxed(&((v)->counter), (new))
410		−
411	387	/**
412	388	* atomic64_fetch_add_unless - add unless the number is a given value
413	389	* @v: pointer of type atomic64_t

arch/powerpc/kernel/smp.c+6−6

@@ -417,9 +417,9 @@ noinstr static void nmi_ipi_lock_start(unsigned long *flags) · 416 unmodified lines
417	417	{
418	418	raw_local_irq_save(*flags);
419	419	hard_irq_disable();
420		− while (arch_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
	420	+ while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
421	421	raw_local_irq_restore(*flags);
422		− spin_until_cond(arch_atomic_read(&__nmi_ipi_lock) == 0);
	422	+ spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0);
423	423	raw_local_irq_save(*flags);
424	424	hard_irq_disable();
425	425	}
@@ -427,15 +427,15 @@ noinstr static void nmi_ipi_lock_start(unsigned long *flags) · 1 unmodified line
427	427
428	428	noinstr static void nmi_ipi_lock(void)
429	429	{
430		− while (arch_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
431		− spin_until_cond(arch_atomic_read(&__nmi_ipi_lock) == 0);
	430	+ while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
	431	+ spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0);
432	432	}
433	433
434	434	noinstr static void nmi_ipi_unlock(void)
435	435	{
436	436	smp_mb();
437		− WARN_ON(arch_atomic_read(&__nmi_ipi_lock) != 1);
438		− arch_atomic_set(&__nmi_ipi_lock, 0);
	437	+ WARN_ON(raw_atomic_read(&__nmi_ipi_lock) != 1);
	438	+ raw_atomic_set(&__nmi_ipi_lock, 0);
439	439	}
440	440
441	441	noinstr static void nmi_ipi_unlock_end(unsigned long *flags)

arch/riscv/include/asm/atomic.h+0−72

@@ -238,78 +238,6 @@ static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, · 237 unmodified lines
238	238	#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
239	239	#endif
240	240
241		−/*
242		− * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
243		− * {cmp,}xchg and the operations that return, so they need a full barrier.
244		− */
245		−#define ATOMIC_OP(c_t, prefix, size) \
246		−static __always_inline \
247		−c_t arch_atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \
248		−{ \
249		− return __xchg_relaxed(&(v->counter), n, size); \
250		−} \
251		−static __always_inline \
252		−c_t arch_atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \
253		−{ \
254		− return __xchg_acquire(&(v->counter), n, size); \
255		−} \
256		−static __always_inline \
257		−c_t arch_atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \
258		−{ \
259		− return __xchg_release(&(v->counter), n, size); \
260		−} \
261		−static __always_inline \
262		−c_t arch_atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \
263		−{ \
264		− return __arch_xchg(&(v->counter), n, size); \
265		−} \
266		−static __always_inline \
267		−c_t arch_atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \
268		− c_t o, c_t n) \
269		−{ \
270		− return __cmpxchg_relaxed(&(v->counter), o, n, size); \
271		−} \
272		−static __always_inline \
273		−c_t arch_atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \
274		− c_t o, c_t n) \
275		−{ \
276		− return __cmpxchg_acquire(&(v->counter), o, n, size); \
277		−} \
278		−static __always_inline \
279		−c_t arch_atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \
280		− c_t o, c_t n) \
281		−{ \
282		− return __cmpxchg_release(&(v->counter), o, n, size); \
283		−} \
284		−static __always_inline \
285		−c_t arch_atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \
286		−{ \
287		− return __cmpxchg(&(v->counter), o, n, size); \
288		−}
289		−
290		−#ifdef CONFIG_GENERIC_ATOMIC64
291		−#define ATOMIC_OPS() \
292		− ATOMIC_OP(int, , 4)
293		−#else
294		−#define ATOMIC_OPS() \
295		− ATOMIC_OP(int, , 4) \
296		− ATOMIC_OP(s64, 64, 8)
297		−#endif
298		−
299		−ATOMIC_OPS()
300		−
301		−#define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed
302		−#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
303		−#define arch_atomic_xchg_release arch_atomic_xchg_release
304		−#define arch_atomic_xchg arch_atomic_xchg
305		−#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed
306		−#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
307		−#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
308		−#define arch_atomic_cmpxchg arch_atomic_cmpxchg
309		−
310		−#undef ATOMIC_OPS
311		−#undef ATOMIC_OP
312		−
313	241	static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
314	242	{
315	243	int prev, rc;

arch/s390/include/asm/cmpxchg.h+6−26

@@ -190,38 +190,18 @@ static __always_inline unsigned long __cmpxchg(unsigned long address, · 189 unmodified lines
190	190	#define arch_cmpxchg_local arch_cmpxchg
191	191	#define arch_cmpxchg64_local arch_cmpxchg
192	192
193		−#define system_has_cmpxchg_double() 1
	193	+#define system_has_cmpxchg128() 1
194	194
195		−static __always_inline int __cmpxchg_double(unsigned long p1, unsigned long p2,
196		− unsigned long o1, unsigned long o2,
197		− unsigned long n1, unsigned long n2)
	195	+static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
198	196	{
199		− union register_pair old = { .even = o1, .odd = o2, };
200		− union register_pair new = { .even = n1, .odd = n2, };
201		− int cc;
202		−
203	197	asm volatile(
204	198	" cdsg %[old],%[new],%[ptr]\n"
205		− " ipm %[cc]\n"
206		− " srl %[cc],28\n"
207		− : [cc] "=&d" (cc), [old] "+&d" (old.pair)
208		− : [new] "d" (new.pair),
209		− [ptr] "QS" ((unsigned long )p1), "Q" ((unsigned long )p2)
	199	+ : [old] "+d" (old), [ptr] "+QS" (*ptr)
	200	+ : [new] "d" (new)
210	201	: "memory", "cc");
211		− return !cc;
	202	+ return old;
212	203	}
213	204
214		−#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \
215		−({ \
216		− typeof(p1) __p1 = (p1); \
217		− typeof(p2) __p2 = (p2); \
218		− \
219		− BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
220		− BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
221		− VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\
222		− __cmpxchg_double((unsigned long)__p1, (unsigned long)__p2, \
223		− (unsigned long)(o1), (unsigned long)(o2), \
224		− (unsigned long)(n1), (unsigned long)(n2)); \
225		−})
	205	+#define arch_cmpxchg128 arch_cmpxchg128
226	206
227	207	#endif /* __ASM_CMPXCHG_H */

arch/s390/include/asm/cpu_mf.h+1−1

@@ -140,7 +140,7 @@ union hws_trailer_header { · 139 unmodified lines
140	140	unsigned int dsdes:16; /* 48-63: size of diagnostic SDE */
141	141	unsigned long long overflow; /* 64 - Overflow Count */
142	142	};
143		− __uint128_t val;
	143	+ u128 val;
144	144	};
145	145
146	146	struct hws_trailer_entry {

arch/s390/include/asm/percpu.h+16−18

@@ -148,6 +148,22 @@ · 147 unmodified lines
148	148	#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149	149	#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
150	150
	151	+#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
	152	+
	153	+#define this_cpu_cmpxchg128(pcp, oval, nval) \
	154	+({ \
	155	+ typedef typeof(pcp) pcp_op_T__; \
	156	+ u128 old__, new__, ret__; \
	157	+ pcp_op_T__ *ptr__; \
	158	+ old__ = oval; \
	159	+ new__ = nval; \
	160	+ preempt_disable_notrace(); \
	161	+ ptr__ = raw_cpu_ptr(&(pcp)); \
	162	+ ret__ = cmpxchg128((void *)ptr__, old__, new__); \
	163	+ preempt_enable_notrace(); \
	164	+ ret__; \
	165	+})
	166	+
151	167	#define arch_this_cpu_xchg(pcp, nval) \
152	168	({ \
153	169	typeof(pcp) *ptr__; \
@@ -164,24 +180,6 @@ · 10 unmodified lines
164	180	#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
165	181	#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
166	182
167		−#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \
168		−({ \
169		− typeof(pcp1) *p1__; \
170		− typeof(pcp2) *p2__; \
171		− int ret__; \
172		− \
173		− preempt_disable_notrace(); \
174		− p1__ = raw_cpu_ptr(&(pcp1)); \
175		− p2__ = raw_cpu_ptr(&(pcp2)); \
176		− ret__ = __cmpxchg_double((unsigned long)p1__, (unsigned long)p2__, \
177		− (unsigned long)(o1), (unsigned long)(o2), \
178		− (unsigned long)(n1), (unsigned long)(n2)); \
179		− preempt_enable_notrace(); \
180		− ret__; \
181		−})
182		−
183		−#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
184		−
185	183	#include <asm-generic/percpu.h>
186	184
187	185	#endif /* __ARCH_S390_PERCPU__ */

arch/s390/kernel/perf_cpum_sf.c+3−13

@@ -1271,16 +1271,6 @@ static void hw_collect_samples(struct perf_event event, unsigned long sdbt, · 1270 unmodified lines
1271	1271	}
1272	1272	}
1273	1273
1274		−static inline __uint128_t __cdsg(__uint128_t *ptr, __uint128_t old, __uint128_t new)
1275		−{
1276		− asm volatile(
1277		− " cdsg %[old],%[new],%[ptr]\n"
1278		− : [old] "+d" (old), [ptr] "+QS" (*ptr)
1279		− : [new] "d" (new)
1280		− : "memory", "cc");
1281		− return old;
1282		−}
1283		−
1284	1274	/* hw_perf_event_update() - Process sampling buffer
1285	1275	* @event: The perf event
1286	1276	* @flush_all: Flag to also flush partially filled sample-data-blocks
@@ -1352,7 +1342,7 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) · 65 unmodified lines
1352	1342	new.f = 0;
1353	1343	new.a = 1;
1354	1344	new.overflow = 0;
1355		− prev.val = __cdsg(&te->header.val, old.val, new.val);
	1345	+ prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1356	1346	} while (prev.val != old.val);
1357	1347
1358	1348	/* Advance to next sample-data-block */
@@ -1562,7 +1552,7 @@ static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index, · 203 unmodified lines
1562	1552	}
1563	1553	new.a = 1;
1564	1554	new.overflow = 0;
1565		− prev.val = __cdsg(&te->header.val, old.val, new.val);
	1555	+ prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1566	1556	} while (prev.val != old.val);
1567	1557	return true;
1568	1558	}
@@ -1636,7 +1626,7 @@ static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range, · 67 unmodified lines
1636	1626	new.a = 1;
1637	1627	else
1638	1628	new.a = 0;
1639		− prev.val = __cdsg(&te->header.val, old.val, new.val);
	1629	+ prev.val = cmpxchg128(&te->header.val, old.val, new.val);
1640	1630	} while (prev.val != old.val);
1641	1631	*overflow += orig_overflow;
1642	1632	}

arch/sh/include/asm/atomic-grb.h+9−0

@@ -71,6 +71,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 70 unmodified lines
71	71	ATOMIC_OPS(add)
72	72	ATOMIC_OPS(sub)
73	73
	74	+#define arch_atomic_add_return arch_atomic_add_return
	75	+#define arch_atomic_sub_return arch_atomic_sub_return
	76	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	77	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	78	+
74	79	#undef ATOMIC_OPS
75	80	#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
76	81
@@ -78,6 +83,10 @@ ATOMIC_OPS(and) · 1 unmodified line
78	83	ATOMIC_OPS(or)
79	84	ATOMIC_OPS(xor)
80	85
	86	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	87	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	88	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	89	+
81	90	#undef ATOMIC_OPS
82	91	#undef ATOMIC_FETCH_OP
83	92	#undef ATOMIC_OP_RETURN

arch/sh/include/asm/atomic-irq.h+9−0

@@ -55,6 +55,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 54 unmodified lines
55	55	ATOMIC_OPS(add, +=)
56	56	ATOMIC_OPS(sub, -=)
57	57
	58	+#define arch_atomic_add_return arch_atomic_add_return
	59	+#define arch_atomic_sub_return arch_atomic_sub_return
	60	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	61	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	62	+
58	63	#undef ATOMIC_OPS
59	64	#define ATOMIC_OPS(op, c_op) \
60	65	ATOMIC_OP(op, c_op) \
@@ -64,6 +69,10 @@ ATOMIC_OPS(and, &=) · 3 unmodified lines
64	69	ATOMIC_OPS(or, \|=)
65	70	ATOMIC_OPS(xor, ^=)
66	71
	72	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	73	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	74	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	75	+
67	76	#undef ATOMIC_OPS
68	77	#undef ATOMIC_FETCH_OP
69	78	#undef ATOMIC_OP_RETURN

arch/sh/include/asm/atomic-llsc.h+9−0

@@ -73,6 +73,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ · 72 unmodified lines
73	73	ATOMIC_OPS(add)
74	74	ATOMIC_OPS(sub)
75	75
	76	+#define arch_atomic_add_return arch_atomic_add_return
	77	+#define arch_atomic_sub_return arch_atomic_sub_return
	78	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	79	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	80	+
76	81	#undef ATOMIC_OPS
77	82	#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
78	83
@@ -80,6 +85,10 @@ ATOMIC_OPS(and) · 1 unmodified line
80	85	ATOMIC_OPS(or)
81	86	ATOMIC_OPS(xor)
82	87
	88	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	89	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	90	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	91	+
83	92	#undef ATOMIC_OPS
84	93	#undef ATOMIC_FETCH_OP
85	94	#undef ATOMIC_OP_RETURN

arch/sh/include/asm/atomic.h+0−3

@@ -30,9 +30,6 @@ · 29 unmodified lines
30	30	#include <asm/atomic-irq.h>
31	31	#endif
32	32
33		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
34		−#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
35		−
36	33	#endif /* CONFIG_CPU_J2 */
37	34
38	35	#endif /* __ASM_SH_ATOMIC_H */

arch/sparc/include/asm/atomic_32.h+16−2

@@ -19,17 +19,31 @@ · 18 unmodified lines
19	19	#include <asm-generic/atomic64.h>
20	20
21	21	int arch_atomic_add_return(int, atomic_t *);
	22	+#define arch_atomic_add_return arch_atomic_add_return
	23	+
22	24	int arch_atomic_fetch_add(int, atomic_t *);
	25	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	26	+
23	27	int arch_atomic_fetch_and(int, atomic_t *);
	28	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	29	+
24	30	int arch_atomic_fetch_or(int, atomic_t *);
	31	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	32	+
25	33	int arch_atomic_fetch_xor(int, atomic_t *);
	34	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	35	+
26	36	int arch_atomic_cmpxchg(atomic_t *, int, int);
	37	+#define arch_atomic_cmpxchg arch_atomic_cmpxchg
	38	+
27	39	int arch_atomic_xchg(atomic_t *, int);
28		−int arch_atomic_fetch_add_unless(atomic_t *, int, int);
29		−void arch_atomic_set(atomic_t *, int);
	40	+#define arch_atomic_xchg arch_atomic_xchg
30	41
	42	+int arch_atomic_fetch_add_unless(atomic_t *, int, int);
31	43	#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
32	44
	45	+void arch_atomic_set(atomic_t *, int);
	46	+
33	47	#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i))
34	48
35	49	#define arch_atomic_read(v) READ_ONCE((v)->counter)

arch/sparc/include/asm/atomic_64.h+18−11

@@ -37,6 +37,16 @@ s64 arch_atomic64_fetch_##op(s64, atomic64_t *); · 36 unmodified lines
37	37	ATOMIC_OPS(add)
38	38	ATOMIC_OPS(sub)
39	39
	40	+#define arch_atomic_add_return arch_atomic_add_return
	41	+#define arch_atomic_sub_return arch_atomic_sub_return
	42	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	43	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	44	+
	45	+#define arch_atomic64_add_return arch_atomic64_add_return
	46	+#define arch_atomic64_sub_return arch_atomic64_sub_return
	47	+#define arch_atomic64_fetch_add arch_atomic64_fetch_add
	48	+#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
	49	+
40	50	#undef ATOMIC_OPS
41	51	#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
42	52
@@ -44,22 +54,19 @@ ATOMIC_OPS(and) · 1 unmodified line
44	54	ATOMIC_OPS(or)
45	55	ATOMIC_OPS(xor)
46	56
	57	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	58	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	59	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	60	+
	61	+#define arch_atomic64_fetch_and arch_atomic64_fetch_and
	62	+#define arch_atomic64_fetch_or arch_atomic64_fetch_or
	63	+#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
	64	+
47	65	#undef ATOMIC_OPS
48	66	#undef ATOMIC_FETCH_OP
49	67	#undef ATOMIC_OP_RETURN
50	68	#undef ATOMIC_OP
51	69
52		−#define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
53		−
54		−static inline int arch_atomic_xchg(atomic_t *v, int new)
55		−{
56		− return arch_xchg(&v->counter, new);
57		−}
58		−
59		−#define arch_atomic64_cmpxchg(v, o, n) \
60		− ((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
61		−#define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), new))
62		−
63	70	s64 arch_atomic64_dec_if_positive(atomic64_t *v);
64	71	#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
65	72

arch/x86/include/asm/atomic.h+0−87

@@ -14,12 +14,6 @@ · 13 unmodified lines
14	14	* resource counting etc..
15	15	*/
16	16
17		−/**
18		− * arch_atomic_read - read atomic variable
19		− * @v: pointer of type atomic_t
20		− *
21		− * Atomically reads the value of @v.
22		− */
23	17	static __always_inline int arch_atomic_read(const atomic_t *v)
24	18	{
25	19	/*
@@ -29,25 +23,11 @@ static __always_inline int arch_atomic_read(const atomic_t *v) · 3 unmodified lines
29	23	return __READ_ONCE((v)->counter);
30	24	}
31	25
32		−/**
33		− * arch_atomic_set - set atomic variable
34		− * @v: pointer of type atomic_t
35		− * @i: required value
36		− *
37		− * Atomically sets the value of @v to @i.
38		− */
39	26	static __always_inline void arch_atomic_set(atomic_t *v, int i)
40	27	{
41	28	__WRITE_ONCE(v->counter, i);
42	29	}
43	30
44		−/**
45		− * arch_atomic_add - add integer to atomic variable
46		− * @i: integer value to add
47		− * @v: pointer of type atomic_t
48		− *
49		− * Atomically adds @i to @v.
50		− */
51	31	static __always_inline void arch_atomic_add(int i, atomic_t *v)
52	32	{
53	33	asm volatile(LOCK_PREFIX "addl %1,%0"
@@ -55,13 +35,6 @@ static __always_inline void arch_atomic_add(int i, atomic_t *v) · 1 unmodified line
55	35	: "ir" (i) : "memory");
56	36	}
57	37
58		−/**
59		− * arch_atomic_sub - subtract integer from atomic variable
60		− * @i: integer value to subtract
61		− * @v: pointer of type atomic_t
62		− *
63		− * Atomically subtracts @i from @v.
64		− */
65	38	static __always_inline void arch_atomic_sub(int i, atomic_t *v)
66	39	{
67	40	asm volatile(LOCK_PREFIX "subl %1,%0"
@@ -69,27 +42,12 @@ static __always_inline void arch_atomic_sub(int i, atomic_t *v) · 1 unmodified line
69	42	: "ir" (i) : "memory");
70	43	}
71	44
72		−/**
73		− * arch_atomic_sub_and_test - subtract value from variable and test result
74		− * @i: integer value to subtract
75		− * @v: pointer of type atomic_t
76		− *
77		− * Atomically subtracts @i from @v and returns
78		− * true if the result is zero, or false for all
79		− * other cases.
80		− */
81	45	static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
82	46	{
83	47	return GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, e, "er", i);
84	48	}
85	49	#define arch_atomic_sub_and_test arch_atomic_sub_and_test
86	50
87		−/**
88		− * arch_atomic_inc - increment atomic variable
89		− * @v: pointer of type atomic_t
90		− *
91		− * Atomically increments @v by 1.
92		− */
93	51	static __always_inline void arch_atomic_inc(atomic_t *v)
94	52	{
95	53	asm volatile(LOCK_PREFIX "incl %0"
@@ -97,12 +55,6 @@ static __always_inline void arch_atomic_inc(atomic_t *v) · 1 unmodified line
97	55	}
98	56	#define arch_atomic_inc arch_atomic_inc
99	57
100		−/**
101		− * arch_atomic_dec - decrement atomic variable
102		− * @v: pointer of type atomic_t
103		− *
104		− * Atomically decrements @v by 1.
105		− */
106	58	static __always_inline void arch_atomic_dec(atomic_t *v)
107	59	{
108	60	asm volatile(LOCK_PREFIX "decl %0"
@@ -110,69 +62,30 @@ static __always_inline void arch_atomic_dec(atomic_t *v) · 1 unmodified line
110	62	}
111	63	#define arch_atomic_dec arch_atomic_dec
112	64
113		−/**
114		− * arch_atomic_dec_and_test - decrement and test
115		− * @v: pointer of type atomic_t
116		− *
117		− * Atomically decrements @v by 1 and
118		− * returns true if the result is 0, or false for all other
119		− * cases.
120		− */
121	65	static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
122	66	{
123	67	return GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, e);
124	68	}
125	69	#define arch_atomic_dec_and_test arch_atomic_dec_and_test
126	70
127		−/**
128		− * arch_atomic_inc_and_test - increment and test
129		− * @v: pointer of type atomic_t
130		− *
131		− * Atomically increments @v by 1
132		− * and returns true if the result is zero, or false for all
133		− * other cases.
134		− */
135	71	static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
136	72	{
137	73	return GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, e);
138	74	}
139	75	#define arch_atomic_inc_and_test arch_atomic_inc_and_test
140	76
141		−/**
142		− * arch_atomic_add_negative - add and test if negative
143		− * @i: integer value to add
144		− * @v: pointer of type atomic_t
145		− *
146		− * Atomically adds @i to @v and returns true
147		− * if the result is negative, or false when
148		− * result is greater than or equal to zero.
149		− */
150	77	static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
151	78	{
152	79	return GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, s, "er", i);
153	80	}
154	81	#define arch_atomic_add_negative arch_atomic_add_negative
155	82
156		−/**
157		− * arch_atomic_add_return - add integer and return
158		− * @i: integer value to add
159		− * @v: pointer of type atomic_t
160		− *
161		− * Atomically adds @i to @v and returns @i + @v
162		− */
163	83	static __always_inline int arch_atomic_add_return(int i, atomic_t *v)
164	84	{
165	85	return i + xadd(&v->counter, i);
166	86	}
167	87	#define arch_atomic_add_return arch_atomic_add_return
168	88
169		−/**
170		− * arch_atomic_sub_return - subtract integer and return
171		− * @v: pointer of type atomic_t
172		− * @i: integer value to subtract
173		− *
174		− * Atomically subtracts @i from @v and returns @v - @i
175		− */
176	89	static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)
177	90	{
178	91	return arch_atomic_add_return(-i, v);

arch/x86/include/asm/atomic64_32.h+0−76

@@ -61,30 +61,12 @@ ATOMIC64_DECL(add_unless); · 60 unmodified lines
61	61	#undef __ATOMIC64_DECL
62	62	#undef ATOMIC64_EXPORT
63	63
64		−/**
65		− * arch_atomic64_cmpxchg - cmpxchg atomic64 variable
66		− * @v: pointer to type atomic64_t
67		− * @o: expected value
68		− * @n: new value
69		− *
70		− * Atomically sets @v to @n if it was equal to @o and returns
71		− * the old value.
72		− */
73		−
74	64	static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
75	65	{
76	66	return arch_cmpxchg64(&v->counter, o, n);
77	67	}
78	68	#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
79	69
80		−/**
81		− * arch_atomic64_xchg - xchg atomic64 variable
82		− * @v: pointer to type atomic64_t
83		− * @n: value to assign
84		− *
85		− * Atomically xchgs the value of @v to @n and returns
86		− * the old value.
87		− */
88	70	static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n)
89	71	{
90	72	s64 o;
@@ -97,13 +79,6 @@ static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) · 6 unmodified lines
97	79	}
98	80	#define arch_atomic64_xchg arch_atomic64_xchg
99	81
100		−/**
101		− * arch_atomic64_set - set atomic64 variable
102		− * @v: pointer to type atomic64_t
103		− * @i: value to assign
104		− *
105		− * Atomically sets the value of @v to @n.
106		− */
107	82	static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
108	83	{
109	84	unsigned high = (unsigned)(i >> 32);
@@ -113,12 +88,6 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) · 3 unmodified lines
113	88	: "eax", "edx", "memory");
114	89	}
115	90
116		−/**
117		− * arch_atomic64_read - read atomic64 variable
118		− * @v: pointer to type atomic64_t
119		− *
120		− * Atomically reads the value of @v and returns it.
121		− */
122	91	static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
123	92	{
124	93	s64 r;
@@ -126,13 +95,6 @@ static __always_inline s64 arch_atomic64_read(const atomic64_t *v) · 1 unmodified line
126	95	return r;
127	96	}
128	97
129		−/**
130		− * arch_atomic64_add_return - add and return
131		− * @i: integer value to add
132		− * @v: pointer to type atomic64_t
133		− *
134		− * Atomically adds @i to @v and returns @i + *@v
135		− */
136	98	static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
137	99	{
138	100	alternative_atomic64(add_return,
@@ -142,9 +104,6 @@ static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) · 3 unmodified lines
142	104	}
143	105	#define arch_atomic64_add_return arch_atomic64_add_return
144	106
145		−/*
146		− * Other variants with different arithmetic operators:
147		− */
148	107	static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
149	108	{
150	109	alternative_atomic64(sub_return,
@@ -172,13 +131,6 @@ static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v) · 21 unmodified lines
172	131	}
173	132	#define arch_atomic64_dec_return arch_atomic64_dec_return
174	133
175		−/**
176		− * arch_atomic64_add - add integer to atomic64 variable
177		− * @i: integer value to add
178		− * @v: pointer to type atomic64_t
179		− *
180		− * Atomically adds @i to @v.
181		− */
182	134	static __always_inline s64 arch_atomic64_add(s64 i, atomic64_t *v)
183	135	{
184	136	__alternative_atomic64(add, add_return,
@@ -187,13 +139,6 @@ static __always_inline s64 arch_atomic64_add(s64 i, atomic64_t *v) · 2 unmodified lines
187	139	return i;
188	140	}
189	141
190		−/**
191		− * arch_atomic64_sub - subtract the atomic64 variable
192		− * @i: integer value to subtract
193		− * @v: pointer to type atomic64_t
194		− *
195		− * Atomically subtracts @i from @v.
196		− */
197	142	static __always_inline s64 arch_atomic64_sub(s64 i, atomic64_t *v)
198	143	{
199	144	__alternative_atomic64(sub, sub_return,
@@ -202,12 +147,6 @@ static __always_inline s64 arch_atomic64_sub(s64 i, atomic64_t *v) · 2 unmodified lines
202	147	return i;
203	148	}
204	149
205		−/**
206		− * arch_atomic64_inc - increment atomic64 variable
207		− * @v: pointer to type atomic64_t
208		− *
209		− * Atomically increments @v by 1.
210		− */
211	150	static __always_inline void arch_atomic64_inc(atomic64_t *v)
212	151	{
213	152	__alternative_atomic64(inc, inc_return, /* no output */,
@@ -215,12 +154,6 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v) · 1 unmodified line
215	154	}
216	155	#define arch_atomic64_inc arch_atomic64_inc
217	156
218		−/**
219		− * arch_atomic64_dec - decrement atomic64 variable
220		− * @v: pointer to type atomic64_t
221		− *
222		− * Atomically decrements @v by 1.
223		− */
224	157	static __always_inline void arch_atomic64_dec(atomic64_t *v)
225	158	{
226	159	__alternative_atomic64(dec, dec_return, /* no output */,
@@ -228,15 +161,6 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v) · 1 unmodified line
228	161	}
229	162	#define arch_atomic64_dec arch_atomic64_dec
230	163
231		−/**
232		− * arch_atomic64_add_unless - add unless the number is a given value
233		− * @v: pointer of type atomic64_t
234		− * @a: the amount to add to v...
235		− * @u: ...unless v is equal to u.
236		− *
237		− * Atomically adds @a to @v, so long as it was not @u.
238		− * Returns non-zero if the add was done, zero otherwise.
239		− */
240	164	static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
241	165	{
242	166	unsigned low = (unsigned)u;

arch/x86/include/asm/atomic64_64.h+0−81

@@ -10,37 +10,16 @@ · 9 unmodified lines
10	10
11	11	#define ATOMIC64_INIT(i) { (i) }
12	12
13		−/**
14		− * arch_atomic64_read - read atomic64 variable
15		− * @v: pointer of type atomic64_t
16		− *
17		− * Atomically reads the value of @v.
18		− * Doesn't imply a read memory barrier.
19		− */
20	13	static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
21	14	{
22	15	return __READ_ONCE((v)->counter);
23	16	}
24	17
25		−/**
26		− * arch_atomic64_set - set atomic64 variable
27		− * @v: pointer to type atomic64_t
28		− * @i: required value
29		− *
30		− * Atomically sets the value of @v to @i.
31		− */
32	18	static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
33	19	{
34	20	__WRITE_ONCE(v->counter, i);
35	21	}
36	22
37		−/**
38		− * arch_atomic64_add - add integer to atomic64 variable
39		− * @i: integer value to add
40		− * @v: pointer to type atomic64_t
41		− *
42		− * Atomically adds @i to @v.
43		− */
44	23	static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
45	24	{
46	25	asm volatile(LOCK_PREFIX "addq %1,%0"
@@ -48,13 +27,6 @@ static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v) · 1 unmodified line
48	27	: "er" (i), "m" (v->counter) : "memory");
49	28	}
50	29
51		−/**
52		− * arch_atomic64_sub - subtract the atomic64 variable
53		− * @i: integer value to subtract
54		− * @v: pointer to type atomic64_t
55		− *
56		− * Atomically subtracts @i from @v.
57		− */
58	30	static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
59	31	{
60	32	asm volatile(LOCK_PREFIX "subq %1,%0"
@@ -62,27 +34,12 @@ static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v) · 1 unmodified line
62	34	: "er" (i), "m" (v->counter) : "memory");
63	35	}
64	36
65		−/**
66		− * arch_atomic64_sub_and_test - subtract value from variable and test result
67		− * @i: integer value to subtract
68		− * @v: pointer to type atomic64_t
69		− *
70		− * Atomically subtracts @i from @v and returns
71		− * true if the result is zero, or false for all
72		− * other cases.
73		− */
74	37	static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
75	38	{
76	39	return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);
77	40	}
78	41	#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
79	42
80		−/**
81		− * arch_atomic64_inc - increment atomic64 variable
82		− * @v: pointer to type atomic64_t
83		− *
84		− * Atomically increments @v by 1.
85		− */
86	43	static __always_inline void arch_atomic64_inc(atomic64_t *v)
87	44	{
88	45	asm volatile(LOCK_PREFIX "incq %0"
@@ -91,12 +48,6 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v) · 2 unmodified lines
91	48	}
92	49	#define arch_atomic64_inc arch_atomic64_inc
93	50
94		−/**
95		− * arch_atomic64_dec - decrement atomic64 variable
96		− * @v: pointer to type atomic64_t
97		− *
98		− * Atomically decrements @v by 1.
99		− */
100	51	static __always_inline void arch_atomic64_dec(atomic64_t *v)
101	52	{
102	53	asm volatile(LOCK_PREFIX "decq %0"
@@ -105,56 +56,24 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v) · 2 unmodified lines
105	56	}
106	57	#define arch_atomic64_dec arch_atomic64_dec
107	58
108		−/**
109		− * arch_atomic64_dec_and_test - decrement and test
110		− * @v: pointer to type atomic64_t
111		− *
112		− * Atomically decrements @v by 1 and
113		− * returns true if the result is 0, or false for all other
114		− * cases.
115		− */
116	59	static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v)
117	60	{
118	61	return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);
119	62	}
120	63	#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
121	64
122		−/**
123		− * arch_atomic64_inc_and_test - increment and test
124		− * @v: pointer to type atomic64_t
125		− *
126		− * Atomically increments @v by 1
127		− * and returns true if the result is zero, or false for all
128		− * other cases.
129		− */
130	65	static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v)
131	66	{
132	67	return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);
133	68	}
134	69	#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
135	70
136		−/**
137		− * arch_atomic64_add_negative - add and test if negative
138		− * @i: integer value to add
139		− * @v: pointer to type atomic64_t
140		− *
141		− * Atomically adds @i to @v and returns true
142		− * if the result is negative, or false when
143		− * result is greater than or equal to zero.
144		− */
145	71	static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)
146	72	{
147	73	return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);
148	74	}
149	75	#define arch_atomic64_add_negative arch_atomic64_add_negative
150	76
151		−/**
152		− * arch_atomic64_add_return - add and return
153		− * @i: integer value to add
154		− * @v: pointer to type atomic64_t
155		− *
156		− * Atomically adds @i to @v and returns @i + @v
157		− */
158	77	static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
159	78	{
160	79	return i + xadd(&v->counter, i);

arch/x86/include/asm/cmpxchg.h+0−25

@@ -239,29 +239,4 @@ extern void __add_wrong_size(void) · 238 unmodified lines
239	239	#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
240	240	#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
241	241
242		−#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
243		−({ \
244		− bool __ret; \
245		− __typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \
246		− __typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \
247		− BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
248		− BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
249		− VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \
250		− VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \
251		− asm volatile(pfx "cmpxchg%c5b %1" \
252		− CC_SET(e) \
253		− : CC_OUT(e) (__ret), \
254		− "+m" ((p1)), "+m" ((p2)), \
255		− "+a" (__old1), "+d" (__old2) \
256		− : "i" (2 * sizeof(long)), \
257		− "b" (__new1), "c" (__new2)); \
258		− __ret; \
259		−})
260		−
261		−#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \
262		− __cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)
263		−
264		−#define arch_cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
265		− __cmpxchg_double(, p1, p2, o1, o2, n1, n2)
266		−
267	242	#endif /* ASM_X86_CMPXCHG_H */

arch/x86/include/asm/cmpxchg_32.h+1−1

@@ -103,6 +103,6 @@ static inline bool __try_cmpxchg64(volatile u64 ptr, u64 pold, u64 new) · 102 unmodified lines
103	103
104	104	#endif
105	105
106		−#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)
	106	+#define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8)
107	107
108	108	#endif /* _ASM_X86_CMPXCHG_32_H */

arch/x86/include/asm/cmpxchg_64.h+66−1

@@ -20,6 +20,71 @@ · 19 unmodified lines
20	20	arch_try_cmpxchg((ptr), (po), (n)); \
21	21	})
22	22
23		−#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16)
	23	+union __u128_halves {
	24	+ u128 full;
	25	+ struct {
	26	+ u64 low, high;
	27	+ };
	28	+};
	29	+
	30	+#define __arch_cmpxchg128(_ptr, _old, _new, _lock) \
	31	+({ \
	32	+ union __u128_halves o = { .full = (_old), }, \
	33	+ n = { .full = (_new), }; \
	34	+ \
	35	+ asm volatile(_lock "cmpxchg16b %[ptr]" \
	36	+ : [ptr] "+m" (*(_ptr)), \
	37	+ "+a" (o.low), "+d" (o.high) \
	38	+ : "b" (n.low), "c" (n.high) \
	39	+ : "memory"); \
	40	+ \
	41	+ o.full; \
	42	+})
	43	+
	44	+static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
	45	+{
	46	+ return __arch_cmpxchg128(ptr, old, new, LOCK_PREFIX);
	47	+}
	48	+#define arch_cmpxchg128 arch_cmpxchg128
	49	+
	50	+static __always_inline u128 arch_cmpxchg128_local(volatile u128 *ptr, u128 old, u128 new)
	51	+{
	52	+ return __arch_cmpxchg128(ptr, old, new,);
	53	+}
	54	+#define arch_cmpxchg128_local arch_cmpxchg128_local
	55	+
	56	+#define __arch_try_cmpxchg128(_ptr, _oldp, _new, _lock) \
	57	+({ \
	58	+ union __u128_halves o = { .full = *(_oldp), }, \
	59	+ n = { .full = (_new), }; \
	60	+ bool ret; \
	61	+ \
	62	+ asm volatile(_lock "cmpxchg16b %[ptr]" \
	63	+ CC_SET(e) \
	64	+ : CC_OUT(e) (ret), \
	65	+ [ptr] "+m" (*ptr), \
	66	+ "+a" (o.low), "+d" (o.high) \
	67	+ : "b" (n.low), "c" (n.high) \
	68	+ : "memory"); \
	69	+ \
	70	+ if (unlikely(!ret)) \
	71	+ *(_oldp) = o.full; \
	72	+ \
	73	+ likely(ret); \
	74	+})
	75	+
	76	+static __always_inline bool arch_try_cmpxchg128(volatile u128 ptr, u128 oldp, u128 new)
	77	+{
	78	+ return __arch_try_cmpxchg128(ptr, oldp, new, LOCK_PREFIX);
	79	+}
	80	+#define arch_try_cmpxchg128 arch_try_cmpxchg128
	81	+
	82	+static __always_inline bool arch_try_cmpxchg128_local(volatile u128 ptr, u128 oldp, u128 new)
	83	+{
	84	+ return __arch_try_cmpxchg128(ptr, oldp, new,);
	85	+}
	86	+#define arch_try_cmpxchg128_local arch_try_cmpxchg128_local
	87	+
	88	+#define system_has_cmpxchg128() boot_cpu_has(X86_FEATURE_CX16)
24	89
25	90	#endif /* _ASM_X86_CMPXCHG_64_H */

arch/x86/include/asm/percpu.h+61−41

@@ -210,6 +210,67 @@ do { \ · 209 unmodified lines
210	210	(typeof(_var))(unsigned long) pco_old__; \
211	211	})
212	212
	213	+#if defined(CONFIG_X86_32) && !defined(CONFIG_UML)
	214	+#define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \
	215	+({ \
	216	+ union { \
	217	+ u64 var; \
	218	+ struct { \
	219	+ u32 low, high; \
	220	+ }; \
	221	+ } old__, new__; \
	222	+ \
	223	+ old__.var = _oval; \
	224	+ new__.var = _nval; \
	225	+ \
	226	+ asm qual (ALTERNATIVE("leal %P[var], %%esi; call this_cpu_cmpxchg8b_emu", \
	227	+ "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
	228	+ : [var] "+m" (_var), \
	229	+ "+a" (old__.low), \
	230	+ "+d" (old__.high) \
	231	+ : "b" (new__.low), \
	232	+ "c" (new__.high) \
	233	+ : "memory", "esi"); \
	234	+ \
	235	+ old__.var; \
	236	+})
	237	+
	238	+#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval)
	239	+#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
	240	+#endif
	241	+
	242	+#ifdef CONFIG_X86_64
	243	+#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval);
	244	+#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
	245	+
	246	+#define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval) \
	247	+({ \
	248	+ union { \
	249	+ u128 var; \
	250	+ struct { \
	251	+ u64 low, high; \
	252	+ }; \
	253	+ } old__, new__; \
	254	+ \
	255	+ old__.var = _oval; \
	256	+ new__.var = _nval; \
	257	+ \
	258	+ asm qual (ALTERNATIVE("leaq %P[var], %%rsi; call this_cpu_cmpxchg16b_emu", \
	259	+ "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
	260	+ : [var] "+m" (_var), \
	261	+ "+a" (old__.low), \
	262	+ "+d" (old__.high) \
	263	+ : "b" (new__.low), \
	264	+ "c" (new__.high) \
	265	+ : "memory", "rsi"); \
	266	+ \
	267	+ old__.var; \
	268	+})
	269	+
	270	+#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval)
	271	+#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
	272	+#endif
	273	+
213	274	/*
214	275	* this_cpu_read() makes gcc load the percpu variable every time it is
215	276	* accessed while this_cpu_read_stable() allows the value to be cached.
@@ -290,23 +351,6 @@ do { \ · 74 unmodified lines
290	351	#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
291	352	#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
292	353
293		−#ifdef CONFIG_X86_CMPXCHG64
294		−#define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2) \
295		−({ \
296		− bool __ret; \
297		− typeof(pcp1) __o1 = (o1), __n1 = (n1); \
298		− typeof(pcp2) __o2 = (o2), __n2 = (n2); \
299		− asm volatile("cmpxchg8b "__percpu_arg(1) \
300		− CC_SET(z) \
301		− : CC_OUT(z) (__ret), "+m" (pcp1), "+m" (pcp2), "+a" (__o1), "+d" (__o2) \
302		− : "b" (__n1), "c" (__n2)); \
303		− __ret; \
304		−})
305		−
306		−#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
307		−#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
308		−#endif /* CONFIG_X86_CMPXCHG64 */
309		−
310	354	/*
311	355	* Per cpu atomic 64 bit operations are only available under 64 bit.
312	356	* 32 bit must fall back to generic operations.
@@ -329,30 +373,6 @@ do { \ · 16 unmodified lines
329	373	#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
330	374	#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval)
331	375	#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
332		−
333		−/*
334		− * Pretty complex macro to generate cmpxchg16 instruction. The instruction
335		− * is not supported on early AMD64 processors so we must be able to emulate
336		− * it in software. The address used in the cmpxchg16 instruction must be
337		− * aligned to a 16 byte boundary.
338		− */
339		−#define percpu_cmpxchg16b_double(pcp1, pcp2, o1, o2, n1, n2) \
340		−({ \
341		− bool __ret; \
342		− typeof(pcp1) __o1 = (o1), __n1 = (n1); \
343		− typeof(pcp2) __o2 = (o2), __n2 = (n2); \
344		− alternative_io("leaq %P1,%%rsi\n\tcall this_cpu_cmpxchg16b_emu\n\t", \
345		− "cmpxchg16b " __percpu_arg(1) "\n\tsetz %0\n\t", \
346		− X86_FEATURE_CX16, \
347		− ASM_OUTPUT2("=a" (__ret), "+m" (pcp1), \
348		− "+m" (pcp2), "+d" (__o2)), \
349		− "b" (__n1), "c" (__n2), "a" (__o1) : "rsi"); \
350		− __ret; \
351		−})
352		−
353		−#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
354		−#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
355		−
356	376	#endif
357	377
358	378	static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,

arch/x86/kernel/alternative.c+2−2

@@ -1967,7 +1967,7 @@ struct bp_patching_desc *try_get_desc(void) · 1966 unmodified lines
1967	1967	{
1968	1968	struct bp_patching_desc *desc = &bp_desc;
1969	1969
1970		− if (!arch_atomic_inc_not_zero(&desc->refs))
	1970	+ if (!raw_atomic_inc_not_zero(&desc->refs))
1971	1971	return NULL;
1972	1972
1973	1973	return desc;
@@ -1978,7 +1978,7 @@ static __always_inline void put_desc(void) · 4 unmodified lines
1978	1978	struct bp_patching_desc *desc = &bp_desc;
1979	1979
1980	1980	smp_mb__before_atomic();
1981		− arch_atomic_dec(&desc->refs);
	1981	+ raw_atomic_dec(&desc->refs);
1982	1982	}
1983	1983
1984	1984	static __always_inline void text_poke_addr(struct text_poke_loc tp)

arch/x86/kernel/cpu/mce/core.c+8−8

@@ -1022,12 +1022,12 @@ static noinstr int mce_start(int *no_way_out) · 1021 unmodified lines
1022	1022	if (!timeout)
1023	1023	return ret;
1024	1024
1025		− arch_atomic_add(*no_way_out, &global_nwo);
	1025	+ raw_atomic_add(*no_way_out, &global_nwo);
1026	1026	/*
1027	1027	* Rely on the implied barrier below, such that global_nwo
1028	1028	* is updated before mce_callin.
1029	1029	*/
1030		− order = arch_atomic_inc_return(&mce_callin);
	1030	+ order = raw_atomic_inc_return(&mce_callin);
1031	1031	arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus);
1032	1032
1033	1033	/* Enable instrumentation around calls to external facilities */
@@ -1036,10 +1036,10 @@ static noinstr int mce_start(int *no_way_out) · 2 unmodified lines
1036	1036	/*
1037	1037	* Wait for everyone.
1038	1038	*/
1039		− while (arch_atomic_read(&mce_callin) != num_online_cpus()) {
	1039	+ while (raw_atomic_read(&mce_callin) != num_online_cpus()) {
1040	1040	if (mce_timed_out(&timeout,
1041	1041	"Timeout: Not all CPUs entered broadcast exception handler")) {
1042		− arch_atomic_set(&global_nwo, 0);
	1042	+ raw_atomic_set(&global_nwo, 0);
1043	1043	goto out;
1044	1044	}
1045	1045	ndelay(SPINUNIT);
@@ -1054,7 +1054,7 @@ static noinstr int mce_start(int *no_way_out) · 8 unmodified lines
1054	1054	/*
1055	1055	* Monarch: Starts executing now, the others wait.
1056	1056	*/
1057		− arch_atomic_set(&mce_executing, 1);
	1057	+ raw_atomic_set(&mce_executing, 1);
1058	1058	} else {
1059	1059	/*
1060	1060	* Subject: Now start the scanning loop one by one in
@@ -1062,10 +1062,10 @@ static noinstr int mce_start(int *no_way_out) · 1 unmodified line
1062	1062	* This way when there are any shared banks it will be
1063	1063	* only seen by one CPU before cleared, avoiding duplicates.
1064	1064	*/
1065		− while (arch_atomic_read(&mce_executing) < order) {
	1065	+ while (raw_atomic_read(&mce_executing) < order) {
1066	1066	if (mce_timed_out(&timeout,
1067	1067	"Timeout: Subject CPUs unable to finish machine check processing")) {
1068		− arch_atomic_set(&global_nwo, 0);
	1068	+ raw_atomic_set(&global_nwo, 0);
1069	1069	goto out;
1070	1070	}
1071	1071	ndelay(SPINUNIT);
@@ -1075,7 +1075,7 @@ static noinstr int mce_start(int *no_way_out) · 3 unmodified lines
1075	1075	/*
1076	1076	* Cache the global no_way_out state.
1077	1077	*/
1078		− *no_way_out = arch_atomic_read(&global_nwo);
	1078	+ *no_way_out = raw_atomic_read(&global_nwo);
1079	1079
1080	1080	ret = order;
1081	1081

arch/x86/kernel/nmi.c+1−1

@@ -496,7 +496,7 @@ DEFINE_IDTENTRY_RAW(exc_nmi) · 495 unmodified lines
496	496	*/
497	497	sev_es_nmi_complete();
498	498	if (IS_ENABLED(CONFIG_NMI_CHECK_CPU))
499		− arch_atomic_long_inc(&nsp->idt_calls);
	499	+ raw_atomic_long_inc(&nsp->idt_calls);
500	500
501	501	if (IS_ENABLED(CONFIG_SMP) && arch_cpu_is_offline(smp_processor_id()))
502	502	return;

arch/x86/kernel/pvclock.c+2−2

@@ -101,11 +101,11 @@ u64 __pvclock_clocksource_read(struct pvclock_vcpu_time_info *src, bool dowd) · 100 unmodified lines
101	101	* updating at the same time, and one of them could be slightly behind,
102	102	* making the assumption that last_value always go forward fail to hold.
103	103	*/
104		− last = arch_atomic64_read(&last_value);
	104	+ last = raw_atomic64_read(&last_value);
105	105	do {
106	106	if (ret <= last)
107	107	return last;
108		− } while (!arch_atomic64_try_cmpxchg(&last_value, &last, ret));
	108	+ } while (!raw_atomic64_try_cmpxchg(&last_value, &last, ret));
109	109
110	110	return ret;
111	111	}

arch/x86/kvm/x86.c+1−1

@@ -13161,7 +13161,7 @@ EXPORT_SYMBOL_GPL(kvm_arch_end_assignment); · 13160 unmodified lines
13161	13161
13162	13162	bool noinstr kvm_arch_has_assigned_device(struct kvm *kvm)
13163	13163	{
13164		− return arch_atomic_read(&kvm->arch.assigned_device_count);
	13164	+ return raw_atomic_read(&kvm->arch.assigned_device_count);
13165	13165	}
13166	13166	EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);
13167	13167

arch/x86/lib/Makefile+2−1

@@ -61,8 +61,9 @@ ifeq ($(CONFIG_X86_32),y) · 60 unmodified lines
61	61	lib-y += strstr_32.o
62	62	lib-y += string_32.o
63	63	lib-y += memmove_32.o
	64	+ lib-y += cmpxchg8b_emu.o
64	65	ifneq ($(CONFIG_X86_CMPXCHG64),y)
65		− lib-y += cmpxchg8b_emu.o atomic64_386_32.o
	66	+ lib-y += atomic64_386_32.o
66	67	endif
67	68	else
68	69	obj-y += iomap_copy_64.o

arch/x86/lib/cmpxchg16b_emu.S+25−18

@@ -1,47 +1,54 @@
1	1	/* SPDX-License-Identifier: GPL-2.0-only */
2	2	#include <linux/linkage.h>
3	3	#include <asm/percpu.h>
	4	+#include <asm/processor-flags.h>
4	5
5	6	.text
6	7
7	8	/*
	9	+ * Emulate 'cmpxchg16b %gs:(%rsi)'
	10	+ *
8	11	* Inputs:
9	12	* %rsi : memory location to compare
10	13	* %rax : low 64 bits of old value
11	14	* %rdx : high 64 bits of old value
12	15	* %rbx : low 64 bits of new value
13	16	* %rcx : high 64 bits of new value
14		− * %al : Operation successful
	17	+ *
	18	+ * Notably this is not LOCK prefixed and is not safe against NMIs
15	19	*/
16	20	SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
17	21
18		−#
19		−# Emulate 'cmpxchg16b %gs:(%rsi)' except we return the result in %al not
20		−# via the ZF. Caller will access %al to get result.
21		−#
22		−# Note that this is only useful for a cpuops operation. Meaning that we
23		−# do not have a fully atomic operation but just an operation that is
24		−# atomic on a single cpu (as provided by the this_cpu_xx class of
25		−# macros).
26		−#
27	22	pushfq
28	23	cli
29	24
30		− cmpq PER_CPU_VAR((%rsi)), %rax
31		− jne .Lnot_same
32		− cmpq PER_CPU_VAR(8(%rsi)), %rdx
33		− jne .Lnot_same
	25	+ /* if (ptr == old) /
	26	+ cmpq PER_CPU_VAR(0(%rsi)), %rax
	27	+ jne .Lnot_same
	28	+ cmpq PER_CPU_VAR(8(%rsi)), %rdx
	29	+ jne .Lnot_same
34	30
35		− movq %rbx, PER_CPU_VAR((%rsi))
36		− movq %rcx, PER_CPU_VAR(8(%rsi))
	31	+ /* ptr = new /
	32	+ movq %rbx, PER_CPU_VAR(0(%rsi))
	33	+ movq %rcx, PER_CPU_VAR(8(%rsi))
	34	+
	35	+ /* set ZF in EFLAGS to indicate success */
	36	+ orl $X86_EFLAGS_ZF, (%rsp)
37	37
38	38	popfq
39		− mov $1, %al
40	39	RET
41	40
42	41	.Lnot_same:
	42	+ /* ptr != old /
	43	+
	44	+ /* old = ptr /
	45	+ movq PER_CPU_VAR(0(%rsi)), %rax
	46	+ movq PER_CPU_VAR(8(%rsi)), %rdx
	47	+
	48	+ /* clear ZF in EFLAGS to indicate failure */
	49	+ andl $(~X86_EFLAGS_ZF), (%rsp)
	50	+
43	51	popfq
44		− xor %al,%al
45	52	RET
46	53
47	54	SYM_FUNC_END(this_cpu_cmpxchg16b_emu)

arch/x86/lib/cmpxchg8b_emu.S+53−14

@@ -2,10 +2,16 @@ · 1 unmodified line
2	2
3	3	#include <linux/linkage.h>
4	4	#include <asm/export.h>
	5	+#include <asm/percpu.h>
	6	+#include <asm/processor-flags.h>
5	7
6	8	.text
7	9
	10	+#ifndef CONFIG_X86_CMPXCHG64
	11	+
8	12	/*
	13	+ * Emulate 'cmpxchg8b (%esi)' on UP
	14	+ *
9	15	* Inputs:
10	16	* %esi : memory location to compare
11	17	* %eax : low 32 bits of old value
@@ -15,32 +21,65 @@ · 3 unmodified lines
15	21	*/
16	22	SYM_FUNC_START(cmpxchg8b_emu)
17	23
18		−#
19		−# Emulate 'cmpxchg8b (%esi)' on UP except we don't
20		−# set the whole ZF thing (caller will just compare
21		−# eax:edx with the expected value)
22		−#
23	24	pushfl
24	25	cli
25	26
26		− cmpl (%esi), %eax
27		− jne .Lnot_same
28		− cmpl 4(%esi), %edx
29		− jne .Lhalf_same
	27	+ cmpl 0(%esi), %eax
	28	+ jne .Lnot_same
	29	+ cmpl 4(%esi), %edx
	30	+ jne .Lnot_same
	31	+
	32	+ movl %ebx, 0(%esi)
	33	+ movl %ecx, 4(%esi)
30	34
31		− movl %ebx, (%esi)
32		− movl %ecx, 4(%esi)
	35	+ orl $X86_EFLAGS_ZF, (%esp)
33	36
34	37	popfl
35	38	RET
36	39
37	40	.Lnot_same:
38		− movl (%esi), %eax
39		−.Lhalf_same:
40		− movl 4(%esi), %edx
	41	+ movl 0(%esi), %eax
	42	+ movl 4(%esi), %edx
	43	+
	44	+ andl $(~X86_EFLAGS_ZF), (%esp)
41	45
42	46	popfl
43	47	RET
44	48
45	49	SYM_FUNC_END(cmpxchg8b_emu)
46	50	EXPORT_SYMBOL(cmpxchg8b_emu)
	51	+
	52	+#endif
	53	+
	54	+#ifndef CONFIG_UML
	55	+
	56	+SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
	57	+
	58	+ pushfl
	59	+ cli
	60	+
	61	+ cmpl PER_CPU_VAR(0(%esi)), %eax
	62	+ jne .Lnot_same2
	63	+ cmpl PER_CPU_VAR(4(%esi)), %edx
	64	+ jne .Lnot_same2
	65	+
	66	+ movl %ebx, PER_CPU_VAR(0(%esi))
	67	+ movl %ecx, PER_CPU_VAR(4(%esi))
	68	+
	69	+ orl $X86_EFLAGS_ZF, (%esp)
	70	+
	71	+ popfl
	72	+ RET
	73	+
	74	+.Lnot_same2:
	75	+ movl PER_CPU_VAR(0(%esi)), %eax
	76	+ movl PER_CPU_VAR(4(%esi)), %edx
	77	+
	78	+ andl $(~X86_EFLAGS_ZF), (%esp)
	79	+
	80	+ popfl
	81	+ RET
	82	+
	83	+SYM_FUNC_END(this_cpu_cmpxchg8b_emu)
	84	+
	85	+#endif

arch/xtensa/include/asm/atomic.h+9−3

@@ -245,6 +245,11 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t * v) \ · 244 unmodified lines
245	245	ATOMIC_OPS(add)
246	246	ATOMIC_OPS(sub)
247	247
	248	+#define arch_atomic_add_return arch_atomic_add_return
	249	+#define arch_atomic_sub_return arch_atomic_sub_return
	250	+#define arch_atomic_fetch_add arch_atomic_fetch_add
	251	+#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	252	+
248	253	#undef ATOMIC_OPS
249	254	#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
250	255
@@ -252,12 +257,13 @@ ATOMIC_OPS(and) · 1 unmodified line
252	257	ATOMIC_OPS(or)
253	258	ATOMIC_OPS(xor)
254	259
	260	+#define arch_atomic_fetch_and arch_atomic_fetch_and
	261	+#define arch_atomic_fetch_or arch_atomic_fetch_or
	262	+#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	263	+
255	264	#undef ATOMIC_OPS
256	265	#undef ATOMIC_FETCH_OP
257	266	#undef ATOMIC_OP_RETURN
258	267	#undef ATOMIC_OP
259	268
260		−#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n)))
261		−#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
262		−
263	269	#endif /* _XTENSA_ATOMIC_H */

drivers/iommu/amd/amd_iommu_types.h+7−2

@@ -986,8 +986,13 @@ union irte_ga_hi { · 985 unmodified lines
986	986	};
987	987
988	988	struct irte_ga {
989		− union irte_ga_lo lo;
990		− union irte_ga_hi hi;
	989	+ union {
	990	+ struct {
	991	+ union irte_ga_lo lo;
	992	+ union irte_ga_hi hi;
	993	+ };
	994	+ u128 irte;
	995	+ };
991	996	};
992	997
993	998	struct irq_2_irte {

drivers/iommu/amd/iommu.c+4−6

@@ -3023,10 +3023,10 @@ out: · 3022 unmodified lines
3023	3023	static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
3024	3024	struct irte_ga irte, struct amd_ir_data data)
3025	3025	{
3026		− bool ret;
3027	3026	struct irq_remap_table *table;
3028		− unsigned long flags;
3029	3027	struct irte_ga *entry;
	3028	+ unsigned long flags;
	3029	+ u128 old;
3030	3030
3031	3031	table = get_irq_table(iommu, devid);
3032	3032	if (!table)
@@ -3037,16 +3037,14 @@ static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, · 4 unmodified lines
3037	3037	entry = (struct irte_ga *)table->table;
3038	3038	entry = &entry[index];
3039	3039
3040		− ret = cmpxchg_double(&entry->lo.val, &entry->hi.val,
3041		− entry->lo.val, entry->hi.val,
3042		− irte->lo.val, irte->hi.val);
3043	3040	/*
3044	3041	* We use cmpxchg16 to atomically update the 128-bit IRTE,
3045	3042	* and it cannot be updated by the hardware or other processors
3046	3043	* behind us, so the return value of cmpxchg16 should be the
3047	3044	* same as the old value.
3048	3045	*/
3049		− WARN_ON(!ret);
	3046	+ old = entry->irte;
	3047	+ WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte));
3050	3048
3051	3049	if (data)
3052	3050	data->ref = entry;

drivers/iommu/intel/irq_remapping.c+2−6

@@ -175,18 +175,14 @@ static int modify_irte(struct irq_2_iommu *irq_iommu, · 174 unmodified lines
175	175	irte = &iommu->ir_table->base[index];
176	176
177	177	if ((irte->pst == 1) \|\| (irte_modified->pst == 1)) {
178		− bool ret;
179		−
180		− ret = cmpxchg_double(&irte->low, &irte->high,
181		− irte->low, irte->high,
182		− irte_modified->low, irte_modified->high);
183	178	/*
184	179	* We use cmpxchg16 to atomically update the 128-bit IRTE,
185	180	* and it cannot be updated by the hardware or other processors
186	181	* behind us, so the return value of cmpxchg16 should be the
187	182	* same as the old value.
188	183	*/
189		− WARN_ON(!ret);
	184	+ u128 old = irte->irte;
	185	+ WARN_ON(!try_cmpxchg128(&irte->irte, &old, irte_modified->irte));
190	186	} else {
191	187	WRITE_ONCE(irte->low, irte_modified->low);
192	188	WRITE_ONCE(irte->high, irte_modified->high);

drivers/md/bcache/btree.c+22−1

@@ -559,6 +559,27 @@ static void mca_data_alloc(struct btree b, struct bkey k, gfp_t gfp) · 558 unmodified lines
559	559	}
560	560	}
561	561
	562	+#define cmp_int(l, r) ((l > r) - (l < r))
	563	+
	564	+#ifdef CONFIG_PROVE_LOCKING
	565	+static int btree_lock_cmp_fn(const struct lockdep_map *_a,
	566	+ const struct lockdep_map *_b)
	567	+{
	568	+ const struct btree *a = container_of(_a, struct btree, lock.dep_map);
	569	+ const struct btree *b = container_of(_b, struct btree, lock.dep_map);
	570	+
	571	+ return -cmp_int(a->level, b->level) ?: bkey_cmp(&a->key, &b->key);
	572	+}
	573	+
	574	+static void btree_lock_print_fn(const struct lockdep_map *map)
	575	+{
	576	+ const struct btree *b = container_of(map, struct btree, lock.dep_map);
	577	+
	578	+ printk(KERN_CONT " l=%u %llu:%llu", b->level,
	579	+ KEY_INODE(&b->key), KEY_OFFSET(&b->key));
	580	+}
	581	+#endif
	582	+
562	583	static struct btree mca_bucket_alloc(struct cache_set c,
563	584	struct bkey *k, gfp_t gfp)
564	585	{
@@ -572,7 +593,7 @@ static struct btree mca_bucket_alloc(struct cache_set c, · 7 unmodified lines
572	593	return NULL;
573	594
574	595	init_rwsem(&b->lock);
575		− lockdep_set_novalidate_class(&b->lock);
	596	+ lock_set_cmp_fn(&b->lock, btree_lock_cmp_fn, btree_lock_print_fn);
576	597	mutex_init(&b->write_lock);
577	598	lockdep_set_novalidate_class(&b->write_lock);
578	599	INIT_LIST_HEAD(&b->list);

drivers/md/bcache/btree.h+2−2

@@ -247,8 +247,8 @@ static inline void bch_btree_op_init(struct btree_op *op, int write_lock_level) · 246 unmodified lines
247	247
248	248	static inline void rw_lock(bool w, struct btree *b, int level)
249	249	{
250		− w ? down_write_nested(&b->lock, level + 1)
251		− : down_read_nested(&b->lock, level + 1);
	250	+ w ? down_write(&b->lock)
	251	+ : down_read(&b->lock);
252	252	if (w)
253	253	b->seq++;
254	254	}

include/asm-generic/atomic.h+0−3

@@ -130,7 +130,4 @@ ATOMIC_OP(xor, ^) · 129 unmodified lines
130	130	#define arch_atomic_read(v) READ_ONCE((v)->counter)
131	131	#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
132	132
133		−#define arch_atomic_xchg(ptr, v) (arch_xchg(&(ptr)->counter, (u32)(v)))
134		−#define arch_atomic_cmpxchg(v, old, new) (arch_cmpxchg(&((v)->counter), (u32)(old), (u32)(new)))
135		−
136	133	#endif /* __ASM_GENERIC_ATOMIC_H */

include/asm-generic/bitops/atomic.h+6−6

@@ -15,21 +15,21 @@ static __always_inline void · 14 unmodified lines
15	15	arch_set_bit(unsigned int nr, volatile unsigned long *p)
16	16	{
17	17	p += BIT_WORD(nr);
18		− arch_atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p);
	18	+ raw_atomic_long_or(BIT_MASK(nr), (atomic_long_t *)p);
19	19	}
20	20
21	21	static __always_inline void
22	22	arch_clear_bit(unsigned int nr, volatile unsigned long *p)
23	23	{
24	24	p += BIT_WORD(nr);
25		− arch_atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p);
	25	+ raw_atomic_long_andnot(BIT_MASK(nr), (atomic_long_t *)p);
26	26	}
27	27
28	28	static __always_inline void
29	29	arch_change_bit(unsigned int nr, volatile unsigned long *p)
30	30	{
31	31	p += BIT_WORD(nr);
32		− arch_atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p);
	32	+ raw_atomic_long_xor(BIT_MASK(nr), (atomic_long_t *)p);
33	33	}
34	34
35	35	static __always_inline int
@@ -39,7 +39,7 @@ arch_test_and_set_bit(unsigned int nr, volatile unsigned long *p) · 3 unmodified lines
39	39	unsigned long mask = BIT_MASK(nr);
40	40
41	41	p += BIT_WORD(nr);
42		− old = arch_atomic_long_fetch_or(mask, (atomic_long_t *)p);
	42	+ old = raw_atomic_long_fetch_or(mask, (atomic_long_t *)p);
43	43	return !!(old & mask);
44	44	}
45	45
@@ -50,7 +50,7 @@ arch_test_and_clear_bit(unsigned int nr, volatile unsigned long *p) · 4 unmodified lines
50	50	unsigned long mask = BIT_MASK(nr);
51	51
52	52	p += BIT_WORD(nr);
53		− old = arch_atomic_long_fetch_andnot(mask, (atomic_long_t *)p);
	53	+ old = raw_atomic_long_fetch_andnot(mask, (atomic_long_t *)p);
54	54	return !!(old & mask);
55	55	}
56	56
@@ -61,7 +61,7 @@ arch_test_and_change_bit(unsigned int nr, volatile unsigned long *p) · 4 unmodified lines
61	61	unsigned long mask = BIT_MASK(nr);
62	62
63	63	p += BIT_WORD(nr);
64		− old = arch_atomic_long_fetch_xor(mask, (atomic_long_t *)p);
	64	+ old = raw_atomic_long_fetch_xor(mask, (atomic_long_t *)p);
65	65	return !!(old & mask);
66	66	}
67	67

include/asm-generic/bitops/lock.h+4−4

@@ -25,7 +25,7 @@ arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p) · 24 unmodified lines
25	25	if (READ_ONCE(*p) & mask)
26	26	return 1;
27	27
28		− old = arch_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p);
	28	+ old = raw_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p);
29	29	return !!(old & mask);
30	30	}
31	31
@@ -41,7 +41,7 @@ static __always_inline void · 9 unmodified lines
41	41	arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
42	42	{
43	43	p += BIT_WORD(nr);
44		− arch_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p);
	44	+ raw_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p);
45	45	}
46	46
47	47	/**
@@ -63,7 +63,7 @@ arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *p) · 15 unmodified lines
63	63	p += BIT_WORD(nr);
64	64	old = READ_ONCE(*p);
65	65	old &= ~BIT_MASK(nr);
66		− arch_atomic_long_set_release((atomic_long_t *)p, old);
	66	+ raw_atomic_long_set_release((atomic_long_t *)p, old);
67	67	}
68	68
69	69	/**
@@ -83,7 +83,7 @@ static inline bool arch_clear_bit_unlock_is_negative_byte(unsigned int nr, · 13 unmodified lines
83	83	unsigned long mask = BIT_MASK(nr);
84	84
85	85	p += BIT_WORD(nr);
86		− old = arch_atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p);
	86	+ old = raw_atomic_long_fetch_andnot_release(mask, (atomic_long_t *)p);
87	87	return !!(old & BIT(7));
88	88	}
89	89	#define arch_clear_bit_unlock_is_negative_byte arch_clear_bit_unlock_is_negative_byte

include/asm-generic/percpu.h+152−49

@@ -89,27 +89,35 @@ do { \ · 88 unmodified lines
89	89	__ret; \
90	90	})
91	91
92		−#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
	92	+#define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg) \
	93	+({ \
	94	+ typeof(pcp) __val, __old = *(ovalp); \
	95	+ __val = _cmpxchg(pcp, __old, nval); \
	96	+ if (__val != __old) \
	97	+ *(ovalp) = __val; \
	98	+ __val == __old; \
	99	+})
	100	+
	101	+#define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
93	102	({ \
94	103	typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
95		− typeof(pcp) __ret; \
96		− __ret = *__p; \
97		− if (__ret == (oval)) \
	104	+ typeof(pcp) __val = __p, ___old = (ovalp); \
	105	+ bool __ret; \
	106	+ if (__val == ___old) { \
98	107	*__p = nval; \
	108	+ __ret = true; \
	109	+ } else { \
	110	+ *(ovalp) = __val; \
	111	+ __ret = false; \
	112	+ } \
99	113	__ret; \
100	114	})
101	115
102		−#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	116	+#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
103	117	({ \
104		− typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \
105		− typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2)); \
106		− int __ret = 0; \
107		− if (__p1 == (oval1) && __p2 == (oval2)) { \
108		− *__p1 = nval1; \
109		− *__p2 = nval2; \
110		− __ret = 1; \
111		− } \
112		− (__ret); \
	118	+ typeof(pcp) __old = (oval); \
	119	+ raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \
	120	+ __old; \
113	121	})
114	122
115	123	#define __this_cpu_generic_read_nopreempt(pcp) \
@@ -170,23 +178,22 @@ do { \ · 54 unmodified lines
170	178	__ret; \
171	179	})
172	180
173		−#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
	181	+#define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
174	182	({ \
175		− typeof(pcp) __ret; \
	183	+ bool __ret; \
176	184	unsigned long __flags; \
177	185	raw_local_irq_save(__flags); \
178		− __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
	186	+ __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \
179	187	raw_local_irq_restore(__flags); \
180	188	__ret; \
181	189	})
182	190
183		−#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	191	+#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
184	192	({ \
185		− int __ret; \
	193	+ typeof(pcp) __ret; \
186	194	unsigned long __flags; \
187	195	raw_local_irq_save(__flags); \
188		− __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
189		− oval1, oval2, nval1, nval2); \
	196	+ __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
190	197	raw_local_irq_restore(__flags); \
191	198	__ret; \
192	199	})
@@ -282,6 +289,62 @@ do { \ · 89 unmodified lines
282	289	#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
283	290	#endif
284	291
	292	+#ifndef raw_cpu_try_cmpxchg_1
	293	+#ifdef raw_cpu_cmpxchg_1
	294	+#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
	295	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1)
	296	+#else
	297	+#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
	298	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	299	+#endif
	300	+#endif
	301	+#ifndef raw_cpu_try_cmpxchg_2
	302	+#ifdef raw_cpu_cmpxchg_2
	303	+#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
	304	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2)
	305	+#else
	306	+#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
	307	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	308	+#endif
	309	+#endif
	310	+#ifndef raw_cpu_try_cmpxchg_4
	311	+#ifdef raw_cpu_cmpxchg_4
	312	+#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
	313	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4)
	314	+#else
	315	+#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
	316	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	317	+#endif
	318	+#endif
	319	+#ifndef raw_cpu_try_cmpxchg_8
	320	+#ifdef raw_cpu_cmpxchg_8
	321	+#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
	322	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8)
	323	+#else
	324	+#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
	325	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	326	+#endif
	327	+#endif
	328	+
	329	+#ifndef raw_cpu_try_cmpxchg64
	330	+#ifdef raw_cpu_cmpxchg64
	331	+#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	332	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64)
	333	+#else
	334	+#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	335	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	336	+#endif
	337	+#endif
	338	+#ifndef raw_cpu_try_cmpxchg128
	339	+#ifdef raw_cpu_cmpxchg128
	340	+#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	341	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128)
	342	+#else
	343	+#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	344	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	345	+#endif
	346	+#endif
	347	+
285	348	#ifndef raw_cpu_cmpxchg_1
286	349	#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
287	350	raw_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -299,21 +362,13 @@ do { \ · 11 unmodified lines
299	362	raw_cpu_generic_cmpxchg(pcp, oval, nval)
300	363	#endif
301	364
302		−#ifndef raw_cpu_cmpxchg_double_1
303		−#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
304		− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
305		−#endif
306		−#ifndef raw_cpu_cmpxchg_double_2
307		−#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
308		− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
309		−#endif
310		−#ifndef raw_cpu_cmpxchg_double_4
311		−#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
312		− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	365	+#ifndef raw_cpu_cmpxchg64
	366	+#define raw_cpu_cmpxchg64(pcp, oval, nval) \
	367	+ raw_cpu_generic_cmpxchg(pcp, oval, nval)
313	368	#endif
314		−#ifndef raw_cpu_cmpxchg_double_8
315		−#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
316		− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	369	+#ifndef raw_cpu_cmpxchg128
	370	+#define raw_cpu_cmpxchg128(pcp, oval, nval) \
	371	+ raw_cpu_generic_cmpxchg(pcp, oval, nval)
317	372	#endif
318	373
319	374	#ifndef this_cpu_read_1
@@ -407,6 +462,62 @@ do { \ · 87 unmodified lines
407	462	#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
408	463	#endif
409	464
	465	+#ifndef this_cpu_try_cmpxchg_1
	466	+#ifdef this_cpu_cmpxchg_1
	467	+#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
	468	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1)
	469	+#else
	470	+#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
	471	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	472	+#endif
	473	+#endif
	474	+#ifndef this_cpu_try_cmpxchg_2
	475	+#ifdef this_cpu_cmpxchg_2
	476	+#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
	477	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2)
	478	+#else
	479	+#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
	480	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	481	+#endif
	482	+#endif
	483	+#ifndef this_cpu_try_cmpxchg_4
	484	+#ifdef this_cpu_cmpxchg_4
	485	+#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
	486	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4)
	487	+#else
	488	+#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
	489	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	490	+#endif
	491	+#endif
	492	+#ifndef this_cpu_try_cmpxchg_8
	493	+#ifdef this_cpu_cmpxchg_8
	494	+#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
	495	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8)
	496	+#else
	497	+#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
	498	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	499	+#endif
	500	+#endif
	501	+
	502	+#ifndef this_cpu_try_cmpxchg64
	503	+#ifdef this_cpu_cmpxchg64
	504	+#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	505	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64)
	506	+#else
	507	+#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	508	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	509	+#endif
	510	+#endif
	511	+#ifndef this_cpu_try_cmpxchg128
	512	+#ifdef this_cpu_cmpxchg128
	513	+#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	514	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128)
	515	+#else
	516	+#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	517	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	518	+#endif
	519	+#endif
	520	+
410	521	#ifndef this_cpu_cmpxchg_1
411	522	#define this_cpu_cmpxchg_1(pcp, oval, nval) \
412	523	this_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -424,21 +535,13 @@ do { \ · 11 unmodified lines
424	535	this_cpu_generic_cmpxchg(pcp, oval, nval)
425	536	#endif
426	537
427		−#ifndef this_cpu_cmpxchg_double_1
428		−#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
429		− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
430		−#endif
431		−#ifndef this_cpu_cmpxchg_double_2
432		−#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
433		− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
434		−#endif
435		−#ifndef this_cpu_cmpxchg_double_4
436		−#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
437		− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	538	+#ifndef this_cpu_cmpxchg64
	539	+#define this_cpu_cmpxchg64(pcp, oval, nval) \
	540	+ this_cpu_generic_cmpxchg(pcp, oval, nval)
438	541	#endif
439		−#ifndef this_cpu_cmpxchg_double_8
440		−#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
441		− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
	542	+#ifndef this_cpu_cmpxchg128
	543	+#define this_cpu_cmpxchg128(pcp, oval, nval) \
	544	+ this_cpu_generic_cmpxchg(pcp, oval, nval)
442	545	#endif
443	546
444	547	#endif /* _ASM_GENERIC_PERCPU_H_ */

include/crypto/b128ops.h+3−11

@@ -49,10 +49,6 @@ · 48 unmodified lines
49	49
50	50	#include <linux/types.h>
51	51
52		−typedef struct {
53		− u64 a, b;
54		−} u128;
55		−
56	52	typedef struct {
57	53	__be64 a, b;
58	54	} be128;
@@ -61,20 +57,16 @@ typedef struct { · 2 unmodified lines
61	57	__le64 b, a;
62	58	} le128;
63	59
64		−static inline void u128_xor(u128 r, const u128 p, const u128 *q)
	60	+static inline void be128_xor(be128 r, const be128 p, const be128 *q)
65	61	{
66	62	r->a = p->a ^ q->a;
67	63	r->b = p->b ^ q->b;
68	64	}
69	65
70		−static inline void be128_xor(be128 r, const be128 p, const be128 *q)
71		−{
72		− u128_xor((u128 )r, (u128 )p, (u128 *)q);
73		−}
74		−
75	66	static inline void le128_xor(le128 r, const le128 p, const le128 *q)
76	67	{
77		− u128_xor((u128 )r, (u128 )p, (u128 *)q);
	68	+ r->a = p->a ^ q->a;
	69	+ r->b = p->b ^ q->b;
78	70	}
79	71
80	72	#endif /* _CRYPTO_B128OPS_H */

include/linux/atomic/atomic-arch-fallback.h+3576−1587

@@ -8,2664 +8,4653 @@ · 7 unmodified lines
8	8
9	9	#include <linux/compiler.h>
10	10
11		−#ifndef arch_xchg_relaxed
12		−#define arch_xchg_acquire arch_xchg
13		−#define arch_xchg_release arch_xchg
14		−#define arch_xchg_relaxed arch_xchg
15		−#else /* arch_xchg_relaxed */
16		−
17		−#ifndef arch_xchg_acquire
18		−#define arch_xchg_acquire(...) \
19		− __atomic_op_acquire(arch_xchg, __VA_ARGS__)
	11	+#if defined(arch_xchg)
	12	+#define raw_xchg arch_xchg
	13	+#elif defined(arch_xchg_relaxed)
	14	+#define raw_xchg(...) \
	15	+ __atomic_op_fence(arch_xchg, __VA_ARGS__)
	16	+#else
	17	+extern void raw_xchg_not_implemented(void);
	18	+#define raw_xchg(...) raw_xchg_not_implemented()
20	19	#endif
21	20
22		−#ifndef arch_xchg_release
23		−#define arch_xchg_release(...) \
24		− __atomic_op_release(arch_xchg, __VA_ARGS__)
	21	+#if defined(arch_xchg_acquire)
	22	+#define raw_xchg_acquire arch_xchg_acquire
	23	+#elif defined(arch_xchg_relaxed)
	24	+#define raw_xchg_acquire(...) \
	25	+ __atomic_op_acquire(arch_xchg, __VA_ARGS__)
	26	+#elif defined(arch_xchg)
	27	+#define raw_xchg_acquire arch_xchg
	28	+#else
	29	+extern void raw_xchg_acquire_not_implemented(void);
	30	+#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
25	31	#endif
26	32
27		−#ifndef arch_xchg
28		−#define arch_xchg(...) \
29		− __atomic_op_fence(arch_xchg, __VA_ARGS__)
	33	+#if defined(arch_xchg_release)
	34	+#define raw_xchg_release arch_xchg_release
	35	+#elif defined(arch_xchg_relaxed)
	36	+#define raw_xchg_release(...) \
	37	+ __atomic_op_release(arch_xchg, __VA_ARGS__)
	38	+#elif defined(arch_xchg)
	39	+#define raw_xchg_release arch_xchg
	40	+#else
	41	+extern void raw_xchg_release_not_implemented(void);
	42	+#define raw_xchg_release(...) raw_xchg_release_not_implemented()
	43	+#endif
	44	+
	45	+#if defined(arch_xchg_relaxed)
	46	+#define raw_xchg_relaxed arch_xchg_relaxed
	47	+#elif defined(arch_xchg)
	48	+#define raw_xchg_relaxed arch_xchg
	49	+#else
	50	+extern void raw_xchg_relaxed_not_implemented(void);
	51	+#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
	52	+#endif
	53	+
	54	+#if defined(arch_cmpxchg)
	55	+#define raw_cmpxchg arch_cmpxchg
	56	+#elif defined(arch_cmpxchg_relaxed)
	57	+#define raw_cmpxchg(...) \
	58	+ __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
	59	+#else
	60	+extern void raw_cmpxchg_not_implemented(void);
	61	+#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
30	62	#endif
31	63
32		−#endif /* arch_xchg_relaxed */
33		−
34		−#ifndef arch_cmpxchg_relaxed
35		−#define arch_cmpxchg_acquire arch_cmpxchg
36		−#define arch_cmpxchg_release arch_cmpxchg
37		−#define arch_cmpxchg_relaxed arch_cmpxchg
38		−#else /* arch_cmpxchg_relaxed */
39		−
40		−#ifndef arch_cmpxchg_acquire
41		−#define arch_cmpxchg_acquire(...) \
	64	+#if defined(arch_cmpxchg_acquire)
	65	+#define raw_cmpxchg_acquire arch_cmpxchg_acquire
	66	+#elif defined(arch_cmpxchg_relaxed)
	67	+#define raw_cmpxchg_acquire(...) \
42	68	__atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
	69	+#elif defined(arch_cmpxchg)
	70	+#define raw_cmpxchg_acquire arch_cmpxchg
	71	+#else
	72	+extern void raw_cmpxchg_acquire_not_implemented(void);
	73	+#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
43	74	#endif
44	75
45		−#ifndef arch_cmpxchg_release
46		−#define arch_cmpxchg_release(...) \
	76	+#if defined(arch_cmpxchg_release)
	77	+#define raw_cmpxchg_release arch_cmpxchg_release
	78	+#elif defined(arch_cmpxchg_relaxed)
	79	+#define raw_cmpxchg_release(...) \
47	80	__atomic_op_release(arch_cmpxchg, __VA_ARGS__)
	81	+#elif defined(arch_cmpxchg)
	82	+#define raw_cmpxchg_release arch_cmpxchg
	83	+#else
	84	+extern void raw_cmpxchg_release_not_implemented(void);
	85	+#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
	86	+#endif
	87	+
	88	+#if defined(arch_cmpxchg_relaxed)
	89	+#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
	90	+#elif defined(arch_cmpxchg)
	91	+#define raw_cmpxchg_relaxed arch_cmpxchg
	92	+#else
	93	+extern void raw_cmpxchg_relaxed_not_implemented(void);
	94	+#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
	95	+#endif
	96	+
	97	+#if defined(arch_cmpxchg64)
	98	+#define raw_cmpxchg64 arch_cmpxchg64
	99	+#elif defined(arch_cmpxchg64_relaxed)
	100	+#define raw_cmpxchg64(...) \
	101	+ __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
	102	+#else
	103	+extern void raw_cmpxchg64_not_implemented(void);
	104	+#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
48	105	#endif
49	106
50		−#ifndef arch_cmpxchg
51		−#define arch_cmpxchg(...) \
52		− __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53		−#endif
54		−
55		−#endif /* arch_cmpxchg_relaxed */
56		−
57		−#ifndef arch_cmpxchg64_relaxed
58		−#define arch_cmpxchg64_acquire arch_cmpxchg64
59		−#define arch_cmpxchg64_release arch_cmpxchg64
60		−#define arch_cmpxchg64_relaxed arch_cmpxchg64
61		−#else /* arch_cmpxchg64_relaxed */
62		−
63		−#ifndef arch_cmpxchg64_acquire
64		−#define arch_cmpxchg64_acquire(...) \
	107	+#if defined(arch_cmpxchg64_acquire)
	108	+#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
	109	+#elif defined(arch_cmpxchg64_relaxed)
	110	+#define raw_cmpxchg64_acquire(...) \
65	111	__atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
	112	+#elif defined(arch_cmpxchg64)
	113	+#define raw_cmpxchg64_acquire arch_cmpxchg64
	114	+#else
	115	+extern void raw_cmpxchg64_acquire_not_implemented(void);
	116	+#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
66	117	#endif
67	118
68		−#ifndef arch_cmpxchg64_release
69		−#define arch_cmpxchg64_release(...) \
	119	+#if defined(arch_cmpxchg64_release)
	120	+#define raw_cmpxchg64_release arch_cmpxchg64_release
	121	+#elif defined(arch_cmpxchg64_relaxed)
	122	+#define raw_cmpxchg64_release(...) \
70	123	__atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
	124	+#elif defined(arch_cmpxchg64)
	125	+#define raw_cmpxchg64_release arch_cmpxchg64
	126	+#else
	127	+extern void raw_cmpxchg64_release_not_implemented(void);
	128	+#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
	129	+#endif
	130	+
	131	+#if defined(arch_cmpxchg64_relaxed)
	132	+#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
	133	+#elif defined(arch_cmpxchg64)
	134	+#define raw_cmpxchg64_relaxed arch_cmpxchg64
	135	+#else
	136	+extern void raw_cmpxchg64_relaxed_not_implemented(void);
	137	+#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
	138	+#endif
	139	+
	140	+#if defined(arch_cmpxchg128)
	141	+#define raw_cmpxchg128 arch_cmpxchg128
	142	+#elif defined(arch_cmpxchg128_relaxed)
	143	+#define raw_cmpxchg128(...) \
	144	+ __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
	145	+#else
	146	+extern void raw_cmpxchg128_not_implemented(void);
	147	+#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
	148	+#endif
	149	+
	150	+#if defined(arch_cmpxchg128_acquire)
	151	+#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
	152	+#elif defined(arch_cmpxchg128_relaxed)
	153	+#define raw_cmpxchg128_acquire(...) \
	154	+ __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
	155	+#elif defined(arch_cmpxchg128)
	156	+#define raw_cmpxchg128_acquire arch_cmpxchg128
	157	+#else
	158	+extern void raw_cmpxchg128_acquire_not_implemented(void);
	159	+#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
	160	+#endif
	161	+
	162	+#if defined(arch_cmpxchg128_release)
	163	+#define raw_cmpxchg128_release arch_cmpxchg128_release
	164	+#elif defined(arch_cmpxchg128_relaxed)
	165	+#define raw_cmpxchg128_release(...) \
	166	+ __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
	167	+#elif defined(arch_cmpxchg128)
	168	+#define raw_cmpxchg128_release arch_cmpxchg128
	169	+#else
	170	+extern void raw_cmpxchg128_release_not_implemented(void);
	171	+#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
	172	+#endif
	173	+
	174	+#if defined(arch_cmpxchg128_relaxed)
	175	+#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
	176	+#elif defined(arch_cmpxchg128)
	177	+#define raw_cmpxchg128_relaxed arch_cmpxchg128
	178	+#else
	179	+extern void raw_cmpxchg128_relaxed_not_implemented(void);
	180	+#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
	181	+#endif
	182	+
	183	+#if defined(arch_try_cmpxchg)
	184	+#define raw_try_cmpxchg arch_try_cmpxchg
	185	+#elif defined(arch_try_cmpxchg_relaxed)
	186	+#define raw_try_cmpxchg(...) \
	187	+ __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
	188	+#else
	189	+#define raw_try_cmpxchg(_ptr, _oldp, _new) \
	190	+({ \
	191	+ typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
	192	+ ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
	193	+ if (unlikely(___r != ___o)) \
	194	+ *___op = ___r; \
	195	+ likely(___r == ___o); \
	196	+})
71	197	#endif
72	198
73		−#ifndef arch_cmpxchg64
74		−#define arch_cmpxchg64(...) \
75		− __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
	199	+#if defined(arch_try_cmpxchg_acquire)
	200	+#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
	201	+#elif defined(arch_try_cmpxchg_relaxed)
	202	+#define raw_try_cmpxchg_acquire(...) \
	203	+ __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
	204	+#elif defined(arch_try_cmpxchg)
	205	+#define raw_try_cmpxchg_acquire arch_try_cmpxchg
	206	+#else
	207	+#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
	208	+({ \
	209	+ typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
	210	+ ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
	211	+ if (unlikely(___r != ___o)) \
	212	+ *___op = ___r; \
	213	+ likely(___r == ___o); \
	214	+})
76	215	#endif
77	216
78		−#endif /* arch_cmpxchg64_relaxed */
79		−
80		−#ifndef arch_try_cmpxchg_relaxed
81		−#ifdef arch_try_cmpxchg
82		−#define arch_try_cmpxchg_acquire arch_try_cmpxchg
83		−#define arch_try_cmpxchg_release arch_try_cmpxchg
84		−#define arch_try_cmpxchg_relaxed arch_try_cmpxchg
85		−#endif /* arch_try_cmpxchg */
86		−
87		−#ifndef arch_try_cmpxchg
88		−#define arch_try_cmpxchg(_ptr, _oldp, _new) \
	217	+#if defined(arch_try_cmpxchg_release)
	218	+#define raw_try_cmpxchg_release arch_try_cmpxchg_release
	219	+#elif defined(arch_try_cmpxchg_relaxed)
	220	+#define raw_try_cmpxchg_release(...) \
	221	+ __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
	222	+#elif defined(arch_try_cmpxchg)
	223	+#define raw_try_cmpxchg_release arch_try_cmpxchg
	224	+#else
	225	+#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
89	226	({ \
90	227	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
91		− ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
	228	+ ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
92	229	if (unlikely(___r != ___o)) \
93	230	*___op = ___r; \
94	231	likely(___r == ___o); \
95	232	})
96		−#endif /* arch_try_cmpxchg */
	233	+#endif
97	234
98		−#ifndef arch_try_cmpxchg_acquire
99		−#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
	235	+#if defined(arch_try_cmpxchg_relaxed)
	236	+#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
	237	+#elif defined(arch_try_cmpxchg)
	238	+#define raw_try_cmpxchg_relaxed arch_try_cmpxchg
	239	+#else
	240	+#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
100	241	({ \
101	242	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
102		− ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
	243	+ ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
103	244	if (unlikely(___r != ___o)) \
104	245	*___op = ___r; \
105	246	likely(___r == ___o); \
106	247	})
107		−#endif /* arch_try_cmpxchg_acquire */
	248	+#endif
108	249
109		−#ifndef arch_try_cmpxchg_release
110		−#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
	250	+#if defined(arch_try_cmpxchg64)
	251	+#define raw_try_cmpxchg64 arch_try_cmpxchg64
	252	+#elif defined(arch_try_cmpxchg64_relaxed)
	253	+#define raw_try_cmpxchg64(...) \
	254	+ __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
	255	+#else
	256	+#define raw_try_cmpxchg64(_ptr, _oldp, _new) \
111	257	({ \
112	258	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
113		− ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
	259	+ ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
114	260	if (unlikely(___r != ___o)) \
115	261	*___op = ___r; \
116	262	likely(___r == ___o); \
117	263	})
118		−#endif /* arch_try_cmpxchg_release */
	264	+#endif
119	265
120		−#ifndef arch_try_cmpxchg_relaxed
121		−#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
	266	+#if defined(arch_try_cmpxchg64_acquire)
	267	+#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
	268	+#elif defined(arch_try_cmpxchg64_relaxed)
	269	+#define raw_try_cmpxchg64_acquire(...) \
	270	+ __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
	271	+#elif defined(arch_try_cmpxchg64)
	272	+#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
	273	+#else
	274	+#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
122	275	({ \
123	276	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
124		− ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
	277	+ ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
125	278	if (unlikely(___r != ___o)) \
126	279	*___op = ___r; \
127	280	likely(___r == ___o); \
128	281	})
129		−#endif /* arch_try_cmpxchg_relaxed */
130		−
131		−#else /* arch_try_cmpxchg_relaxed */
132		−
133		−#ifndef arch_try_cmpxchg_acquire
134		−#define arch_try_cmpxchg_acquire(...) \
135		− __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
136	282	#endif
137	283
138		−#ifndef arch_try_cmpxchg_release
139		−#define arch_try_cmpxchg_release(...) \
140		− __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
	284	+#if defined(arch_try_cmpxchg64_release)
	285	+#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
	286	+#elif defined(arch_try_cmpxchg64_relaxed)
	287	+#define raw_try_cmpxchg64_release(...) \
	288	+ __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
	289	+#elif defined(arch_try_cmpxchg64)
	290	+#define raw_try_cmpxchg64_release arch_try_cmpxchg64
	291	+#else
	292	+#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
	293	+({ \
	294	+ typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
	295	+ ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
	296	+ if (unlikely(___r != ___o)) \
	297	+ *___op = ___r; \
	298	+ likely(___r == ___o); \
	299	+})
141	300	#endif
142	301
143		−#ifndef arch_try_cmpxchg
144		−#define arch_try_cmpxchg(...) \
145		− __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
	302	+#if defined(arch_try_cmpxchg64_relaxed)
	303	+#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
	304	+#elif defined(arch_try_cmpxchg64)
	305	+#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
	306	+#else
	307	+#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
	308	+({ \
	309	+ typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
	310	+ ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
	311	+ if (unlikely(___r != ___o)) \
	312	+ *___op = ___r; \
	313	+ likely(___r == ___o); \
	314	+})
146	315	#endif
147	316
148		−#endif /* arch_try_cmpxchg_relaxed */
149		−
150		−#ifndef arch_try_cmpxchg64_relaxed
151		−#ifdef arch_try_cmpxchg64
152		−#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64
153		−#define arch_try_cmpxchg64_release arch_try_cmpxchg64
154		−#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64
155		−#endif /* arch_try_cmpxchg64 */
156		−
157		−#ifndef arch_try_cmpxchg64
158		−#define arch_try_cmpxchg64(_ptr, _oldp, _new) \
	317	+#if defined(arch_try_cmpxchg128)
	318	+#define raw_try_cmpxchg128 arch_try_cmpxchg128
	319	+#elif defined(arch_try_cmpxchg128_relaxed)
	320	+#define raw_try_cmpxchg128(...) \
	321	+ __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
	322	+#else
	323	+#define raw_try_cmpxchg128(_ptr, _oldp, _new) \
159	324	({ \
160	325	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
161		− ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \
	326	+ ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
162	327	if (unlikely(___r != ___o)) \
163	328	*___op = ___r; \
164	329	likely(___r == ___o); \
165	330	})
166		−#endif /* arch_try_cmpxchg64 */
	331	+#endif
167	332
168		−#ifndef arch_try_cmpxchg64_acquire
169		−#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
	333	+#if defined(arch_try_cmpxchg128_acquire)
	334	+#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
	335	+#elif defined(arch_try_cmpxchg128_relaxed)
	336	+#define raw_try_cmpxchg128_acquire(...) \
	337	+ __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
	338	+#elif defined(arch_try_cmpxchg128)
	339	+#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
	340	+#else
	341	+#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
170	342	({ \
171	343	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
172		− ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \
	344	+ ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
173	345	if (unlikely(___r != ___o)) \
174	346	*___op = ___r; \
175	347	likely(___r == ___o); \
176	348	})
177		−#endif /* arch_try_cmpxchg64_acquire */
	349	+#endif
178	350
179		−#ifndef arch_try_cmpxchg64_release
180		−#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \
	351	+#if defined(arch_try_cmpxchg128_release)
	352	+#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
	353	+#elif defined(arch_try_cmpxchg128_relaxed)
	354	+#define raw_try_cmpxchg128_release(...) \
	355	+ __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
	356	+#elif defined(arch_try_cmpxchg128)
	357	+#define raw_try_cmpxchg128_release arch_try_cmpxchg128
	358	+#else
	359	+#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
181	360	({ \
182	361	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
183		− ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \
	362	+ ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
184	363	if (unlikely(___r != ___o)) \
185	364	*___op = ___r; \
186	365	likely(___r == ___o); \
187	366	})
188		−#endif /* arch_try_cmpxchg64_release */
	367	+#endif
189	368
190		−#ifndef arch_try_cmpxchg64_relaxed
191		−#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
	369	+#if defined(arch_try_cmpxchg128_relaxed)
	370	+#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
	371	+#elif defined(arch_try_cmpxchg128)
	372	+#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
	373	+#else
	374	+#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
192	375	({ \
193	376	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
194		− ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
	377	+ ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
195	378	if (unlikely(___r != ___o)) \
196	379	*___op = ___r; \
197	380	likely(___r == ___o); \
198	381	})
199		−#endif /* arch_try_cmpxchg64_relaxed */
200		−
201		−#else /* arch_try_cmpxchg64_relaxed */
202		−
203		−#ifndef arch_try_cmpxchg64_acquire
204		−#define arch_try_cmpxchg64_acquire(...) \
205		− __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
206	382	#endif
207	383
208		−#ifndef arch_try_cmpxchg64_release
209		−#define arch_try_cmpxchg64_release(...) \
210		− __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
211		−#endif
	384	+#define raw_cmpxchg_local arch_cmpxchg_local
212	385
213		−#ifndef arch_try_cmpxchg64
214		−#define arch_try_cmpxchg64(...) \
215		− __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
	386	+#ifdef arch_try_cmpxchg_local
	387	+#define raw_try_cmpxchg_local arch_try_cmpxchg_local
	388	+#else
	389	+#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
	390	+({ \
	391	+ typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
	392	+ ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
	393	+ if (unlikely(___r != ___o)) \
	394	+ *___op = ___r; \
	395	+ likely(___r == ___o); \
	396	+})
216	397	#endif
217	398
218		−#endif /* arch_try_cmpxchg64_relaxed */
	399	+#define raw_cmpxchg64_local arch_cmpxchg64_local
219	400
220		−#ifndef arch_try_cmpxchg_local
221		−#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \
	401	+#ifdef arch_try_cmpxchg64_local
	402	+#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
	403	+#else
	404	+#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
222	405	({ \
223	406	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
224		− ___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \
	407	+ ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
225	408	if (unlikely(___r != ___o)) \
226	409	*___op = ___r; \
227	410	likely(___r == ___o); \
228	411	})
229		−#endif /* arch_try_cmpxchg_local */
	412	+#endif
230	413
231		−#ifndef arch_try_cmpxchg64_local
232		−#define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \
	414	+#define raw_cmpxchg128_local arch_cmpxchg128_local
	415	+
	416	+#ifdef arch_try_cmpxchg128_local
	417	+#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
	418	+#else
	419	+#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
233	420	({ \
234	421	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \
235		− ___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \
	422	+ ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
236	423	if (unlikely(___r != ___o)) \
237	424	*___op = ___r; \
238	425	likely(___r == ___o); \
239	426	})
240		−#endif /* arch_try_cmpxchg64_local */
	427	+#endif
	428	+
	429	+#define raw_sync_cmpxchg arch_sync_cmpxchg
	430	+
	431	+/**
	432	+ * raw_atomic_read() - atomic load with relaxed ordering
	433	+ * @v: pointer to atomic_t
	434	+ *
	435	+ * Atomically loads the value of @v with relaxed ordering.
	436	+ *
	437	+ * Safe to use in noinstr code; prefer atomic_read() elsewhere.
	438	+ *
	439	+ * Return: The value loaded from @v.
	440	+ */
	441	+static __always_inline int
	442	+raw_atomic_read(const atomic_t *v)
	443	+{
	444	+ return arch_atomic_read(v);
	445	+}
241	446
242		−#ifndef arch_atomic_read_acquire
	447	+/**
	448	+ * raw_atomic_read_acquire() - atomic load with acquire ordering
	449	+ * @v: pointer to atomic_t
	450	+ *
	451	+ * Atomically loads the value of @v with acquire ordering.
	452	+ *
	453	+ * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
	454	+ *
	455	+ * Return: The value loaded from @v.
	456	+ */
243	457	static __always_inline int
244		−arch_atomic_read_acquire(const atomic_t *v)
	458	+raw_atomic_read_acquire(const atomic_t *v)
245	459	{
	460	+#if defined(arch_atomic_read_acquire)
	461	+ return arch_atomic_read_acquire(v);
	462	+#elif defined(arch_atomic_read)
	463	+ return arch_atomic_read(v);
	464	+#else
246	465	int ret;
247	466
248	467	if (__native_word(atomic_t)) {
249	468	ret = smp_load_acquire(&(v)->counter);
250	469	} else {
251		− ret = arch_atomic_read(v);
	470	+ ret = raw_atomic_read(v);
252	471	__atomic_acquire_fence();
253	472	}
254	473
255	474	return ret;
256		−}
257		−#define arch_atomic_read_acquire arch_atomic_read_acquire
258	475	#endif
	476	+}
	477	+
	478	+/**
	479	+ * raw_atomic_set() - atomic set with relaxed ordering
	480	+ * @v: pointer to atomic_t
	481	+ * @i: int value to assign
	482	+ *
	483	+ * Atomically sets @v to @i with relaxed ordering.
	484	+ *
	485	+ * Safe to use in noinstr code; prefer atomic_set() elsewhere.
	486	+ *
	487	+ * Return: Nothing.
	488	+ */
	489	+static __always_inline void
	490	+raw_atomic_set(atomic_t *v, int i)
	491	+{
	492	+ arch_atomic_set(v, i);
	493	+}
259	494
260		−#ifndef arch_atomic_set_release
	495	+/**
	496	+ * raw_atomic_set_release() - atomic set with release ordering
	497	+ * @v: pointer to atomic_t
	498	+ * @i: int value to assign
	499	+ *
	500	+ * Atomically sets @v to @i with release ordering.
	501	+ *
	502	+ * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
	503	+ *
	504	+ * Return: Nothing.
	505	+ */
261	506	static __always_inline void
262		−arch_atomic_set_release(atomic_t *v, int i)
	507	+raw_atomic_set_release(atomic_t *v, int i)
263	508	{
	509	+#if defined(arch_atomic_set_release)
	510	+ arch_atomic_set_release(v, i);
	511	+#elif defined(arch_atomic_set)
	512	+ arch_atomic_set(v, i);
	513	+#else
264	514	if (__native_word(atomic_t)) {
265	515	smp_store_release(&(v)->counter, i);
266	516	} else {
267	517	__atomic_release_fence();
268		− arch_atomic_set(v, i);
	518	+ raw_atomic_set(v, i);
269	519	}
270		−}
271		−#define arch_atomic_set_release arch_atomic_set_release
272	520	#endif
	521	+}
	522	+
	523	+/**
	524	+ * raw_atomic_add() - atomic add with relaxed ordering
	525	+ * @i: int value to add
	526	+ * @v: pointer to atomic_t
	527	+ *
	528	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	529	+ *
	530	+ * Safe to use in noinstr code; prefer atomic_add() elsewhere.
	531	+ *
	532	+ * Return: Nothing.
	533	+ */
	534	+static __always_inline void
	535	+raw_atomic_add(int i, atomic_t *v)
	536	+{
	537	+ arch_atomic_add(i, v);
	538	+}
273	539
274		−#ifndef arch_atomic_add_return_relaxed
275		−#define arch_atomic_add_return_acquire arch_atomic_add_return
276		−#define arch_atomic_add_return_release arch_atomic_add_return
277		−#define arch_atomic_add_return_relaxed arch_atomic_add_return
278		−#else /* arch_atomic_add_return_relaxed */
	540	+/**
	541	+ * raw_atomic_add_return() - atomic add with full ordering
	542	+ * @i: int value to add
	543	+ * @v: pointer to atomic_t
	544	+ *
	545	+ * Atomically updates @v to (@v + @i) with full ordering.
	546	+ *
	547	+ * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
	548	+ *
	549	+ * Return: The updated value of @v.
	550	+ */
	551	+static __always_inline int
	552	+raw_atomic_add_return(int i, atomic_t *v)
	553	+{
	554	+#if defined(arch_atomic_add_return)
	555	+ return arch_atomic_add_return(i, v);
	556	+#elif defined(arch_atomic_add_return_relaxed)
	557	+ int ret;
	558	+ __atomic_pre_full_fence();
	559	+ ret = arch_atomic_add_return_relaxed(i, v);
	560	+ __atomic_post_full_fence();
	561	+ return ret;
	562	+#else
	563	+#error "Unable to define raw_atomic_add_return"
	564	+#endif
	565	+}
279	566
280		−#ifndef arch_atomic_add_return_acquire
	567	+/**
	568	+ * raw_atomic_add_return_acquire() - atomic add with acquire ordering
	569	+ * @i: int value to add
	570	+ * @v: pointer to atomic_t
	571	+ *
	572	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	573	+ *
	574	+ * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
	575	+ *
	576	+ * Return: The updated value of @v.
	577	+ */
281	578	static __always_inline int
282		−arch_atomic_add_return_acquire(int i, atomic_t *v)
	579	+raw_atomic_add_return_acquire(int i, atomic_t *v)
283	580	{
	581	+#if defined(arch_atomic_add_return_acquire)
	582	+ return arch_atomic_add_return_acquire(i, v);
	583	+#elif defined(arch_atomic_add_return_relaxed)
284	584	int ret = arch_atomic_add_return_relaxed(i, v);
285	585	__atomic_acquire_fence();
286	586	return ret;
287		−}
288		−#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
	587	+#elif defined(arch_atomic_add_return)
	588	+ return arch_atomic_add_return(i, v);
	589	+#else
	590	+#error "Unable to define raw_atomic_add_return_acquire"
289	591	#endif
	592	+}
290	593
291		−#ifndef arch_atomic_add_return_release
	594	+/**
	595	+ * raw_atomic_add_return_release() - atomic add with release ordering
	596	+ * @i: int value to add
	597	+ * @v: pointer to atomic_t
	598	+ *
	599	+ * Atomically updates @v to (@v + @i) with release ordering.
	600	+ *
	601	+ * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
	602	+ *
	603	+ * Return: The updated value of @v.
	604	+ */
292	605	static __always_inline int
293		−arch_atomic_add_return_release(int i, atomic_t *v)
	606	+raw_atomic_add_return_release(int i, atomic_t *v)
294	607	{
	608	+#if defined(arch_atomic_add_return_release)
	609	+ return arch_atomic_add_return_release(i, v);
	610	+#elif defined(arch_atomic_add_return_relaxed)
295	611	__atomic_release_fence();
296	612	return arch_atomic_add_return_relaxed(i, v);
	613	+#elif defined(arch_atomic_add_return)
	614	+ return arch_atomic_add_return(i, v);
	615	+#else
	616	+#error "Unable to define raw_atomic_add_return_release"
	617	+#endif
297	618	}
298		−#define arch_atomic_add_return_release arch_atomic_add_return_release
	619	+
	620	+/**
	621	+ * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
	622	+ * @i: int value to add
	623	+ * @v: pointer to atomic_t
	624	+ *
	625	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	626	+ *
	627	+ * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
	628	+ *
	629	+ * Return: The updated value of @v.
	630	+ */
	631	+static __always_inline int
	632	+raw_atomic_add_return_relaxed(int i, atomic_t *v)
	633	+{
	634	+#if defined(arch_atomic_add_return_relaxed)
	635	+ return arch_atomic_add_return_relaxed(i, v);
	636	+#elif defined(arch_atomic_add_return)
	637	+ return arch_atomic_add_return(i, v);
	638	+#else
	639	+#error "Unable to define raw_atomic_add_return_relaxed"
299	640	#endif
	641	+}
300	642
301		−#ifndef arch_atomic_add_return
	643	+/**
	644	+ * raw_atomic_fetch_add() - atomic add with full ordering
	645	+ * @i: int value to add
	646	+ * @v: pointer to atomic_t
	647	+ *
	648	+ * Atomically updates @v to (@v + @i) with full ordering.
	649	+ *
	650	+ * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
	651	+ *
	652	+ * Return: The original value of @v.
	653	+ */
302	654	static __always_inline int
303		−arch_atomic_add_return(int i, atomic_t *v)
	655	+raw_atomic_fetch_add(int i, atomic_t *v)
304	656	{
	657	+#if defined(arch_atomic_fetch_add)
	658	+ return arch_atomic_fetch_add(i, v);
	659	+#elif defined(arch_atomic_fetch_add_relaxed)
305	660	int ret;
306	661	__atomic_pre_full_fence();
307		− ret = arch_atomic_add_return_relaxed(i, v);
	662	+ ret = arch_atomic_fetch_add_relaxed(i, v);
308	663	__atomic_post_full_fence();
309	664	return ret;
310		−}
311		−#define arch_atomic_add_return arch_atomic_add_return
	665	+#else
	666	+#error "Unable to define raw_atomic_fetch_add"
312	667	#endif
	668	+}
313	669
314		−#endif /* arch_atomic_add_return_relaxed */
315		−
316		−#ifndef arch_atomic_fetch_add_relaxed
317		−#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
318		−#define arch_atomic_fetch_add_release arch_atomic_fetch_add
319		−#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
320		−#else /* arch_atomic_fetch_add_relaxed */
321		−
322		−#ifndef arch_atomic_fetch_add_acquire
	670	+/**
	671	+ * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
	672	+ * @i: int value to add
	673	+ * @v: pointer to atomic_t
	674	+ *
	675	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	676	+ *
	677	+ * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
	678	+ *
	679	+ * Return: The original value of @v.
	680	+ */
323	681	static __always_inline int
324		−arch_atomic_fetch_add_acquire(int i, atomic_t *v)
	682	+raw_atomic_fetch_add_acquire(int i, atomic_t *v)
325	683	{
	684	+#if defined(arch_atomic_fetch_add_acquire)
	685	+ return arch_atomic_fetch_add_acquire(i, v);
	686	+#elif defined(arch_atomic_fetch_add_relaxed)
326	687	int ret = arch_atomic_fetch_add_relaxed(i, v);
327	688	__atomic_acquire_fence();
328	689	return ret;
329		−}
330		−#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
	690	+#elif defined(arch_atomic_fetch_add)
	691	+ return arch_atomic_fetch_add(i, v);
	692	+#else
	693	+#error "Unable to define raw_atomic_fetch_add_acquire"
331	694	#endif
	695	+}
332	696
333		−#ifndef arch_atomic_fetch_add_release
	697	+/**
	698	+ * raw_atomic_fetch_add_release() - atomic add with release ordering
	699	+ * @i: int value to add
	700	+ * @v: pointer to atomic_t
	701	+ *
	702	+ * Atomically updates @v to (@v + @i) with release ordering.
	703	+ *
	704	+ * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
	705	+ *
	706	+ * Return: The original value of @v.
	707	+ */
334	708	static __always_inline int
335		−arch_atomic_fetch_add_release(int i, atomic_t *v)
	709	+raw_atomic_fetch_add_release(int i, atomic_t *v)
336	710	{
	711	+#if defined(arch_atomic_fetch_add_release)
	712	+ return arch_atomic_fetch_add_release(i, v);
	713	+#elif defined(arch_atomic_fetch_add_relaxed)
337	714	__atomic_release_fence();
338	715	return arch_atomic_fetch_add_relaxed(i, v);
	716	+#elif defined(arch_atomic_fetch_add)
	717	+ return arch_atomic_fetch_add(i, v);
	718	+#else
	719	+#error "Unable to define raw_atomic_fetch_add_release"
	720	+#endif
339	721	}
340		−#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
	722	+
	723	+/**
	724	+ * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
	725	+ * @i: int value to add
	726	+ * @v: pointer to atomic_t
	727	+ *
	728	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	729	+ *
	730	+ * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
	731	+ *
	732	+ * Return: The original value of @v.
	733	+ */
	734	+static __always_inline int
	735	+raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
	736	+{
	737	+#if defined(arch_atomic_fetch_add_relaxed)
	738	+ return arch_atomic_fetch_add_relaxed(i, v);
	739	+#elif defined(arch_atomic_fetch_add)
	740	+ return arch_atomic_fetch_add(i, v);
	741	+#else
	742	+#error "Unable to define raw_atomic_fetch_add_relaxed"
341	743	#endif
	744	+}
	745	+
	746	+/**
	747	+ * raw_atomic_sub() - atomic subtract with relaxed ordering
	748	+ * @i: int value to subtract
	749	+ * @v: pointer to atomic_t
	750	+ *
	751	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	752	+ *
	753	+ * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
	754	+ *
	755	+ * Return: Nothing.
	756	+ */
	757	+static __always_inline void
	758	+raw_atomic_sub(int i, atomic_t *v)
	759	+{
	760	+ arch_atomic_sub(i, v);
	761	+}
342	762
343		−#ifndef arch_atomic_fetch_add
	763	+/**
	764	+ * raw_atomic_sub_return() - atomic subtract with full ordering
	765	+ * @i: int value to subtract
	766	+ * @v: pointer to atomic_t
	767	+ *
	768	+ * Atomically updates @v to (@v - @i) with full ordering.
	769	+ *
	770	+ * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
	771	+ *
	772	+ * Return: The updated value of @v.
	773	+ */
344	774	static __always_inline int
345		−arch_atomic_fetch_add(int i, atomic_t *v)
	775	+raw_atomic_sub_return(int i, atomic_t *v)
346	776	{
	777	+#if defined(arch_atomic_sub_return)
	778	+ return arch_atomic_sub_return(i, v);
	779	+#elif defined(arch_atomic_sub_return_relaxed)
347	780	int ret;
348	781	__atomic_pre_full_fence();
349		− ret = arch_atomic_fetch_add_relaxed(i, v);
	782	+ ret = arch_atomic_sub_return_relaxed(i, v);
350	783	__atomic_post_full_fence();
351	784	return ret;
352		−}
353		−#define arch_atomic_fetch_add arch_atomic_fetch_add
	785	+#else
	786	+#error "Unable to define raw_atomic_sub_return"
354	787	#endif
	788	+}
355	789
356		−#endif /* arch_atomic_fetch_add_relaxed */
357		−
358		−#ifndef arch_atomic_sub_return_relaxed
359		−#define arch_atomic_sub_return_acquire arch_atomic_sub_return
360		−#define arch_atomic_sub_return_release arch_atomic_sub_return
361		−#define arch_atomic_sub_return_relaxed arch_atomic_sub_return
362		−#else /* arch_atomic_sub_return_relaxed */
363		−
364		−#ifndef arch_atomic_sub_return_acquire
	790	+/**
	791	+ * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
	792	+ * @i: int value to subtract
	793	+ * @v: pointer to atomic_t
	794	+ *
	795	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	796	+ *
	797	+ * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
	798	+ *
	799	+ * Return: The updated value of @v.
	800	+ */
365	801	static __always_inline int
366		−arch_atomic_sub_return_acquire(int i, atomic_t *v)
	802	+raw_atomic_sub_return_acquire(int i, atomic_t *v)
367	803	{
	804	+#if defined(arch_atomic_sub_return_acquire)
	805	+ return arch_atomic_sub_return_acquire(i, v);
	806	+#elif defined(arch_atomic_sub_return_relaxed)
368	807	int ret = arch_atomic_sub_return_relaxed(i, v);
369	808	__atomic_acquire_fence();
370	809	return ret;
371		−}
372		−#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
	810	+#elif defined(arch_atomic_sub_return)
	811	+ return arch_atomic_sub_return(i, v);
	812	+#else
	813	+#error "Unable to define raw_atomic_sub_return_acquire"
373	814	#endif
	815	+}
374	816
375		−#ifndef arch_atomic_sub_return_release
	817	+/**
	818	+ * raw_atomic_sub_return_release() - atomic subtract with release ordering
	819	+ * @i: int value to subtract
	820	+ * @v: pointer to atomic_t
	821	+ *
	822	+ * Atomically updates @v to (@v - @i) with release ordering.
	823	+ *
	824	+ * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
	825	+ *
	826	+ * Return: The updated value of @v.
	827	+ */
376	828	static __always_inline int
377		−arch_atomic_sub_return_release(int i, atomic_t *v)
	829	+raw_atomic_sub_return_release(int i, atomic_t *v)
378	830	{
	831	+#if defined(arch_atomic_sub_return_release)
	832	+ return arch_atomic_sub_return_release(i, v);
	833	+#elif defined(arch_atomic_sub_return_relaxed)
379	834	__atomic_release_fence();
380	835	return arch_atomic_sub_return_relaxed(i, v);
	836	+#elif defined(arch_atomic_sub_return)
	837	+ return arch_atomic_sub_return(i, v);
	838	+#else
	839	+#error "Unable to define raw_atomic_sub_return_release"
	840	+#endif
381	841	}
382		−#define arch_atomic_sub_return_release arch_atomic_sub_return_release
	842	+
	843	+/**
	844	+ * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
	845	+ * @i: int value to subtract
	846	+ * @v: pointer to atomic_t
	847	+ *
	848	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	849	+ *
	850	+ * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
	851	+ *
	852	+ * Return: The updated value of @v.
	853	+ */
	854	+static __always_inline int
	855	+raw_atomic_sub_return_relaxed(int i, atomic_t *v)
	856	+{
	857	+#if defined(arch_atomic_sub_return_relaxed)
	858	+ return arch_atomic_sub_return_relaxed(i, v);
	859	+#elif defined(arch_atomic_sub_return)
	860	+ return arch_atomic_sub_return(i, v);
	861	+#else
	862	+#error "Unable to define raw_atomic_sub_return_relaxed"
383	863	#endif
	864	+}
384	865
385		−#ifndef arch_atomic_sub_return
	866	+/**
	867	+ * raw_atomic_fetch_sub() - atomic subtract with full ordering
	868	+ * @i: int value to subtract
	869	+ * @v: pointer to atomic_t
	870	+ *
	871	+ * Atomically updates @v to (@v - @i) with full ordering.
	872	+ *
	873	+ * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
	874	+ *
	875	+ * Return: The original value of @v.
	876	+ */
386	877	static __always_inline int
387		−arch_atomic_sub_return(int i, atomic_t *v)
	878	+raw_atomic_fetch_sub(int i, atomic_t *v)
388	879	{
	880	+#if defined(arch_atomic_fetch_sub)
	881	+ return arch_atomic_fetch_sub(i, v);
	882	+#elif defined(arch_atomic_fetch_sub_relaxed)
389	883	int ret;
390	884	__atomic_pre_full_fence();
391		− ret = arch_atomic_sub_return_relaxed(i, v);
	885	+ ret = arch_atomic_fetch_sub_relaxed(i, v);
392	886	__atomic_post_full_fence();
393	887	return ret;
394		−}
395		−#define arch_atomic_sub_return arch_atomic_sub_return
	888	+#else
	889	+#error "Unable to define raw_atomic_fetch_sub"
396	890	#endif
	891	+}
397	892
398		−#endif /* arch_atomic_sub_return_relaxed */
399		−
400		−#ifndef arch_atomic_fetch_sub_relaxed
401		−#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
402		−#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
403		−#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
404		−#else /* arch_atomic_fetch_sub_relaxed */
405		−
406		−#ifndef arch_atomic_fetch_sub_acquire
	893	+/**
	894	+ * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
	895	+ * @i: int value to subtract
	896	+ * @v: pointer to atomic_t
	897	+ *
	898	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	899	+ *
	900	+ * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
	901	+ *
	902	+ * Return: The original value of @v.
	903	+ */
407	904	static __always_inline int
408		−arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
	905	+raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
409	906	{
	907	+#if defined(arch_atomic_fetch_sub_acquire)
	908	+ return arch_atomic_fetch_sub_acquire(i, v);
	909	+#elif defined(arch_atomic_fetch_sub_relaxed)
410	910	int ret = arch_atomic_fetch_sub_relaxed(i, v);
411	911	__atomic_acquire_fence();
412	912	return ret;
413		−}
414		−#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
	913	+#elif defined(arch_atomic_fetch_sub)
	914	+ return arch_atomic_fetch_sub(i, v);
	915	+#else
	916	+#error "Unable to define raw_atomic_fetch_sub_acquire"
415	917	#endif
	918	+}
416	919
417		−#ifndef arch_atomic_fetch_sub_release
	920	+/**
	921	+ * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
	922	+ * @i: int value to subtract
	923	+ * @v: pointer to atomic_t
	924	+ *
	925	+ * Atomically updates @v to (@v - @i) with release ordering.
	926	+ *
	927	+ * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
	928	+ *
	929	+ * Return: The original value of @v.
	930	+ */
418	931	static __always_inline int
419		−arch_atomic_fetch_sub_release(int i, atomic_t *v)
	932	+raw_atomic_fetch_sub_release(int i, atomic_t *v)
420	933	{
	934	+#if defined(arch_atomic_fetch_sub_release)
	935	+ return arch_atomic_fetch_sub_release(i, v);
	936	+#elif defined(arch_atomic_fetch_sub_relaxed)
421	937	__atomic_release_fence();
422	938	return arch_atomic_fetch_sub_relaxed(i, v);
423		−}
424		−#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
	939	+#elif defined(arch_atomic_fetch_sub)
	940	+ return arch_atomic_fetch_sub(i, v);
	941	+#else
	942	+#error "Unable to define raw_atomic_fetch_sub_release"
425	943	#endif
	944	+}
426	945
427		−#ifndef arch_atomic_fetch_sub
	946	+/**
	947	+ * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
	948	+ * @i: int value to subtract
	949	+ * @v: pointer to atomic_t
	950	+ *
	951	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	952	+ *
	953	+ * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
	954	+ *
	955	+ * Return: The original value of @v.
	956	+ */
428	957	static __always_inline int
429		−arch_atomic_fetch_sub(int i, atomic_t *v)
	958	+raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
430	959	{
431		− int ret;
432		− __atomic_pre_full_fence();
433		− ret = arch_atomic_fetch_sub_relaxed(i, v);
434		− __atomic_post_full_fence();
435		− return ret;
436		−}
437		−#define arch_atomic_fetch_sub arch_atomic_fetch_sub
	960	+#if defined(arch_atomic_fetch_sub_relaxed)
	961	+ return arch_atomic_fetch_sub_relaxed(i, v);
	962	+#elif defined(arch_atomic_fetch_sub)
	963	+ return arch_atomic_fetch_sub(i, v);
	964	+#else
	965	+#error "Unable to define raw_atomic_fetch_sub_relaxed"
438	966	#endif
439		−
440		−#endif /* arch_atomic_fetch_sub_relaxed */
441		−
442		−#ifndef arch_atomic_inc
443		−static __always_inline void
444		−arch_atomic_inc(atomic_t *v)
445		−{
446		− arch_atomic_add(1, v);
447	967	}
448		−#define arch_atomic_inc arch_atomic_inc
449		−#endif
450		−
451		−#ifndef arch_atomic_inc_return_relaxed
452		−#ifdef arch_atomic_inc_return
453		−#define arch_atomic_inc_return_acquire arch_atomic_inc_return
454		−#define arch_atomic_inc_return_release arch_atomic_inc_return
455		−#define arch_atomic_inc_return_relaxed arch_atomic_inc_return
456		−#endif /* arch_atomic_inc_return */
457	968
458		−#ifndef arch_atomic_inc_return
459		−static __always_inline int
460		−arch_atomic_inc_return(atomic_t *v)
	969	+/**
	970	+ * raw_atomic_inc() - atomic increment with relaxed ordering
	971	+ * @v: pointer to atomic_t
	972	+ *
	973	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	974	+ *
	975	+ * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
	976	+ *
	977	+ * Return: Nothing.
	978	+ */
	979	+static __always_inline void
	980	+raw_atomic_inc(atomic_t *v)
461	981	{
462		− return arch_atomic_add_return(1, v);
463		−}
464		−#define arch_atomic_inc_return arch_atomic_inc_return
	982	+#if defined(arch_atomic_inc)
	983	+ arch_atomic_inc(v);
	984	+#else
	985	+ raw_atomic_add(1, v);
465	986	#endif
466		−
467		−#ifndef arch_atomic_inc_return_acquire
468		−static __always_inline int
469		−arch_atomic_inc_return_acquire(atomic_t *v)
470		−{
471		− return arch_atomic_add_return_acquire(1, v);
472	987	}
473		−#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
474		−#endif
475	988
476		−#ifndef arch_atomic_inc_return_release
	989	+/**
	990	+ * raw_atomic_inc_return() - atomic increment with full ordering
	991	+ * @v: pointer to atomic_t
	992	+ *
	993	+ * Atomically updates @v to (@v + 1) with full ordering.
	994	+ *
	995	+ * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
	996	+ *
	997	+ * Return: The updated value of @v.
	998	+ */
477	999	static __always_inline int
478		−arch_atomic_inc_return_release(atomic_t *v)
	1000	+raw_atomic_inc_return(atomic_t *v)
479	1001	{
480		− return arch_atomic_add_return_release(1, v);
481		−}
482		−#define arch_atomic_inc_return_release arch_atomic_inc_return_release
	1002	+#if defined(arch_atomic_inc_return)
	1003	+ return arch_atomic_inc_return(v);
	1004	+#elif defined(arch_atomic_inc_return_relaxed)
	1005	+ int ret;
	1006	+ __atomic_pre_full_fence();
	1007	+ ret = arch_atomic_inc_return_relaxed(v);
	1008	+ __atomic_post_full_fence();
	1009	+ return ret;
	1010	+#else
	1011	+ return raw_atomic_add_return(1, v);
483	1012	#endif
484		−
485		−#ifndef arch_atomic_inc_return_relaxed
486		−static __always_inline int
487		−arch_atomic_inc_return_relaxed(atomic_t *v)
488		−{
489		− return arch_atomic_add_return_relaxed(1, v);
490	1013	}
491		−#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
492		−#endif
493	1014
494		−#else /* arch_atomic_inc_return_relaxed */
495		−
496		−#ifndef arch_atomic_inc_return_acquire
	1015	+/**
	1016	+ * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
	1017	+ * @v: pointer to atomic_t
	1018	+ *
	1019	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	1020	+ *
	1021	+ * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
	1022	+ *
	1023	+ * Return: The updated value of @v.
	1024	+ */
497	1025	static __always_inline int
498		−arch_atomic_inc_return_acquire(atomic_t *v)
	1026	+raw_atomic_inc_return_acquire(atomic_t *v)
499	1027	{
	1028	+#if defined(arch_atomic_inc_return_acquire)
	1029	+ return arch_atomic_inc_return_acquire(v);
	1030	+#elif defined(arch_atomic_inc_return_relaxed)
500	1031	int ret = arch_atomic_inc_return_relaxed(v);
501	1032	__atomic_acquire_fence();
502	1033	return ret;
503		−}
504		−#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
	1034	+#elif defined(arch_atomic_inc_return)
	1035	+ return arch_atomic_inc_return(v);
	1036	+#else
	1037	+ return raw_atomic_add_return_acquire(1, v);
505	1038	#endif
	1039	+}
506	1040
507		−#ifndef arch_atomic_inc_return_release
	1041	+/**
	1042	+ * raw_atomic_inc_return_release() - atomic increment with release ordering
	1043	+ * @v: pointer to atomic_t
	1044	+ *
	1045	+ * Atomically updates @v to (@v + 1) with release ordering.
	1046	+ *
	1047	+ * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
	1048	+ *
	1049	+ * Return: The updated value of @v.
	1050	+ */
508	1051	static __always_inline int
509		−arch_atomic_inc_return_release(atomic_t *v)
	1052	+raw_atomic_inc_return_release(atomic_t *v)
510	1053	{
	1054	+#if defined(arch_atomic_inc_return_release)
	1055	+ return arch_atomic_inc_return_release(v);
	1056	+#elif defined(arch_atomic_inc_return_relaxed)
511	1057	__atomic_release_fence();
512	1058	return arch_atomic_inc_return_relaxed(v);
513		−}
514		−#define arch_atomic_inc_return_release arch_atomic_inc_return_release
	1059	+#elif defined(arch_atomic_inc_return)
	1060	+ return arch_atomic_inc_return(v);
	1061	+#else
	1062	+ return raw_atomic_add_return_release(1, v);
515	1063	#endif
516		−
517		−#ifndef arch_atomic_inc_return
518		−static __always_inline int
519		−arch_atomic_inc_return(atomic_t *v)
520		−{
521		− int ret;
522		− __atomic_pre_full_fence();
523		− ret = arch_atomic_inc_return_relaxed(v);
524		− __atomic_post_full_fence();
525		− return ret;
526	1064	}
527		−#define arch_atomic_inc_return arch_atomic_inc_return
528		−#endif
529		−
530		−#endif /* arch_atomic_inc_return_relaxed */
531	1065
532		−#ifndef arch_atomic_fetch_inc_relaxed
533		−#ifdef arch_atomic_fetch_inc
534		−#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
535		−#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
536		−#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
537		−#endif /* arch_atomic_fetch_inc */
538		−
539		−#ifndef arch_atomic_fetch_inc
	1066	+/**
	1067	+ * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
	1068	+ * @v: pointer to atomic_t
	1069	+ *
	1070	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	1071	+ *
	1072	+ * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
	1073	+ *
	1074	+ * Return: The updated value of @v.
	1075	+ */
540	1076	static __always_inline int
541		−arch_atomic_fetch_inc(atomic_t *v)
	1077	+raw_atomic_inc_return_relaxed(atomic_t *v)
542	1078	{
543		− return arch_atomic_fetch_add(1, v);
544		−}
545		−#define arch_atomic_fetch_inc arch_atomic_fetch_inc
	1079	+#if defined(arch_atomic_inc_return_relaxed)
	1080	+ return arch_atomic_inc_return_relaxed(v);
	1081	+#elif defined(arch_atomic_inc_return)
	1082	+ return arch_atomic_inc_return(v);
	1083	+#else
	1084	+ return raw_atomic_add_return_relaxed(1, v);
546	1085	#endif
547		−
548		−#ifndef arch_atomic_fetch_inc_acquire
549		−static __always_inline int
550		−arch_atomic_fetch_inc_acquire(atomic_t *v)
551		−{
552		− return arch_atomic_fetch_add_acquire(1, v);
553	1086	}
554		−#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
555		−#endif
556	1087
557		−#ifndef arch_atomic_fetch_inc_release
	1088	+/**
	1089	+ * raw_atomic_fetch_inc() - atomic increment with full ordering
	1090	+ * @v: pointer to atomic_t
	1091	+ *
	1092	+ * Atomically updates @v to (@v + 1) with full ordering.
	1093	+ *
	1094	+ * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
	1095	+ *
	1096	+ * Return: The original value of @v.
	1097	+ */
558	1098	static __always_inline int
559		−arch_atomic_fetch_inc_release(atomic_t *v)
	1099	+raw_atomic_fetch_inc(atomic_t *v)
560	1100	{
561		− return arch_atomic_fetch_add_release(1, v);
562		−}
563		−#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
	1101	+#if defined(arch_atomic_fetch_inc)
	1102	+ return arch_atomic_fetch_inc(v);
	1103	+#elif defined(arch_atomic_fetch_inc_relaxed)
	1104	+ int ret;
	1105	+ __atomic_pre_full_fence();
	1106	+ ret = arch_atomic_fetch_inc_relaxed(v);
	1107	+ __atomic_post_full_fence();
	1108	+ return ret;
	1109	+#else
	1110	+ return raw_atomic_fetch_add(1, v);
564	1111	#endif
565		−
566		−#ifndef arch_atomic_fetch_inc_relaxed
567		−static __always_inline int
568		−arch_atomic_fetch_inc_relaxed(atomic_t *v)
569		−{
570		− return arch_atomic_fetch_add_relaxed(1, v);
571	1112	}
572		−#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
573		−#endif
574	1113
575		−#else /* arch_atomic_fetch_inc_relaxed */
576		−
577		−#ifndef arch_atomic_fetch_inc_acquire
	1114	+/**
	1115	+ * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
	1116	+ * @v: pointer to atomic_t
	1117	+ *
	1118	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	1119	+ *
	1120	+ * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
	1121	+ *
	1122	+ * Return: The original value of @v.
	1123	+ */
578	1124	static __always_inline int
579		−arch_atomic_fetch_inc_acquire(atomic_t *v)
	1125	+raw_atomic_fetch_inc_acquire(atomic_t *v)
580	1126	{
	1127	+#if defined(arch_atomic_fetch_inc_acquire)
	1128	+ return arch_atomic_fetch_inc_acquire(v);
	1129	+#elif defined(arch_atomic_fetch_inc_relaxed)
581	1130	int ret = arch_atomic_fetch_inc_relaxed(v);
582	1131	__atomic_acquire_fence();
583	1132	return ret;
584		−}
585		−#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
	1133	+#elif defined(arch_atomic_fetch_inc)
	1134	+ return arch_atomic_fetch_inc(v);
	1135	+#else
	1136	+ return raw_atomic_fetch_add_acquire(1, v);
586	1137	#endif
	1138	+}
587	1139
588		−#ifndef arch_atomic_fetch_inc_release
	1140	+/**
	1141	+ * raw_atomic_fetch_inc_release() - atomic increment with release ordering
	1142	+ * @v: pointer to atomic_t
	1143	+ *
	1144	+ * Atomically updates @v to (@v + 1) with release ordering.
	1145	+ *
	1146	+ * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
	1147	+ *
	1148	+ * Return: The original value of @v.
	1149	+ */
589	1150	static __always_inline int
590		−arch_atomic_fetch_inc_release(atomic_t *v)
	1151	+raw_atomic_fetch_inc_release(atomic_t *v)
591	1152	{
	1153	+#if defined(arch_atomic_fetch_inc_release)
	1154	+ return arch_atomic_fetch_inc_release(v);
	1155	+#elif defined(arch_atomic_fetch_inc_relaxed)
592	1156	__atomic_release_fence();
593	1157	return arch_atomic_fetch_inc_relaxed(v);
594		−}
595		−#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
	1158	+#elif defined(arch_atomic_fetch_inc)
	1159	+ return arch_atomic_fetch_inc(v);
	1160	+#else
	1161	+ return raw_atomic_fetch_add_release(1, v);
596	1162	#endif
	1163	+}
597	1164
598		−#ifndef arch_atomic_fetch_inc
	1165	+/**
	1166	+ * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
	1167	+ * @v: pointer to atomic_t
	1168	+ *
	1169	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	1170	+ *
	1171	+ * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
	1172	+ *
	1173	+ * Return: The original value of @v.
	1174	+ */
599	1175	static __always_inline int
600		−arch_atomic_fetch_inc(atomic_t *v)
	1176	+raw_atomic_fetch_inc_relaxed(atomic_t *v)
601	1177	{
602		− int ret;
603		− __atomic_pre_full_fence();
604		− ret = arch_atomic_fetch_inc_relaxed(v);
605		− __atomic_post_full_fence();
606		− return ret;
607		−}
608		−#define arch_atomic_fetch_inc arch_atomic_fetch_inc
	1178	+#if defined(arch_atomic_fetch_inc_relaxed)
	1179	+ return arch_atomic_fetch_inc_relaxed(v);
	1180	+#elif defined(arch_atomic_fetch_inc)
	1181	+ return arch_atomic_fetch_inc(v);
	1182	+#else
	1183	+ return raw_atomic_fetch_add_relaxed(1, v);
609	1184	#endif
610		−
611		−#endif /* arch_atomic_fetch_inc_relaxed */
612		−
613		−#ifndef arch_atomic_dec
614		−static __always_inline void
615		−arch_atomic_dec(atomic_t *v)
616		−{
617		− arch_atomic_sub(1, v);
618	1185	}
619		−#define arch_atomic_dec arch_atomic_dec
620		−#endif
621		−
622		−#ifndef arch_atomic_dec_return_relaxed
623		−#ifdef arch_atomic_dec_return
624		−#define arch_atomic_dec_return_acquire arch_atomic_dec_return
625		−#define arch_atomic_dec_return_release arch_atomic_dec_return
626		−#define arch_atomic_dec_return_relaxed arch_atomic_dec_return
627		−#endif /* arch_atomic_dec_return */
628	1186
629		−#ifndef arch_atomic_dec_return
630		−static __always_inline int
631		−arch_atomic_dec_return(atomic_t *v)
	1187	+/**
	1188	+ * raw_atomic_dec() - atomic decrement with relaxed ordering
	1189	+ * @v: pointer to atomic_t
	1190	+ *
	1191	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	1192	+ *
	1193	+ * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
	1194	+ *
	1195	+ * Return: Nothing.
	1196	+ */
	1197	+static __always_inline void
	1198	+raw_atomic_dec(atomic_t *v)
632	1199	{
633		− return arch_atomic_sub_return(1, v);
634		−}
635		−#define arch_atomic_dec_return arch_atomic_dec_return
	1200	+#if defined(arch_atomic_dec)
	1201	+ arch_atomic_dec(v);
	1202	+#else
	1203	+ raw_atomic_sub(1, v);
636	1204	#endif
637		−
638		−#ifndef arch_atomic_dec_return_acquire
639		−static __always_inline int
640		−arch_atomic_dec_return_acquire(atomic_t *v)
641		−{
642		− return arch_atomic_sub_return_acquire(1, v);
643	1205	}
644		−#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
645		−#endif
646	1206
647		−#ifndef arch_atomic_dec_return_release
	1207	+/**
	1208	+ * raw_atomic_dec_return() - atomic decrement with full ordering
	1209	+ * @v: pointer to atomic_t
	1210	+ *
	1211	+ * Atomically updates @v to (@v - 1) with full ordering.
	1212	+ *
	1213	+ * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
	1214	+ *
	1215	+ * Return: The updated value of @v.
	1216	+ */
648	1217	static __always_inline int
649		−arch_atomic_dec_return_release(atomic_t *v)
	1218	+raw_atomic_dec_return(atomic_t *v)
650	1219	{
651		− return arch_atomic_sub_return_release(1, v);
652		−}
653		−#define arch_atomic_dec_return_release arch_atomic_dec_return_release
	1220	+#if defined(arch_atomic_dec_return)
	1221	+ return arch_atomic_dec_return(v);
	1222	+#elif defined(arch_atomic_dec_return_relaxed)
	1223	+ int ret;
	1224	+ __atomic_pre_full_fence();
	1225	+ ret = arch_atomic_dec_return_relaxed(v);
	1226	+ __atomic_post_full_fence();
	1227	+ return ret;
	1228	+#else
	1229	+ return raw_atomic_sub_return(1, v);
654	1230	#endif
655		−
656		−#ifndef arch_atomic_dec_return_relaxed
657		−static __always_inline int
658		−arch_atomic_dec_return_relaxed(atomic_t *v)
659		−{
660		− return arch_atomic_sub_return_relaxed(1, v);
661	1231	}
662		−#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
663		−#endif
664	1232
665		−#else /* arch_atomic_dec_return_relaxed */
666		−
667		−#ifndef arch_atomic_dec_return_acquire
	1233	+/**
	1234	+ * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
	1235	+ * @v: pointer to atomic_t
	1236	+ *
	1237	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	1238	+ *
	1239	+ * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
	1240	+ *
	1241	+ * Return: The updated value of @v.
	1242	+ */
668	1243	static __always_inline int
669		−arch_atomic_dec_return_acquire(atomic_t *v)
	1244	+raw_atomic_dec_return_acquire(atomic_t *v)
670	1245	{
	1246	+#if defined(arch_atomic_dec_return_acquire)
	1247	+ return arch_atomic_dec_return_acquire(v);
	1248	+#elif defined(arch_atomic_dec_return_relaxed)
671	1249	int ret = arch_atomic_dec_return_relaxed(v);
672	1250	__atomic_acquire_fence();
673	1251	return ret;
674		−}
675		−#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
	1252	+#elif defined(arch_atomic_dec_return)
	1253	+ return arch_atomic_dec_return(v);
	1254	+#else
	1255	+ return raw_atomic_sub_return_acquire(1, v);
676	1256	#endif
	1257	+}
677	1258
678		−#ifndef arch_atomic_dec_return_release
	1259	+/**
	1260	+ * raw_atomic_dec_return_release() - atomic decrement with release ordering
	1261	+ * @v: pointer to atomic_t
	1262	+ *
	1263	+ * Atomically updates @v to (@v - 1) with release ordering.
	1264	+ *
	1265	+ * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
	1266	+ *
	1267	+ * Return: The updated value of @v.
	1268	+ */
679	1269	static __always_inline int
680		−arch_atomic_dec_return_release(atomic_t *v)
	1270	+raw_atomic_dec_return_release(atomic_t *v)
681	1271	{
	1272	+#if defined(arch_atomic_dec_return_release)
	1273	+ return arch_atomic_dec_return_release(v);
	1274	+#elif defined(arch_atomic_dec_return_relaxed)
682	1275	__atomic_release_fence();
683	1276	return arch_atomic_dec_return_relaxed(v);
	1277	+#elif defined(arch_atomic_dec_return)
	1278	+ return arch_atomic_dec_return(v);
	1279	+#else
	1280	+ return raw_atomic_sub_return_release(1, v);
	1281	+#endif
684	1282	}
685		−#define arch_atomic_dec_return_release arch_atomic_dec_return_release
	1283	+
	1284	+/**
	1285	+ * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
	1286	+ * @v: pointer to atomic_t
	1287	+ *
	1288	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	1289	+ *
	1290	+ * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
	1291	+ *
	1292	+ * Return: The updated value of @v.
	1293	+ */
	1294	+static __always_inline int
	1295	+raw_atomic_dec_return_relaxed(atomic_t *v)
	1296	+{
	1297	+#if defined(arch_atomic_dec_return_relaxed)
	1298	+ return arch_atomic_dec_return_relaxed(v);
	1299	+#elif defined(arch_atomic_dec_return)
	1300	+ return arch_atomic_dec_return(v);
	1301	+#else
	1302	+ return raw_atomic_sub_return_relaxed(1, v);
686	1303	#endif
	1304	+}
687	1305
688		−#ifndef arch_atomic_dec_return
	1306	+/**
	1307	+ * raw_atomic_fetch_dec() - atomic decrement with full ordering
	1308	+ * @v: pointer to atomic_t
	1309	+ *
	1310	+ * Atomically updates @v to (@v - 1) with full ordering.
	1311	+ *
	1312	+ * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
	1313	+ *
	1314	+ * Return: The original value of @v.
	1315	+ */
689	1316	static __always_inline int
690		−arch_atomic_dec_return(atomic_t *v)
	1317	+raw_atomic_fetch_dec(atomic_t *v)
691	1318	{
	1319	+#if defined(arch_atomic_fetch_dec)
	1320	+ return arch_atomic_fetch_dec(v);
	1321	+#elif defined(arch_atomic_fetch_dec_relaxed)
692	1322	int ret;
693	1323	__atomic_pre_full_fence();
694		− ret = arch_atomic_dec_return_relaxed(v);
	1324	+ ret = arch_atomic_fetch_dec_relaxed(v);
695	1325	__atomic_post_full_fence();
696	1326	return ret;
697		−}
698		−#define arch_atomic_dec_return arch_atomic_dec_return
	1327	+#else
	1328	+ return raw_atomic_fetch_sub(1, v);
699	1329	#endif
700		−
701		−#endif /* arch_atomic_dec_return_relaxed */
702		−
703		−#ifndef arch_atomic_fetch_dec_relaxed
704		−#ifdef arch_atomic_fetch_dec
705		−#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
706		−#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
707		−#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
708		−#endif /* arch_atomic_fetch_dec */
709		−
710		−#ifndef arch_atomic_fetch_dec
711		−static __always_inline int
712		−arch_atomic_fetch_dec(atomic_t *v)
713		−{
714		− return arch_atomic_fetch_sub(1, v);
715	1330	}
716		−#define arch_atomic_fetch_dec arch_atomic_fetch_dec
717		−#endif
718	1331
719		−#ifndef arch_atomic_fetch_dec_acquire
	1332	+/**
	1333	+ * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
	1334	+ * @v: pointer to atomic_t
	1335	+ *
	1336	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	1337	+ *
	1338	+ * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
	1339	+ *
	1340	+ * Return: The original value of @v.
	1341	+ */
720	1342	static __always_inline int
721		−arch_atomic_fetch_dec_acquire(atomic_t *v)
	1343	+raw_atomic_fetch_dec_acquire(atomic_t *v)
722	1344	{
723		− return arch_atomic_fetch_sub_acquire(1, v);
724		−}
725		−#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
	1345	+#if defined(arch_atomic_fetch_dec_acquire)
	1346	+ return arch_atomic_fetch_dec_acquire(v);
	1347	+#elif defined(arch_atomic_fetch_dec_relaxed)
	1348	+ int ret = arch_atomic_fetch_dec_relaxed(v);
	1349	+ __atomic_acquire_fence();
	1350	+ return ret;
	1351	+#elif defined(arch_atomic_fetch_dec)
	1352	+ return arch_atomic_fetch_dec(v);
	1353	+#else
	1354	+ return raw_atomic_fetch_sub_acquire(1, v);
726	1355	#endif
727		−
728		−#ifndef arch_atomic_fetch_dec_release
729		−static __always_inline int
730		−arch_atomic_fetch_dec_release(atomic_t *v)
731		−{
732		− return arch_atomic_fetch_sub_release(1, v);
733	1356	}
734		−#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
735		−#endif
736	1357
737		−#ifndef arch_atomic_fetch_dec_relaxed
	1358	+/**
	1359	+ * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
	1360	+ * @v: pointer to atomic_t
	1361	+ *
	1362	+ * Atomically updates @v to (@v - 1) with release ordering.
	1363	+ *
	1364	+ * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
	1365	+ *
	1366	+ * Return: The original value of @v.
	1367	+ */
738	1368	static __always_inline int
739		−arch_atomic_fetch_dec_relaxed(atomic_t *v)
	1369	+raw_atomic_fetch_dec_release(atomic_t *v)
740	1370	{
741		− return arch_atomic_fetch_sub_relaxed(1, v);
742		−}
743		−#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
	1371	+#if defined(arch_atomic_fetch_dec_release)
	1372	+ return arch_atomic_fetch_dec_release(v);
	1373	+#elif defined(arch_atomic_fetch_dec_relaxed)
	1374	+ __atomic_release_fence();
	1375	+ return arch_atomic_fetch_dec_relaxed(v);
	1376	+#elif defined(arch_atomic_fetch_dec)
	1377	+ return arch_atomic_fetch_dec(v);
	1378	+#else
	1379	+ return raw_atomic_fetch_sub_release(1, v);
744	1380	#endif
	1381	+}
745	1382
746		−#else /* arch_atomic_fetch_dec_relaxed */
747		−
748		−#ifndef arch_atomic_fetch_dec_acquire
	1383	+/**
	1384	+ * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
	1385	+ * @v: pointer to atomic_t
	1386	+ *
	1387	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	1388	+ *
	1389	+ * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
	1390	+ *
	1391	+ * Return: The original value of @v.
	1392	+ */
749	1393	static __always_inline int
750		−arch_atomic_fetch_dec_acquire(atomic_t *v)
	1394	+raw_atomic_fetch_dec_relaxed(atomic_t *v)
751	1395	{
752		− int ret = arch_atomic_fetch_dec_relaxed(v);
753		− __atomic_acquire_fence();
754		− return ret;
755		−}
756		−#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
	1396	+#if defined(arch_atomic_fetch_dec_relaxed)
	1397	+ return arch_atomic_fetch_dec_relaxed(v);
	1398	+#elif defined(arch_atomic_fetch_dec)
	1399	+ return arch_atomic_fetch_dec(v);
	1400	+#else
	1401	+ return raw_atomic_fetch_sub_relaxed(1, v);
757	1402	#endif
	1403	+}
758	1404
759		−#ifndef arch_atomic_fetch_dec_release
760		−static __always_inline int
761		−arch_atomic_fetch_dec_release(atomic_t *v)
	1405	+/**
	1406	+ * raw_atomic_and() - atomic bitwise AND with relaxed ordering
	1407	+ * @i: int value
	1408	+ * @v: pointer to atomic_t
	1409	+ *
	1410	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	1411	+ *
	1412	+ * Safe to use in noinstr code; prefer atomic_and() elsewhere.
	1413	+ *
	1414	+ * Return: Nothing.
	1415	+ */
	1416	+static __always_inline void
	1417	+raw_atomic_and(int i, atomic_t *v)
762	1418	{
763		− __atomic_release_fence();
764		− return arch_atomic_fetch_dec_relaxed(v);
	1419	+ arch_atomic_and(i, v);
765	1420	}
766		−#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
767		−#endif
768	1421
769		−#ifndef arch_atomic_fetch_dec
	1422	+/**
	1423	+ * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
	1424	+ * @i: int value
	1425	+ * @v: pointer to atomic_t
	1426	+ *
	1427	+ * Atomically updates @v to (@v & @i) with full ordering.
	1428	+ *
	1429	+ * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
	1430	+ *
	1431	+ * Return: The original value of @v.
	1432	+ */
770	1433	static __always_inline int
771		−arch_atomic_fetch_dec(atomic_t *v)
	1434	+raw_atomic_fetch_and(int i, atomic_t *v)
772	1435	{
	1436	+#if defined(arch_atomic_fetch_and)
	1437	+ return arch_atomic_fetch_and(i, v);
	1438	+#elif defined(arch_atomic_fetch_and_relaxed)
773	1439	int ret;
774	1440	__atomic_pre_full_fence();
775		− ret = arch_atomic_fetch_dec_relaxed(v);
	1441	+ ret = arch_atomic_fetch_and_relaxed(i, v);
776	1442	__atomic_post_full_fence();
777	1443	return ret;
778		−}
779		−#define arch_atomic_fetch_dec arch_atomic_fetch_dec
	1444	+#else
	1445	+#error "Unable to define raw_atomic_fetch_and"
780	1446	#endif
	1447	+}
781	1448
782		−#endif /* arch_atomic_fetch_dec_relaxed */
783		−
784		−#ifndef arch_atomic_fetch_and_relaxed
785		−#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
786		−#define arch_atomic_fetch_and_release arch_atomic_fetch_and
787		−#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
788		−#else /* arch_atomic_fetch_and_relaxed */
789		−
790		−#ifndef arch_atomic_fetch_and_acquire
	1449	+/**
	1450	+ * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
	1451	+ * @i: int value
	1452	+ * @v: pointer to atomic_t
	1453	+ *
	1454	+ * Atomically updates @v to (@v & @i) with acquire ordering.
	1455	+ *
	1456	+ * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
	1457	+ *
	1458	+ * Return: The original value of @v.
	1459	+ */
791	1460	static __always_inline int
792		−arch_atomic_fetch_and_acquire(int i, atomic_t *v)
	1461	+raw_atomic_fetch_and_acquire(int i, atomic_t *v)
793	1462	{
	1463	+#if defined(arch_atomic_fetch_and_acquire)
	1464	+ return arch_atomic_fetch_and_acquire(i, v);
	1465	+#elif defined(arch_atomic_fetch_and_relaxed)
794	1466	int ret = arch_atomic_fetch_and_relaxed(i, v);
795	1467	__atomic_acquire_fence();
796	1468	return ret;
797		−}
798		−#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
	1469	+#elif defined(arch_atomic_fetch_and)
	1470	+ return arch_atomic_fetch_and(i, v);
	1471	+#else
	1472	+#error "Unable to define raw_atomic_fetch_and_acquire"
799	1473	#endif
	1474	+}
800	1475
801		−#ifndef arch_atomic_fetch_and_release
	1476	+/**
	1477	+ * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
	1478	+ * @i: int value
	1479	+ * @v: pointer to atomic_t
	1480	+ *
	1481	+ * Atomically updates @v to (@v & @i) with release ordering.
	1482	+ *
	1483	+ * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
	1484	+ *
	1485	+ * Return: The original value of @v.
	1486	+ */
802	1487	static __always_inline int
803		−arch_atomic_fetch_and_release(int i, atomic_t *v)
	1488	+raw_atomic_fetch_and_release(int i, atomic_t *v)
804	1489	{
	1490	+#if defined(arch_atomic_fetch_and_release)
	1491	+ return arch_atomic_fetch_and_release(i, v);
	1492	+#elif defined(arch_atomic_fetch_and_relaxed)
805	1493	__atomic_release_fence();
806	1494	return arch_atomic_fetch_and_relaxed(i, v);
807		−}
808		−#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
	1495	+#elif defined(arch_atomic_fetch_and)
	1496	+ return arch_atomic_fetch_and(i, v);
	1497	+#else
	1498	+#error "Unable to define raw_atomic_fetch_and_release"
809	1499	#endif
	1500	+}
810	1501
811		−#ifndef arch_atomic_fetch_and
	1502	+/**
	1503	+ * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
	1504	+ * @i: int value
	1505	+ * @v: pointer to atomic_t
	1506	+ *
	1507	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	1508	+ *
	1509	+ * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
	1510	+ *
	1511	+ * Return: The original value of @v.
	1512	+ */
812	1513	static __always_inline int
813		−arch_atomic_fetch_and(int i, atomic_t *v)
	1514	+raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
814	1515	{
815		− int ret;
816		− __atomic_pre_full_fence();
817		− ret = arch_atomic_fetch_and_relaxed(i, v);
818		− __atomic_post_full_fence();
819		− return ret;
820		−}
821		−#define arch_atomic_fetch_and arch_atomic_fetch_and
	1516	+#if defined(arch_atomic_fetch_and_relaxed)
	1517	+ return arch_atomic_fetch_and_relaxed(i, v);
	1518	+#elif defined(arch_atomic_fetch_and)
	1519	+ return arch_atomic_fetch_and(i, v);
	1520	+#else
	1521	+#error "Unable to define raw_atomic_fetch_and_relaxed"
822	1522	#endif
	1523	+}
823	1524
824		−#endif /* arch_atomic_fetch_and_relaxed */
825		−
826		−#ifndef arch_atomic_andnot
	1525	+/**
	1526	+ * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
	1527	+ * @i: int value
	1528	+ * @v: pointer to atomic_t
	1529	+ *
	1530	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	1531	+ *
	1532	+ * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
	1533	+ *
	1534	+ * Return: Nothing.
	1535	+ */
827	1536	static __always_inline void
828		−arch_atomic_andnot(int i, atomic_t *v)
	1537	+raw_atomic_andnot(int i, atomic_t *v)
829	1538	{
830		− arch_atomic_and(~i, v);
831		−}
832		−#define arch_atomic_andnot arch_atomic_andnot
	1539	+#if defined(arch_atomic_andnot)
	1540	+ arch_atomic_andnot(i, v);
	1541	+#else
	1542	+ raw_atomic_and(~i, v);
833	1543	#endif
834		−
835		−#ifndef arch_atomic_fetch_andnot_relaxed
836		−#ifdef arch_atomic_fetch_andnot
837		−#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
838		−#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
839		−#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
840		−#endif /* arch_atomic_fetch_andnot */
841		−
842		−#ifndef arch_atomic_fetch_andnot
843		−static __always_inline int
844		−arch_atomic_fetch_andnot(int i, atomic_t *v)
845		−{
846		− return arch_atomic_fetch_and(~i, v);
847	1544	}
848		−#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
849		−#endif
850	1545
851		−#ifndef arch_atomic_fetch_andnot_acquire
852		−static __always_inline int
853		−arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
854		−{
855		− return arch_atomic_fetch_and_acquire(~i, v);
856		−}
857		−#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
858		−#endif
859		−
860		−#ifndef arch_atomic_fetch_andnot_release
	1546	+/**
	1547	+ * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
	1548	+ * @i: int value
	1549	+ * @v: pointer to atomic_t
	1550	+ *
	1551	+ * Atomically updates @v to (@v & ~@i) with full ordering.
	1552	+ *
	1553	+ * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
	1554	+ *
	1555	+ * Return: The original value of @v.
	1556	+ */
861	1557	static __always_inline int
862		−arch_atomic_fetch_andnot_release(int i, atomic_t *v)
	1558	+raw_atomic_fetch_andnot(int i, atomic_t *v)
863	1559	{
864		− return arch_atomic_fetch_and_release(~i, v);
865		−}
866		−#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
	1560	+#if defined(arch_atomic_fetch_andnot)
	1561	+ return arch_atomic_fetch_andnot(i, v);
	1562	+#elif defined(arch_atomic_fetch_andnot_relaxed)
	1563	+ int ret;
	1564	+ __atomic_pre_full_fence();
	1565	+ ret = arch_atomic_fetch_andnot_relaxed(i, v);
	1566	+ __atomic_post_full_fence();
	1567	+ return ret;
	1568	+#else
	1569	+ return raw_atomic_fetch_and(~i, v);
867	1570	#endif
868		−
869		−#ifndef arch_atomic_fetch_andnot_relaxed
870		−static __always_inline int
871		−arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
872		−{
873		− return arch_atomic_fetch_and_relaxed(~i, v);
874	1571	}
875		−#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
876		−#endif
877		−
878		−#else /* arch_atomic_fetch_andnot_relaxed */
879	1572
880		−#ifndef arch_atomic_fetch_andnot_acquire
	1573	+/**
	1574	+ * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
	1575	+ * @i: int value
	1576	+ * @v: pointer to atomic_t
	1577	+ *
	1578	+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
	1579	+ *
	1580	+ * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
	1581	+ *
	1582	+ * Return: The original value of @v.
	1583	+ */
881	1584	static __always_inline int
882		−arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
	1585	+raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
883	1586	{
	1587	+#if defined(arch_atomic_fetch_andnot_acquire)
	1588	+ return arch_atomic_fetch_andnot_acquire(i, v);
	1589	+#elif defined(arch_atomic_fetch_andnot_relaxed)
884	1590	int ret = arch_atomic_fetch_andnot_relaxed(i, v);
885	1591	__atomic_acquire_fence();
886	1592	return ret;
887		−}
888		−#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
	1593	+#elif defined(arch_atomic_fetch_andnot)
	1594	+ return arch_atomic_fetch_andnot(i, v);
	1595	+#else
	1596	+ return raw_atomic_fetch_and_acquire(~i, v);
889	1597	#endif
	1598	+}
890	1599
891		−#ifndef arch_atomic_fetch_andnot_release
	1600	+/**
	1601	+ * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
	1602	+ * @i: int value
	1603	+ * @v: pointer to atomic_t
	1604	+ *
	1605	+ * Atomically updates @v to (@v & ~@i) with release ordering.
	1606	+ *
	1607	+ * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
	1608	+ *
	1609	+ * Return: The original value of @v.
	1610	+ */
892	1611	static __always_inline int
893		−arch_atomic_fetch_andnot_release(int i, atomic_t *v)
	1612	+raw_atomic_fetch_andnot_release(int i, atomic_t *v)
894	1613	{
	1614	+#if defined(arch_atomic_fetch_andnot_release)
	1615	+ return arch_atomic_fetch_andnot_release(i, v);
	1616	+#elif defined(arch_atomic_fetch_andnot_relaxed)
895	1617	__atomic_release_fence();
896	1618	return arch_atomic_fetch_andnot_relaxed(i, v);
	1619	+#elif defined(arch_atomic_fetch_andnot)
	1620	+ return arch_atomic_fetch_andnot(i, v);
	1621	+#else
	1622	+ return raw_atomic_fetch_and_release(~i, v);
	1623	+#endif
897	1624	}
898		−#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
	1625	+
	1626	+/**
	1627	+ * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
	1628	+ * @i: int value
	1629	+ * @v: pointer to atomic_t
	1630	+ *
	1631	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	1632	+ *
	1633	+ * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
	1634	+ *
	1635	+ * Return: The original value of @v.
	1636	+ */
	1637	+static __always_inline int
	1638	+raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
	1639	+{
	1640	+#if defined(arch_atomic_fetch_andnot_relaxed)
	1641	+ return arch_atomic_fetch_andnot_relaxed(i, v);
	1642	+#elif defined(arch_atomic_fetch_andnot)
	1643	+ return arch_atomic_fetch_andnot(i, v);
	1644	+#else
	1645	+ return raw_atomic_fetch_and_relaxed(~i, v);
899	1646	#endif
	1647	+}
	1648	+
	1649	+/**
	1650	+ * raw_atomic_or() - atomic bitwise OR with relaxed ordering
	1651	+ * @i: int value
	1652	+ * @v: pointer to atomic_t
	1653	+ *
	1654	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	1655	+ *
	1656	+ * Safe to use in noinstr code; prefer atomic_or() elsewhere.
	1657	+ *
	1658	+ * Return: Nothing.
	1659	+ */
	1660	+static __always_inline void
	1661	+raw_atomic_or(int i, atomic_t *v)
	1662	+{
	1663	+ arch_atomic_or(i, v);
	1664	+}
900	1665
901		−#ifndef arch_atomic_fetch_andnot
	1666	+/**
	1667	+ * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
	1668	+ * @i: int value
	1669	+ * @v: pointer to atomic_t
	1670	+ *
	1671	+ * Atomically updates @v to (@v \| @i) with full ordering.
	1672	+ *
	1673	+ * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
	1674	+ *
	1675	+ * Return: The original value of @v.
	1676	+ */
902	1677	static __always_inline int
903		−arch_atomic_fetch_andnot(int i, atomic_t *v)
	1678	+raw_atomic_fetch_or(int i, atomic_t *v)
904	1679	{
	1680	+#if defined(arch_atomic_fetch_or)
	1681	+ return arch_atomic_fetch_or(i, v);
	1682	+#elif defined(arch_atomic_fetch_or_relaxed)
905	1683	int ret;
906	1684	__atomic_pre_full_fence();
907		− ret = arch_atomic_fetch_andnot_relaxed(i, v);
	1685	+ ret = arch_atomic_fetch_or_relaxed(i, v);
908	1686	__atomic_post_full_fence();
909	1687	return ret;
910		−}
911		−#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
	1688	+#else
	1689	+#error "Unable to define raw_atomic_fetch_or"
912	1690	#endif
	1691	+}
913	1692
914		−#endif /* arch_atomic_fetch_andnot_relaxed */
915		−
916		−#ifndef arch_atomic_fetch_or_relaxed
917		−#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
918		−#define arch_atomic_fetch_or_release arch_atomic_fetch_or
919		−#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
920		−#else /* arch_atomic_fetch_or_relaxed */
921		−
922		−#ifndef arch_atomic_fetch_or_acquire
	1693	+/**
	1694	+ * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
	1695	+ * @i: int value
	1696	+ * @v: pointer to atomic_t
	1697	+ *
	1698	+ * Atomically updates @v to (@v \| @i) with acquire ordering.
	1699	+ *
	1700	+ * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
	1701	+ *
	1702	+ * Return: The original value of @v.
	1703	+ */
923	1704	static __always_inline int
924		−arch_atomic_fetch_or_acquire(int i, atomic_t *v)
	1705	+raw_atomic_fetch_or_acquire(int i, atomic_t *v)
925	1706	{
	1707	+#if defined(arch_atomic_fetch_or_acquire)
	1708	+ return arch_atomic_fetch_or_acquire(i, v);
	1709	+#elif defined(arch_atomic_fetch_or_relaxed)
926	1710	int ret = arch_atomic_fetch_or_relaxed(i, v);
927	1711	__atomic_acquire_fence();
928	1712	return ret;
929		−}
930		−#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
	1713	+#elif defined(arch_atomic_fetch_or)
	1714	+ return arch_atomic_fetch_or(i, v);
	1715	+#else
	1716	+#error "Unable to define raw_atomic_fetch_or_acquire"
931	1717	#endif
	1718	+}
932	1719
933		−#ifndef arch_atomic_fetch_or_release
	1720	+/**
	1721	+ * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
	1722	+ * @i: int value
	1723	+ * @v: pointer to atomic_t
	1724	+ *
	1725	+ * Atomically updates @v to (@v \| @i) with release ordering.
	1726	+ *
	1727	+ * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
	1728	+ *
	1729	+ * Return: The original value of @v.
	1730	+ */
934	1731	static __always_inline int
935		−arch_atomic_fetch_or_release(int i, atomic_t *v)
	1732	+raw_atomic_fetch_or_release(int i, atomic_t *v)
936	1733	{
	1734	+#if defined(arch_atomic_fetch_or_release)
	1735	+ return arch_atomic_fetch_or_release(i, v);
	1736	+#elif defined(arch_atomic_fetch_or_relaxed)
937	1737	__atomic_release_fence();
938	1738	return arch_atomic_fetch_or_relaxed(i, v);
	1739	+#elif defined(arch_atomic_fetch_or)
	1740	+ return arch_atomic_fetch_or(i, v);
	1741	+#else
	1742	+#error "Unable to define raw_atomic_fetch_or_release"
	1743	+#endif
939	1744	}
940		−#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
	1745	+
	1746	+/**
	1747	+ * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
	1748	+ * @i: int value
	1749	+ * @v: pointer to atomic_t
	1750	+ *
	1751	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	1752	+ *
	1753	+ * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
	1754	+ *
	1755	+ * Return: The original value of @v.
	1756	+ */
	1757	+static __always_inline int
	1758	+raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
	1759	+{
	1760	+#if defined(arch_atomic_fetch_or_relaxed)
	1761	+ return arch_atomic_fetch_or_relaxed(i, v);
	1762	+#elif defined(arch_atomic_fetch_or)
	1763	+ return arch_atomic_fetch_or(i, v);
	1764	+#else
	1765	+#error "Unable to define raw_atomic_fetch_or_relaxed"
941	1766	#endif
	1767	+}
942	1768
943		−#ifndef arch_atomic_fetch_or
	1769	+/**
	1770	+ * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
	1771	+ * @i: int value
	1772	+ * @v: pointer to atomic_t
	1773	+ *
	1774	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	1775	+ *
	1776	+ * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
	1777	+ *
	1778	+ * Return: Nothing.
	1779	+ */
	1780	+static __always_inline void
	1781	+raw_atomic_xor(int i, atomic_t *v)
	1782	+{
	1783	+ arch_atomic_xor(i, v);
	1784	+}
	1785	+
	1786	+/**
	1787	+ * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
	1788	+ * @i: int value
	1789	+ * @v: pointer to atomic_t
	1790	+ *
	1791	+ * Atomically updates @v to (@v ^ @i) with full ordering.
	1792	+ *
	1793	+ * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
	1794	+ *
	1795	+ * Return: The original value of @v.
	1796	+ */
944	1797	static __always_inline int
945		−arch_atomic_fetch_or(int i, atomic_t *v)
	1798	+raw_atomic_fetch_xor(int i, atomic_t *v)
946	1799	{
	1800	+#if defined(arch_atomic_fetch_xor)
	1801	+ return arch_atomic_fetch_xor(i, v);
	1802	+#elif defined(arch_atomic_fetch_xor_relaxed)
947	1803	int ret;
948	1804	__atomic_pre_full_fence();
949		− ret = arch_atomic_fetch_or_relaxed(i, v);
	1805	+ ret = arch_atomic_fetch_xor_relaxed(i, v);
950	1806	__atomic_post_full_fence();
951	1807	return ret;
952		−}
953		−#define arch_atomic_fetch_or arch_atomic_fetch_or
	1808	+#else
	1809	+#error "Unable to define raw_atomic_fetch_xor"
954	1810	#endif
	1811	+}
955	1812
956		−#endif /* arch_atomic_fetch_or_relaxed */
957		−
958		−#ifndef arch_atomic_fetch_xor_relaxed
959		−#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
960		−#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
961		−#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
962		−#else /* arch_atomic_fetch_xor_relaxed */
963		−
964		−#ifndef arch_atomic_fetch_xor_acquire
	1813	+/**
	1814	+ * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
	1815	+ * @i: int value
	1816	+ * @v: pointer to atomic_t
	1817	+ *
	1818	+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
	1819	+ *
	1820	+ * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
	1821	+ *
	1822	+ * Return: The original value of @v.
	1823	+ */
965	1824	static __always_inline int
966		−arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
	1825	+raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
967	1826	{
	1827	+#if defined(arch_atomic_fetch_xor_acquire)
	1828	+ return arch_atomic_fetch_xor_acquire(i, v);
	1829	+#elif defined(arch_atomic_fetch_xor_relaxed)
968	1830	int ret = arch_atomic_fetch_xor_relaxed(i, v);
969	1831	__atomic_acquire_fence();
970	1832	return ret;
971		−}
972		−#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
	1833	+#elif defined(arch_atomic_fetch_xor)
	1834	+ return arch_atomic_fetch_xor(i, v);
	1835	+#else
	1836	+#error "Unable to define raw_atomic_fetch_xor_acquire"
973	1837	#endif
	1838	+}
974	1839
975		−#ifndef arch_atomic_fetch_xor_release
	1840	+/**
	1841	+ * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
	1842	+ * @i: int value
	1843	+ * @v: pointer to atomic_t
	1844	+ *
	1845	+ * Atomically updates @v to (@v ^ @i) with release ordering.
	1846	+ *
	1847	+ * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
	1848	+ *
	1849	+ * Return: The original value of @v.
	1850	+ */
976	1851	static __always_inline int
977		−arch_atomic_fetch_xor_release(int i, atomic_t *v)
	1852	+raw_atomic_fetch_xor_release(int i, atomic_t *v)
978	1853	{
	1854	+#if defined(arch_atomic_fetch_xor_release)
	1855	+ return arch_atomic_fetch_xor_release(i, v);
	1856	+#elif defined(arch_atomic_fetch_xor_relaxed)
979	1857	__atomic_release_fence();
980	1858	return arch_atomic_fetch_xor_relaxed(i, v);
	1859	+#elif defined(arch_atomic_fetch_xor)
	1860	+ return arch_atomic_fetch_xor(i, v);
	1861	+#else
	1862	+#error "Unable to define raw_atomic_fetch_xor_release"
	1863	+#endif
981	1864	}
982		−#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
	1865	+
	1866	+/**
	1867	+ * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
	1868	+ * @i: int value
	1869	+ * @v: pointer to atomic_t
	1870	+ *
	1871	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	1872	+ *
	1873	+ * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
	1874	+ *
	1875	+ * Return: The original value of @v.
	1876	+ */
	1877	+static __always_inline int
	1878	+raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
	1879	+{
	1880	+#if defined(arch_atomic_fetch_xor_relaxed)
	1881	+ return arch_atomic_fetch_xor_relaxed(i, v);
	1882	+#elif defined(arch_atomic_fetch_xor)
	1883	+ return arch_atomic_fetch_xor(i, v);
	1884	+#else
	1885	+#error "Unable to define raw_atomic_fetch_xor_relaxed"
983	1886	#endif
	1887	+}
984	1888
985		−#ifndef arch_atomic_fetch_xor
	1889	+/**
	1890	+ * raw_atomic_xchg() - atomic exchange with full ordering
	1891	+ * @v: pointer to atomic_t
	1892	+ * @new: int value to assign
	1893	+ *
	1894	+ * Atomically updates @v to @new with full ordering.
	1895	+ *
	1896	+ * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
	1897	+ *
	1898	+ * Return: The original value of @v.
	1899	+ */
986	1900	static __always_inline int
987		−arch_atomic_fetch_xor(int i, atomic_t *v)
	1901	+raw_atomic_xchg(atomic_t *v, int new)
988	1902	{
	1903	+#if defined(arch_atomic_xchg)
	1904	+ return arch_atomic_xchg(v, new);
	1905	+#elif defined(arch_atomic_xchg_relaxed)
989	1906	int ret;
990	1907	__atomic_pre_full_fence();
991		− ret = arch_atomic_fetch_xor_relaxed(i, v);
	1908	+ ret = arch_atomic_xchg_relaxed(v, new);
992	1909	__atomic_post_full_fence();
993	1910	return ret;
994		−}
995		−#define arch_atomic_fetch_xor arch_atomic_fetch_xor
	1911	+#else
	1912	+ return raw_xchg(&v->counter, new);
996	1913	#endif
	1914	+}
997	1915
998		−#endif /* arch_atomic_fetch_xor_relaxed */
999		−
1000		−#ifndef arch_atomic_xchg_relaxed
1001		−#define arch_atomic_xchg_acquire arch_atomic_xchg
1002		−#define arch_atomic_xchg_release arch_atomic_xchg
1003		−#define arch_atomic_xchg_relaxed arch_atomic_xchg
1004		−#else /* arch_atomic_xchg_relaxed */
1005		−
1006		−#ifndef arch_atomic_xchg_acquire
	1916	+/**
	1917	+ * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
	1918	+ * @v: pointer to atomic_t
	1919	+ * @new: int value to assign
	1920	+ *
	1921	+ * Atomically updates @v to @new with acquire ordering.
	1922	+ *
	1923	+ * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
	1924	+ *
	1925	+ * Return: The original value of @v.
	1926	+ */
1007	1927	static __always_inline int
1008		−arch_atomic_xchg_acquire(atomic_t *v, int i)
	1928	+raw_atomic_xchg_acquire(atomic_t *v, int new)
1009	1929	{
1010		− int ret = arch_atomic_xchg_relaxed(v, i);
	1930	+#if defined(arch_atomic_xchg_acquire)
	1931	+ return arch_atomic_xchg_acquire(v, new);
	1932	+#elif defined(arch_atomic_xchg_relaxed)
	1933	+ int ret = arch_atomic_xchg_relaxed(v, new);
1011	1934	__atomic_acquire_fence();
1012	1935	return ret;
1013		−}
1014		−#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
	1936	+#elif defined(arch_atomic_xchg)
	1937	+ return arch_atomic_xchg(v, new);
	1938	+#else
	1939	+ return raw_xchg_acquire(&v->counter, new);
1015	1940	#endif
	1941	+}
1016	1942
1017		−#ifndef arch_atomic_xchg_release
	1943	+/**
	1944	+ * raw_atomic_xchg_release() - atomic exchange with release ordering
	1945	+ * @v: pointer to atomic_t
	1946	+ * @new: int value to assign
	1947	+ *
	1948	+ * Atomically updates @v to @new with release ordering.
	1949	+ *
	1950	+ * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
	1951	+ *
	1952	+ * Return: The original value of @v.
	1953	+ */
1018	1954	static __always_inline int
1019		−arch_atomic_xchg_release(atomic_t *v, int i)
	1955	+raw_atomic_xchg_release(atomic_t *v, int new)
1020	1956	{
	1957	+#if defined(arch_atomic_xchg_release)
	1958	+ return arch_atomic_xchg_release(v, new);
	1959	+#elif defined(arch_atomic_xchg_relaxed)
1021	1960	__atomic_release_fence();
1022		− return arch_atomic_xchg_relaxed(v, i);
	1961	+ return arch_atomic_xchg_relaxed(v, new);
	1962	+#elif defined(arch_atomic_xchg)
	1963	+ return arch_atomic_xchg(v, new);
	1964	+#else
	1965	+ return raw_xchg_release(&v->counter, new);
	1966	+#endif
1023	1967	}
1024		−#define arch_atomic_xchg_release arch_atomic_xchg_release
	1968	+
	1969	+/**
	1970	+ * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
	1971	+ * @v: pointer to atomic_t
	1972	+ * @new: int value to assign
	1973	+ *
	1974	+ * Atomically updates @v to @new with relaxed ordering.
	1975	+ *
	1976	+ * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
	1977	+ *
	1978	+ * Return: The original value of @v.
	1979	+ */
	1980	+static __always_inline int
	1981	+raw_atomic_xchg_relaxed(atomic_t *v, int new)
	1982	+{
	1983	+#if defined(arch_atomic_xchg_relaxed)
	1984	+ return arch_atomic_xchg_relaxed(v, new);
	1985	+#elif defined(arch_atomic_xchg)
	1986	+ return arch_atomic_xchg(v, new);
	1987	+#else
	1988	+ return raw_xchg_relaxed(&v->counter, new);
1025	1989	#endif
	1990	+}
1026	1991
1027		−#ifndef arch_atomic_xchg
	1992	+/**
	1993	+ * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
	1994	+ * @v: pointer to atomic_t
	1995	+ * @old: int value to compare with
	1996	+ * @new: int value to assign
	1997	+ *
	1998	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	1999	+ *
	2000	+ * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
	2001	+ *
	2002	+ * Return: The original value of @v.
	2003	+ */
1028	2004	static __always_inline int
1029		−arch_atomic_xchg(atomic_t *v, int i)
	2005	+raw_atomic_cmpxchg(atomic_t *v, int old, int new)
1030	2006	{
	2007	+#if defined(arch_atomic_cmpxchg)
	2008	+ return arch_atomic_cmpxchg(v, old, new);
	2009	+#elif defined(arch_atomic_cmpxchg_relaxed)
1031	2010	int ret;
1032	2011	__atomic_pre_full_fence();
1033		− ret = arch_atomic_xchg_relaxed(v, i);
	2012	+ ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1034	2013	__atomic_post_full_fence();
1035	2014	return ret;
1036		−}
1037		−#define arch_atomic_xchg arch_atomic_xchg
1038		−#endif
1039		−
1040		−#endif /* arch_atomic_xchg_relaxed */
1041		−
1042		−#ifndef arch_atomic_cmpxchg_relaxed
1043		−#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
1044		−#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
1045		−#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
1046		−#else /* arch_atomic_cmpxchg_relaxed */
	2015	+#else
	2016	+ return raw_cmpxchg(&v->counter, old, new);
	2017	+#endif
	2018	+}
1047	2019
1048		−#ifndef arch_atomic_cmpxchg_acquire
	2020	+/**
	2021	+ * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	2022	+ * @v: pointer to atomic_t
	2023	+ * @old: int value to compare with
	2024	+ * @new: int value to assign
	2025	+ *
	2026	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	2027	+ *
	2028	+ * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
	2029	+ *
	2030	+ * Return: The original value of @v.
	2031	+ */
1049	2032	static __always_inline int
1050		−arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
	2033	+raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1051	2034	{
	2035	+#if defined(arch_atomic_cmpxchg_acquire)
	2036	+ return arch_atomic_cmpxchg_acquire(v, old, new);
	2037	+#elif defined(arch_atomic_cmpxchg_relaxed)
1052	2038	int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1053	2039	__atomic_acquire_fence();
1054	2040	return ret;
1055		−}
1056		−#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
	2041	+#elif defined(arch_atomic_cmpxchg)
	2042	+ return arch_atomic_cmpxchg(v, old, new);
	2043	+#else
	2044	+ return raw_cmpxchg_acquire(&v->counter, old, new);
1057	2045	#endif
	2046	+}
1058	2047
1059		−#ifndef arch_atomic_cmpxchg_release
	2048	+/**
	2049	+ * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
	2050	+ * @v: pointer to atomic_t
	2051	+ * @old: int value to compare with
	2052	+ * @new: int value to assign
	2053	+ *
	2054	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	2055	+ *
	2056	+ * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
	2057	+ *
	2058	+ * Return: The original value of @v.
	2059	+ */
1060	2060	static __always_inline int
1061		−arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
	2061	+raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
1062	2062	{
	2063	+#if defined(arch_atomic_cmpxchg_release)
	2064	+ return arch_atomic_cmpxchg_release(v, old, new);
	2065	+#elif defined(arch_atomic_cmpxchg_relaxed)
1063	2066	__atomic_release_fence();
1064	2067	return arch_atomic_cmpxchg_relaxed(v, old, new);
1065		−}
1066		−#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
	2068	+#elif defined(arch_atomic_cmpxchg)
	2069	+ return arch_atomic_cmpxchg(v, old, new);
	2070	+#else
	2071	+ return raw_cmpxchg_release(&v->counter, old, new);
1067	2072	#endif
	2073	+}
1068	2074
1069		−#ifndef arch_atomic_cmpxchg
	2075	+/**
	2076	+ * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	2077	+ * @v: pointer to atomic_t
	2078	+ * @old: int value to compare with
	2079	+ * @new: int value to assign
	2080	+ *
	2081	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	2082	+ *
	2083	+ * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
	2084	+ *
	2085	+ * Return: The original value of @v.
	2086	+ */
1070	2087	static __always_inline int
1071		−arch_atomic_cmpxchg(atomic_t *v, int old, int new)
	2088	+raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
1072	2089	{
1073		− int ret;
1074		− __atomic_pre_full_fence();
1075		− ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1076		− __atomic_post_full_fence();
1077		− return ret;
1078		−}
1079		−#define arch_atomic_cmpxchg arch_atomic_cmpxchg
	2090	+#if defined(arch_atomic_cmpxchg_relaxed)
	2091	+ return arch_atomic_cmpxchg_relaxed(v, old, new);
	2092	+#elif defined(arch_atomic_cmpxchg)
	2093	+ return arch_atomic_cmpxchg(v, old, new);
	2094	+#else
	2095	+ return raw_cmpxchg_relaxed(&v->counter, old, new);
1080	2096	#endif
	2097	+}
1081	2098
1082		−#endif /* arch_atomic_cmpxchg_relaxed */
1083		−
1084		−#ifndef arch_atomic_try_cmpxchg_relaxed
1085		−#ifdef arch_atomic_try_cmpxchg
1086		−#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
1087		−#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
1088		−#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
1089		−#endif /* arch_atomic_try_cmpxchg */
1090		−
1091		−#ifndef arch_atomic_try_cmpxchg
	2099	+/**
	2100	+ * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
	2101	+ * @v: pointer to atomic_t
	2102	+ * @old: pointer to int value to compare with
	2103	+ * @new: int value to assign
	2104	+ *
	2105	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	2106	+ * Otherwise, updates @old to the current value of @v.
	2107	+ *
	2108	+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
	2109	+ *
	2110	+ * Return: @true if the exchange occured, @false otherwise.
	2111	+ */
1092	2112	static __always_inline bool
1093		−arch_atomic_try_cmpxchg(atomic_t v, int old, int new)
	2113	+raw_atomic_try_cmpxchg(atomic_t v, int old, int new)
1094	2114	{
	2115	+#if defined(arch_atomic_try_cmpxchg)
	2116	+ return arch_atomic_try_cmpxchg(v, old, new);
	2117	+#elif defined(arch_atomic_try_cmpxchg_relaxed)
	2118	+ bool ret;
	2119	+ __atomic_pre_full_fence();
	2120	+ ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
	2121	+ __atomic_post_full_fence();
	2122	+ return ret;
	2123	+#else
1095	2124	int r, o = *old;
1096		− r = arch_atomic_cmpxchg(v, o, new);
	2125	+ r = raw_atomic_cmpxchg(v, o, new);
1097	2126	if (unlikely(r != o))
1098	2127	*old = r;
1099	2128	return likely(r == o);
1100		−}
1101		−#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1102	2129	#endif
	2130	+}
1103	2131
1104		−#ifndef arch_atomic_try_cmpxchg_acquire
	2132	+/**
	2133	+ * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	2134	+ * @v: pointer to atomic_t
	2135	+ * @old: pointer to int value to compare with
	2136	+ * @new: int value to assign
	2137	+ *
	2138	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	2139	+ * Otherwise, updates @old to the current value of @v.
	2140	+ *
	2141	+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
	2142	+ *
	2143	+ * Return: @true if the exchange occured, @false otherwise.
	2144	+ */
1105	2145	static __always_inline bool
1106		−arch_atomic_try_cmpxchg_acquire(atomic_t v, int old, int new)
	2146	+raw_atomic_try_cmpxchg_acquire(atomic_t v, int old, int new)
1107	2147	{
	2148	+#if defined(arch_atomic_try_cmpxchg_acquire)
	2149	+ return arch_atomic_try_cmpxchg_acquire(v, old, new);
	2150	+#elif defined(arch_atomic_try_cmpxchg_relaxed)
	2151	+ bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
	2152	+ __atomic_acquire_fence();
	2153	+ return ret;
	2154	+#elif defined(arch_atomic_try_cmpxchg)
	2155	+ return arch_atomic_try_cmpxchg(v, old, new);
	2156	+#else
1108	2157	int r, o = *old;
1109		− r = arch_atomic_cmpxchg_acquire(v, o, new);
	2158	+ r = raw_atomic_cmpxchg_acquire(v, o, new);
1110	2159	if (unlikely(r != o))
1111	2160	*old = r;
1112	2161	return likely(r == o);
1113		−}
1114		−#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1115	2162	#endif
	2163	+}
1116	2164
1117		−#ifndef arch_atomic_try_cmpxchg_release
	2165	+/**
	2166	+ * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
	2167	+ * @v: pointer to atomic_t
	2168	+ * @old: pointer to int value to compare with
	2169	+ * @new: int value to assign
	2170	+ *
	2171	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	2172	+ * Otherwise, updates @old to the current value of @v.
	2173	+ *
	2174	+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
	2175	+ *
	2176	+ * Return: @true if the exchange occured, @false otherwise.
	2177	+ */
1118	2178	static __always_inline bool
1119		−arch_atomic_try_cmpxchg_release(atomic_t v, int old, int new)
	2179	+raw_atomic_try_cmpxchg_release(atomic_t v, int old, int new)
1120	2180	{
	2181	+#if defined(arch_atomic_try_cmpxchg_release)
	2182	+ return arch_atomic_try_cmpxchg_release(v, old, new);
	2183	+#elif defined(arch_atomic_try_cmpxchg_relaxed)
	2184	+ __atomic_release_fence();
	2185	+ return arch_atomic_try_cmpxchg_relaxed(v, old, new);
	2186	+#elif defined(arch_atomic_try_cmpxchg)
	2187	+ return arch_atomic_try_cmpxchg(v, old, new);
	2188	+#else
1121	2189	int r, o = *old;
1122		− r = arch_atomic_cmpxchg_release(v, o, new);
	2190	+ r = raw_atomic_cmpxchg_release(v, o, new);
1123	2191	if (unlikely(r != o))
1124	2192	*old = r;
1125	2193	return likely(r == o);
1126		−}
1127		−#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1128	2194	#endif
	2195	+}
1129	2196
1130		−#ifndef arch_atomic_try_cmpxchg_relaxed
	2197	+/**
	2198	+ * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	2199	+ * @v: pointer to atomic_t
	2200	+ * @old: pointer to int value to compare with
	2201	+ * @new: int value to assign
	2202	+ *
	2203	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	2204	+ * Otherwise, updates @old to the current value of @v.
	2205	+ *
	2206	+ * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
	2207	+ *
	2208	+ * Return: @true if the exchange occured, @false otherwise.
	2209	+ */
1131	2210	static __always_inline bool
1132		−arch_atomic_try_cmpxchg_relaxed(atomic_t v, int old, int new)
	2211	+raw_atomic_try_cmpxchg_relaxed(atomic_t v, int old, int new)
1133	2212	{
	2213	+#if defined(arch_atomic_try_cmpxchg_relaxed)
	2214	+ return arch_atomic_try_cmpxchg_relaxed(v, old, new);
	2215	+#elif defined(arch_atomic_try_cmpxchg)
	2216	+ return arch_atomic_try_cmpxchg(v, old, new);
	2217	+#else
1134	2218	int r, o = *old;
1135		− r = arch_atomic_cmpxchg_relaxed(v, o, new);
	2219	+ r = raw_atomic_cmpxchg_relaxed(v, o, new);
1136	2220	if (unlikely(r != o))
1137	2221	*old = r;
1138	2222	return likely(r == o);
1139		−}
1140		−#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1141		−#endif
1142		−
1143		−#else /* arch_atomic_try_cmpxchg_relaxed */
1144		−
1145		−#ifndef arch_atomic_try_cmpxchg_acquire
1146		−static __always_inline bool
1147		−arch_atomic_try_cmpxchg_acquire(atomic_t v, int old, int new)
1148		−{
1149		− bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1150		− __atomic_acquire_fence();
1151		− return ret;
1152		−}
1153		−#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1154		−#endif
1155		−
1156		−#ifndef arch_atomic_try_cmpxchg_release
1157		−static __always_inline bool
1158		−arch_atomic_try_cmpxchg_release(atomic_t v, int old, int new)
1159		−{
1160		− __atomic_release_fence();
1161		− return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1162		−}
1163		−#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1164	2223	#endif
1165		−
1166		−#ifndef arch_atomic_try_cmpxchg
1167		−static __always_inline bool
1168		−arch_atomic_try_cmpxchg(atomic_t v, int old, int new)
1169		−{
1170		− bool ret;
1171		− __atomic_pre_full_fence();
1172		− ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1173		− __atomic_post_full_fence();
1174		− return ret;
1175	2224	}
1176		−#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1177		−#endif
1178	2225
1179		−#endif /* arch_atomic_try_cmpxchg_relaxed */
1180		−
1181		−#ifndef arch_atomic_sub_and_test
1182	2226	/**
1183		− * arch_atomic_sub_and_test - subtract value from variable and test result
1184		− * @i: integer value to subtract
1185		− * @v: pointer of type atomic_t
	2227	+ * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
	2228	+ * @i: int value to add
	2229	+ * @v: pointer to atomic_t
	2230	+ *
	2231	+ * Atomically updates @v to (@v - @i) with full ordering.
1186	2232	*
1187		− * Atomically subtracts @i from @v and returns
1188		− * true if the result is zero, or false for all
1189		− * other cases.
	2233	+ * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
	2234	+ *
	2235	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
1190	2236	*/
1191	2237	static __always_inline bool
1192		−arch_atomic_sub_and_test(int i, atomic_t *v)
	2238	+raw_atomic_sub_and_test(int i, atomic_t *v)
1193	2239	{
1194		− return arch_atomic_sub_return(i, v) == 0;
1195		−}
1196		−#define arch_atomic_sub_and_test arch_atomic_sub_and_test
	2240	+#if defined(arch_atomic_sub_and_test)
	2241	+ return arch_atomic_sub_and_test(i, v);
	2242	+#else
	2243	+ return raw_atomic_sub_return(i, v) == 0;
1197	2244	#endif
	2245	+}
1198	2246
1199		−#ifndef arch_atomic_dec_and_test
1200	2247	/**
1201		− * arch_atomic_dec_and_test - decrement and test
1202		− * @v: pointer of type atomic_t
	2248	+ * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
	2249	+ * @v: pointer to atomic_t
	2250	+ *
	2251	+ * Atomically updates @v to (@v - 1) with full ordering.
1203	2252	*
1204		− * Atomically decrements @v by 1 and
1205		− * returns true if the result is 0, or false for all other
1206		− * cases.
	2253	+ * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
	2254	+ *
	2255	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
1207	2256	*/
1208	2257	static __always_inline bool
1209		−arch_atomic_dec_and_test(atomic_t *v)
	2258	+raw_atomic_dec_and_test(atomic_t *v)
1210	2259	{
1211		− return arch_atomic_dec_return(v) == 0;
1212		−}
1213		−#define arch_atomic_dec_and_test arch_atomic_dec_and_test
	2260	+#if defined(arch_atomic_dec_and_test)
	2261	+ return arch_atomic_dec_and_test(v);
	2262	+#else
	2263	+ return raw_atomic_dec_return(v) == 0;
1214	2264	#endif
	2265	+}
1215	2266
1216		−#ifndef arch_atomic_inc_and_test
1217	2267	/**
1218		− * arch_atomic_inc_and_test - increment and test
1219		− * @v: pointer of type atomic_t
	2268	+ * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
	2269	+ * @v: pointer to atomic_t
	2270	+ *
	2271	+ * Atomically updates @v to (@v + 1) with full ordering.
1220	2272	*
1221		− * Atomically increments @v by 1
1222		− * and returns true if the result is zero, or false for all
1223		− * other cases.
	2273	+ * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
	2274	+ *
	2275	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
1224	2276	*/
1225	2277	static __always_inline bool
1226		−arch_atomic_inc_and_test(atomic_t *v)
	2278	+raw_atomic_inc_and_test(atomic_t *v)
1227	2279	{
1228		− return arch_atomic_inc_return(v) == 0;
1229		−}
1230		−#define arch_atomic_inc_and_test arch_atomic_inc_and_test
	2280	+#if defined(arch_atomic_inc_and_test)
	2281	+ return arch_atomic_inc_and_test(v);
	2282	+#else
	2283	+ return raw_atomic_inc_return(v) == 0;
1231	2284	#endif
	2285	+}
1232	2286
1233		−#ifndef arch_atomic_add_negative_relaxed
1234		−#ifdef arch_atomic_add_negative
1235		−#define arch_atomic_add_negative_acquire arch_atomic_add_negative
1236		−#define arch_atomic_add_negative_release arch_atomic_add_negative
1237		−#define arch_atomic_add_negative_relaxed arch_atomic_add_negative
1238		−#endif /* arch_atomic_add_negative */
1239		−
1240		−#ifndef arch_atomic_add_negative
1241	2287	/**
1242		− * arch_atomic_add_negative - Add and test if negative
1243		− * @i: integer value to add
1244		− * @v: pointer of type atomic_t
	2288	+ * raw_atomic_add_negative() - atomic add and test if negative with full ordering
	2289	+ * @i: int value to add
	2290	+ * @v: pointer to atomic_t
1245	2291	*
1246		− * Atomically adds @i to @v and returns true if the result is negative,
1247		− * or false when the result is greater than or equal to zero.
	2292	+ * Atomically updates @v to (@v + @i) with full ordering.
	2293	+ *
	2294	+ * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
	2295	+ *
	2296	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
1248	2297	*/
1249	2298	static __always_inline bool
1250		−arch_atomic_add_negative(int i, atomic_t *v)
	2299	+raw_atomic_add_negative(int i, atomic_t *v)
1251	2300	{
1252		− return arch_atomic_add_return(i, v) < 0;
1253		−}
1254		−#define arch_atomic_add_negative arch_atomic_add_negative
	2301	+#if defined(arch_atomic_add_negative)
	2302	+ return arch_atomic_add_negative(i, v);
	2303	+#elif defined(arch_atomic_add_negative_relaxed)
	2304	+ bool ret;
	2305	+ __atomic_pre_full_fence();
	2306	+ ret = arch_atomic_add_negative_relaxed(i, v);
	2307	+ __atomic_post_full_fence();
	2308	+ return ret;
	2309	+#else
	2310	+ return raw_atomic_add_return(i, v) < 0;
1255	2311	#endif
	2312	+}
1256	2313
1257		−#ifndef arch_atomic_add_negative_acquire
1258	2314	/**
1259		− * arch_atomic_add_negative_acquire - Add and test if negative
1260		− * @i: integer value to add
1261		− * @v: pointer of type atomic_t
	2315	+ * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
	2316	+ * @i: int value to add
	2317	+ * @v: pointer to atomic_t
	2318	+ *
	2319	+ * Atomically updates @v to (@v + @i) with acquire ordering.
1262	2320	*
1263		− * Atomically adds @i to @v and returns true if the result is negative,
1264		− * or false when the result is greater than or equal to zero.
	2321	+ * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
	2322	+ *
	2323	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
1265	2324	*/
1266	2325	static __always_inline bool
1267		−arch_atomic_add_negative_acquire(int i, atomic_t *v)
	2326	+raw_atomic_add_negative_acquire(int i, atomic_t *v)
1268	2327	{
1269		− return arch_atomic_add_return_acquire(i, v) < 0;
1270		−}
1271		−#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
	2328	+#if defined(arch_atomic_add_negative_acquire)
	2329	+ return arch_atomic_add_negative_acquire(i, v);
	2330	+#elif defined(arch_atomic_add_negative_relaxed)
	2331	+ bool ret = arch_atomic_add_negative_relaxed(i, v);
	2332	+ __atomic_acquire_fence();
	2333	+ return ret;
	2334	+#elif defined(arch_atomic_add_negative)
	2335	+ return arch_atomic_add_negative(i, v);
	2336	+#else
	2337	+ return raw_atomic_add_return_acquire(i, v) < 0;
1272	2338	#endif
	2339	+}
1273	2340
1274		−#ifndef arch_atomic_add_negative_release
1275	2341	/**
1276		− * arch_atomic_add_negative_release - Add and test if negative
1277		− * @i: integer value to add
1278		− * @v: pointer of type atomic_t
	2342	+ * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
	2343	+ * @i: int value to add
	2344	+ * @v: pointer to atomic_t
1279	2345	*
1280		− * Atomically adds @i to @v and returns true if the result is negative,
1281		− * or false when the result is greater than or equal to zero.
	2346	+ * Atomically updates @v to (@v + @i) with release ordering.
	2347	+ *
	2348	+ * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
	2349	+ *
	2350	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
1282	2351	*/
1283	2352	static __always_inline bool
1284		−arch_atomic_add_negative_release(int i, atomic_t *v)
	2353	+raw_atomic_add_negative_release(int i, atomic_t *v)
1285	2354	{
1286		− return arch_atomic_add_return_release(i, v) < 0;
1287		−}
1288		−#define arch_atomic_add_negative_release arch_atomic_add_negative_release
	2355	+#if defined(arch_atomic_add_negative_release)
	2356	+ return arch_atomic_add_negative_release(i, v);
	2357	+#elif defined(arch_atomic_add_negative_relaxed)
	2358	+ __atomic_release_fence();
	2359	+ return arch_atomic_add_negative_relaxed(i, v);
	2360	+#elif defined(arch_atomic_add_negative)
	2361	+ return arch_atomic_add_negative(i, v);
	2362	+#else
	2363	+ return raw_atomic_add_return_release(i, v) < 0;
1289	2364	#endif
	2365	+}
1290	2366
1291		−#ifndef arch_atomic_add_negative_relaxed
1292	2367	/**
1293		− * arch_atomic_add_negative_relaxed - Add and test if negative
1294		− * @i: integer value to add
1295		− * @v: pointer of type atomic_t
	2368	+ * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
	2369	+ * @i: int value to add
	2370	+ * @v: pointer to atomic_t
	2371	+ *
	2372	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	2373	+ *
	2374	+ * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
1296	2375	*
1297		− * Atomically adds @i to @v and returns true if the result is negative,
1298		− * or false when the result is greater than or equal to zero.
	2376	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
1299	2377	*/
1300	2378	static __always_inline bool
1301		−arch_atomic_add_negative_relaxed(int i, atomic_t *v)
1302		−{
1303		− return arch_atomic_add_return_relaxed(i, v) < 0;
1304		−}
1305		−#define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed
1306		−#endif
1307		−
1308		−#else /* arch_atomic_add_negative_relaxed */
1309		−
1310		−#ifndef arch_atomic_add_negative_acquire
1311		−static __always_inline bool
1312		−arch_atomic_add_negative_acquire(int i, atomic_t *v)
1313		−{
1314		− bool ret = arch_atomic_add_negative_relaxed(i, v);
1315		− __atomic_acquire_fence();
1316		− return ret;
1317		−}
1318		−#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
1319		−#endif
1320		−
1321		−#ifndef arch_atomic_add_negative_release
1322		−static __always_inline bool
1323		−arch_atomic_add_negative_release(int i, atomic_t *v)
	2379	+raw_atomic_add_negative_relaxed(int i, atomic_t *v)
1324	2380	{
1325		− __atomic_release_fence();
	2381	+#if defined(arch_atomic_add_negative_relaxed)
1326	2382	return arch_atomic_add_negative_relaxed(i, v);
1327		−}
1328		−#define arch_atomic_add_negative_release arch_atomic_add_negative_release
	2383	+#elif defined(arch_atomic_add_negative)
	2384	+ return arch_atomic_add_negative(i, v);
	2385	+#else
	2386	+ return raw_atomic_add_return_relaxed(i, v) < 0;
1329	2387	#endif
1330		−
1331		−#ifndef arch_atomic_add_negative
1332		−static __always_inline bool
1333		−arch_atomic_add_negative(int i, atomic_t *v)
1334		−{
1335		− bool ret;
1336		− __atomic_pre_full_fence();
1337		− ret = arch_atomic_add_negative_relaxed(i, v);
1338		− __atomic_post_full_fence();
1339		− return ret;
1340	2388	}
1341		−#define arch_atomic_add_negative arch_atomic_add_negative
1342		−#endif
1343		−
1344		−#endif /* arch_atomic_add_negative_relaxed */
1345	2389
1346		−#ifndef arch_atomic_fetch_add_unless
1347	2390	/**
1348		− * arch_atomic_fetch_add_unless - add unless the number is already a given value
1349		− * @v: pointer of type atomic_t
1350		− * @a: the amount to add to v...
1351		− * @u: ...unless v is equal to u.
	2391	+ * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
	2392	+ * @v: pointer to atomic_t
	2393	+ * @a: int value to add
	2394	+ * @u: int value to compare with
	2395	+ *
	2396	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	2397	+ *
	2398	+ * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
1352	2399	*
1353		− * Atomically adds @a to @v, so long as @v was not already @u.
1354		− * Returns original value of @v
	2400	+ * Return: The original value of @v.
1355	2401	*/
1356	2402	static __always_inline int
1357		−arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
	2403	+raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1358	2404	{
1359		− int c = arch_atomic_read(v);
	2405	+#if defined(arch_atomic_fetch_add_unless)
	2406	+ return arch_atomic_fetch_add_unless(v, a, u);
	2407	+#else
	2408	+ int c = raw_atomic_read(v);
1360	2409
1361	2410	do {
1362	2411	if (unlikely(c == u))
1363	2412	break;
1364		− } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
	2413	+ } while (!raw_atomic_try_cmpxchg(v, &c, c + a));
1365	2414
1366	2415	return c;
1367		−}
1368		−#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1369	2416	#endif
	2417	+}
1370	2418
1371		−#ifndef arch_atomic_add_unless
1372	2419	/**
1373		− * arch_atomic_add_unless - add unless the number is already a given value
1374		− * @v: pointer of type atomic_t
1375		− * @a: the amount to add to v...
1376		− * @u: ...unless v is equal to u.
	2420	+ * raw_atomic_add_unless() - atomic add unless value with full ordering
	2421	+ * @v: pointer to atomic_t
	2422	+ * @a: int value to add
	2423	+ * @u: int value to compare with
	2424	+ *
	2425	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	2426	+ *
	2427	+ * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
1377	2428	*
1378		− * Atomically adds @a to @v, if @v was not already @u.
1379		− * Returns true if the addition was done.
	2429	+ * Return: @true if @v was updated, @false otherwise.
1380	2430	*/
1381	2431	static __always_inline bool
1382		−arch_atomic_add_unless(atomic_t *v, int a, int u)
	2432	+raw_atomic_add_unless(atomic_t *v, int a, int u)
1383	2433	{
1384		− return arch_atomic_fetch_add_unless(v, a, u) != u;
1385		−}
1386		−#define arch_atomic_add_unless arch_atomic_add_unless
	2434	+#if defined(arch_atomic_add_unless)
	2435	+ return arch_atomic_add_unless(v, a, u);
	2436	+#else
	2437	+ return raw_atomic_fetch_add_unless(v, a, u) != u;
1387	2438	#endif
	2439	+}
1388	2440
1389		−#ifndef arch_atomic_inc_not_zero
1390	2441	/**
1391		− * arch_atomic_inc_not_zero - increment unless the number is zero
1392		− * @v: pointer of type atomic_t
	2442	+ * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
	2443	+ * @v: pointer to atomic_t
	2444	+ *
	2445	+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	2446	+ *
	2447	+ * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
1393	2448	*
1394		− * Atomically increments @v by 1, if @v is non-zero.
1395		− * Returns true if the increment was done.
	2449	+ * Return: @true if @v was updated, @false otherwise.
1396	2450	*/
1397	2451	static __always_inline bool
1398		−arch_atomic_inc_not_zero(atomic_t *v)
	2452	+raw_atomic_inc_not_zero(atomic_t *v)
1399	2453	{
1400		− return arch_atomic_add_unless(v, 1, 0);
1401		−}
1402		−#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
	2454	+#if defined(arch_atomic_inc_not_zero)
	2455	+ return arch_atomic_inc_not_zero(v);
	2456	+#else
	2457	+ return raw_atomic_add_unless(v, 1, 0);
1403	2458	#endif
	2459	+}
1404	2460
1405		−#ifndef arch_atomic_inc_unless_negative
	2461	+/**
	2462	+ * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
	2463	+ * @v: pointer to atomic_t
	2464	+ *
	2465	+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	2466	+ *
	2467	+ * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
	2468	+ *
	2469	+ * Return: @true if @v was updated, @false otherwise.
	2470	+ */
1406	2471	static __always_inline bool
1407		−arch_atomic_inc_unless_negative(atomic_t *v)
	2472	+raw_atomic_inc_unless_negative(atomic_t *v)
1408	2473	{
1409		− int c = arch_atomic_read(v);
	2474	+#if defined(arch_atomic_inc_unless_negative)
	2475	+ return arch_atomic_inc_unless_negative(v);
	2476	+#else
	2477	+ int c = raw_atomic_read(v);
1410	2478
1411	2479	do {
1412	2480	if (unlikely(c < 0))
1413	2481	return false;
1414		− } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
	2482	+ } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
1415	2483
1416	2484	return true;
1417		−}
1418		−#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1419	2485	#endif
	2486	+}
1420	2487
1421		−#ifndef arch_atomic_dec_unless_positive
	2488	+/**
	2489	+ * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
	2490	+ * @v: pointer to atomic_t
	2491	+ *
	2492	+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	2493	+ *
	2494	+ * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
	2495	+ *
	2496	+ * Return: @true if @v was updated, @false otherwise.
	2497	+ */
1422	2498	static __always_inline bool
1423		−arch_atomic_dec_unless_positive(atomic_t *v)
	2499	+raw_atomic_dec_unless_positive(atomic_t *v)
1424	2500	{
1425		− int c = arch_atomic_read(v);
	2501	+#if defined(arch_atomic_dec_unless_positive)
	2502	+ return arch_atomic_dec_unless_positive(v);
	2503	+#else
	2504	+ int c = raw_atomic_read(v);
1426	2505
1427	2506	do {
1428	2507	if (unlikely(c > 0))
1429	2508	return false;
1430		− } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
	2509	+ } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
1431	2510
1432	2511	return true;
1433		−}
1434		−#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1435	2512	#endif
	2513	+}
1436	2514
1437		−#ifndef arch_atomic_dec_if_positive
	2515	+/**
	2516	+ * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
	2517	+ * @v: pointer to atomic_t
	2518	+ *
	2519	+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	2520	+ *
	2521	+ * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
	2522	+ *
	2523	+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
	2524	+ */
1438	2525	static __always_inline int
1439		−arch_atomic_dec_if_positive(atomic_t *v)
	2526	+raw_atomic_dec_if_positive(atomic_t *v)
1440	2527	{
1441		− int dec, c = arch_atomic_read(v);
	2528	+#if defined(arch_atomic_dec_if_positive)
	2529	+ return arch_atomic_dec_if_positive(v);
	2530	+#else
	2531	+ int dec, c = raw_atomic_read(v);
1442	2532
1443	2533	do {
1444	2534	dec = c - 1;
1445	2535	if (unlikely(dec < 0))
1446	2536	break;
1447		− } while (!arch_atomic_try_cmpxchg(v, &c, dec));
	2537	+ } while (!raw_atomic_try_cmpxchg(v, &c, dec));
1448	2538
1449	2539	return dec;
1450		−}
1451		−#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1452	2540	#endif
	2541	+}
1453	2542
1454	2543	#ifdef CONFIG_GENERIC_ATOMIC64
1455	2544	#include <asm-generic/atomic64.h>
1456	2545	#endif
1457	2546
1458		−#ifndef arch_atomic64_read_acquire
	2547	+/**
	2548	+ * raw_atomic64_read() - atomic load with relaxed ordering
	2549	+ * @v: pointer to atomic64_t
	2550	+ *
	2551	+ * Atomically loads the value of @v with relaxed ordering.
	2552	+ *
	2553	+ * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
	2554	+ *
	2555	+ * Return: The value loaded from @v.
	2556	+ */
	2557	+static __always_inline s64
	2558	+raw_atomic64_read(const atomic64_t *v)
	2559	+{
	2560	+ return arch_atomic64_read(v);
	2561	+}
	2562	+
	2563	+/**
	2564	+ * raw_atomic64_read_acquire() - atomic load with acquire ordering
	2565	+ * @v: pointer to atomic64_t
	2566	+ *
	2567	+ * Atomically loads the value of @v with acquire ordering.
	2568	+ *
	2569	+ * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
	2570	+ *
	2571	+ * Return: The value loaded from @v.
	2572	+ */
1459	2573	static __always_inline s64
1460		−arch_atomic64_read_acquire(const atomic64_t *v)
	2574	+raw_atomic64_read_acquire(const atomic64_t *v)
1461	2575	{
	2576	+#if defined(arch_atomic64_read_acquire)
	2577	+ return arch_atomic64_read_acquire(v);
	2578	+#elif defined(arch_atomic64_read)
	2579	+ return arch_atomic64_read(v);
	2580	+#else
1462	2581	s64 ret;
1463	2582
1464	2583	if (__native_word(atomic64_t)) {
1465	2584	ret = smp_load_acquire(&(v)->counter);
1466	2585	} else {
1467		− ret = arch_atomic64_read(v);
	2586	+ ret = raw_atomic64_read(v);
1468	2587	__atomic_acquire_fence();
1469	2588	}
1470	2589
1471	2590	return ret;
1472		−}
1473		−#define arch_atomic64_read_acquire arch_atomic64_read_acquire
1474	2591	#endif
	2592	+}
	2593	+
	2594	+/**
	2595	+ * raw_atomic64_set() - atomic set with relaxed ordering
	2596	+ * @v: pointer to atomic64_t
	2597	+ * @i: s64 value to assign
	2598	+ *
	2599	+ * Atomically sets @v to @i with relaxed ordering.
	2600	+ *
	2601	+ * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
	2602	+ *
	2603	+ * Return: Nothing.
	2604	+ */
	2605	+static __always_inline void
	2606	+raw_atomic64_set(atomic64_t *v, s64 i)
	2607	+{
	2608	+ arch_atomic64_set(v, i);
	2609	+}
1475	2610
1476		−#ifndef arch_atomic64_set_release
	2611	+/**
	2612	+ * raw_atomic64_set_release() - atomic set with release ordering
	2613	+ * @v: pointer to atomic64_t
	2614	+ * @i: s64 value to assign
	2615	+ *
	2616	+ * Atomically sets @v to @i with release ordering.
	2617	+ *
	2618	+ * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
	2619	+ *
	2620	+ * Return: Nothing.
	2621	+ */
1477	2622	static __always_inline void
1478		−arch_atomic64_set_release(atomic64_t *v, s64 i)
	2623	+raw_atomic64_set_release(atomic64_t *v, s64 i)
1479	2624	{
	2625	+#if defined(arch_atomic64_set_release)
	2626	+ arch_atomic64_set_release(v, i);
	2627	+#elif defined(arch_atomic64_set)
	2628	+ arch_atomic64_set(v, i);
	2629	+#else
1480	2630	if (__native_word(atomic64_t)) {
1481	2631	smp_store_release(&(v)->counter, i);
1482	2632	} else {
1483	2633	__atomic_release_fence();
1484		− arch_atomic64_set(v, i);
	2634	+ raw_atomic64_set(v, i);
1485	2635	}
1486		−}
1487		−#define arch_atomic64_set_release arch_atomic64_set_release
1488	2636	#endif
	2637	+}
	2638	+
	2639	+/**
	2640	+ * raw_atomic64_add() - atomic add with relaxed ordering
	2641	+ * @i: s64 value to add
	2642	+ * @v: pointer to atomic64_t
	2643	+ *
	2644	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	2645	+ *
	2646	+ * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
	2647	+ *
	2648	+ * Return: Nothing.
	2649	+ */
	2650	+static __always_inline void
	2651	+raw_atomic64_add(s64 i, atomic64_t *v)
	2652	+{
	2653	+ arch_atomic64_add(i, v);
	2654	+}
1489	2655
1490		−#ifndef arch_atomic64_add_return_relaxed
1491		−#define arch_atomic64_add_return_acquire arch_atomic64_add_return
1492		−#define arch_atomic64_add_return_release arch_atomic64_add_return
1493		−#define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1494		−#else /* arch_atomic64_add_return_relaxed */
	2656	+/**
	2657	+ * raw_atomic64_add_return() - atomic add with full ordering
	2658	+ * @i: s64 value to add
	2659	+ * @v: pointer to atomic64_t
	2660	+ *
	2661	+ * Atomically updates @v to (@v + @i) with full ordering.
	2662	+ *
	2663	+ * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
	2664	+ *
	2665	+ * Return: The updated value of @v.
	2666	+ */
	2667	+static __always_inline s64
	2668	+raw_atomic64_add_return(s64 i, atomic64_t *v)
	2669	+{
	2670	+#if defined(arch_atomic64_add_return)
	2671	+ return arch_atomic64_add_return(i, v);
	2672	+#elif defined(arch_atomic64_add_return_relaxed)
	2673	+ s64 ret;
	2674	+ __atomic_pre_full_fence();
	2675	+ ret = arch_atomic64_add_return_relaxed(i, v);
	2676	+ __atomic_post_full_fence();
	2677	+ return ret;
	2678	+#else
	2679	+#error "Unable to define raw_atomic64_add_return"
	2680	+#endif
	2681	+}
1495	2682
1496		−#ifndef arch_atomic64_add_return_acquire
	2683	+/**
	2684	+ * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
	2685	+ * @i: s64 value to add
	2686	+ * @v: pointer to atomic64_t
	2687	+ *
	2688	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	2689	+ *
	2690	+ * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
	2691	+ *
	2692	+ * Return: The updated value of @v.
	2693	+ */
1497	2694	static __always_inline s64
1498		−arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
	2695	+raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1499	2696	{
	2697	+#if defined(arch_atomic64_add_return_acquire)
	2698	+ return arch_atomic64_add_return_acquire(i, v);
	2699	+#elif defined(arch_atomic64_add_return_relaxed)
1500	2700	s64 ret = arch_atomic64_add_return_relaxed(i, v);
1501	2701	__atomic_acquire_fence();
1502	2702	return ret;
1503		−}
1504		−#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
	2703	+#elif defined(arch_atomic64_add_return)
	2704	+ return arch_atomic64_add_return(i, v);
	2705	+#else
	2706	+#error "Unable to define raw_atomic64_add_return_acquire"
1505	2707	#endif
	2708	+}
1506	2709
1507		−#ifndef arch_atomic64_add_return_release
	2710	+/**
	2711	+ * raw_atomic64_add_return_release() - atomic add with release ordering
	2712	+ * @i: s64 value to add
	2713	+ * @v: pointer to atomic64_t
	2714	+ *
	2715	+ * Atomically updates @v to (@v + @i) with release ordering.
	2716	+ *
	2717	+ * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
	2718	+ *
	2719	+ * Return: The updated value of @v.
	2720	+ */
1508	2721	static __always_inline s64
1509		−arch_atomic64_add_return_release(s64 i, atomic64_t *v)
	2722	+raw_atomic64_add_return_release(s64 i, atomic64_t *v)
1510	2723	{
	2724	+#if defined(arch_atomic64_add_return_release)
	2725	+ return arch_atomic64_add_return_release(i, v);
	2726	+#elif defined(arch_atomic64_add_return_relaxed)
1511	2727	__atomic_release_fence();
1512	2728	return arch_atomic64_add_return_relaxed(i, v);
	2729	+#elif defined(arch_atomic64_add_return)
	2730	+ return arch_atomic64_add_return(i, v);
	2731	+#else
	2732	+#error "Unable to define raw_atomic64_add_return_release"
	2733	+#endif
1513	2734	}
1514		−#define arch_atomic64_add_return_release arch_atomic64_add_return_release
	2735	+
	2736	+/**
	2737	+ * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
	2738	+ * @i: s64 value to add
	2739	+ * @v: pointer to atomic64_t
	2740	+ *
	2741	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	2742	+ *
	2743	+ * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
	2744	+ *
	2745	+ * Return: The updated value of @v.
	2746	+ */
	2747	+static __always_inline s64
	2748	+raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
	2749	+{
	2750	+#if defined(arch_atomic64_add_return_relaxed)
	2751	+ return arch_atomic64_add_return_relaxed(i, v);
	2752	+#elif defined(arch_atomic64_add_return)
	2753	+ return arch_atomic64_add_return(i, v);
	2754	+#else
	2755	+#error "Unable to define raw_atomic64_add_return_relaxed"
1515	2756	#endif
	2757	+}
1516	2758
1517		−#ifndef arch_atomic64_add_return
	2759	+/**
	2760	+ * raw_atomic64_fetch_add() - atomic add with full ordering
	2761	+ * @i: s64 value to add
	2762	+ * @v: pointer to atomic64_t
	2763	+ *
	2764	+ * Atomically updates @v to (@v + @i) with full ordering.
	2765	+ *
	2766	+ * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
	2767	+ *
	2768	+ * Return: The original value of @v.
	2769	+ */
1518	2770	static __always_inline s64
1519		−arch_atomic64_add_return(s64 i, atomic64_t *v)
	2771	+raw_atomic64_fetch_add(s64 i, atomic64_t *v)
1520	2772	{
	2773	+#if defined(arch_atomic64_fetch_add)
	2774	+ return arch_atomic64_fetch_add(i, v);
	2775	+#elif defined(arch_atomic64_fetch_add_relaxed)
1521	2776	s64 ret;
1522	2777	__atomic_pre_full_fence();
1523		− ret = arch_atomic64_add_return_relaxed(i, v);
	2778	+ ret = arch_atomic64_fetch_add_relaxed(i, v);
1524	2779	__atomic_post_full_fence();
1525	2780	return ret;
1526		−}
1527		−#define arch_atomic64_add_return arch_atomic64_add_return
	2781	+#else
	2782	+#error "Unable to define raw_atomic64_fetch_add"
1528	2783	#endif
	2784	+}
1529	2785
1530		−#endif /* arch_atomic64_add_return_relaxed */
1531		−
1532		−#ifndef arch_atomic64_fetch_add_relaxed
1533		−#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1534		−#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1535		−#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1536		−#else /* arch_atomic64_fetch_add_relaxed */
1537		−
1538		−#ifndef arch_atomic64_fetch_add_acquire
	2786	+/**
	2787	+ * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
	2788	+ * @i: s64 value to add
	2789	+ * @v: pointer to atomic64_t
	2790	+ *
	2791	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	2792	+ *
	2793	+ * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
	2794	+ *
	2795	+ * Return: The original value of @v.
	2796	+ */
1539	2797	static __always_inline s64
1540		−arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
	2798	+raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1541	2799	{
	2800	+#if defined(arch_atomic64_fetch_add_acquire)
	2801	+ return arch_atomic64_fetch_add_acquire(i, v);
	2802	+#elif defined(arch_atomic64_fetch_add_relaxed)
1542	2803	s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1543	2804	__atomic_acquire_fence();
1544	2805	return ret;
1545		−}
1546		−#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
	2806	+#elif defined(arch_atomic64_fetch_add)
	2807	+ return arch_atomic64_fetch_add(i, v);
	2808	+#else
	2809	+#error "Unable to define raw_atomic64_fetch_add_acquire"
1547	2810	#endif
	2811	+}
1548	2812
1549		−#ifndef arch_atomic64_fetch_add_release
	2813	+/**
	2814	+ * raw_atomic64_fetch_add_release() - atomic add with release ordering
	2815	+ * @i: s64 value to add
	2816	+ * @v: pointer to atomic64_t
	2817	+ *
	2818	+ * Atomically updates @v to (@v + @i) with release ordering.
	2819	+ *
	2820	+ * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
	2821	+ *
	2822	+ * Return: The original value of @v.
	2823	+ */
1550	2824	static __always_inline s64
1551		−arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
	2825	+raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1552	2826	{
	2827	+#if defined(arch_atomic64_fetch_add_release)
	2828	+ return arch_atomic64_fetch_add_release(i, v);
	2829	+#elif defined(arch_atomic64_fetch_add_relaxed)
1553	2830	__atomic_release_fence();
1554	2831	return arch_atomic64_fetch_add_relaxed(i, v);
	2832	+#elif defined(arch_atomic64_fetch_add)
	2833	+ return arch_atomic64_fetch_add(i, v);
	2834	+#else
	2835	+#error "Unable to define raw_atomic64_fetch_add_release"
	2836	+#endif
1555	2837	}
1556		−#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
	2838	+
	2839	+/**
	2840	+ * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
	2841	+ * @i: s64 value to add
	2842	+ * @v: pointer to atomic64_t
	2843	+ *
	2844	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	2845	+ *
	2846	+ * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
	2847	+ *
	2848	+ * Return: The original value of @v.
	2849	+ */
	2850	+static __always_inline s64
	2851	+raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
	2852	+{
	2853	+#if defined(arch_atomic64_fetch_add_relaxed)
	2854	+ return arch_atomic64_fetch_add_relaxed(i, v);
	2855	+#elif defined(arch_atomic64_fetch_add)
	2856	+ return arch_atomic64_fetch_add(i, v);
	2857	+#else
	2858	+#error "Unable to define raw_atomic64_fetch_add_relaxed"
1557	2859	#endif
	2860	+}
	2861	+
	2862	+/**
	2863	+ * raw_atomic64_sub() - atomic subtract with relaxed ordering
	2864	+ * @i: s64 value to subtract
	2865	+ * @v: pointer to atomic64_t
	2866	+ *
	2867	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	2868	+ *
	2869	+ * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
	2870	+ *
	2871	+ * Return: Nothing.
	2872	+ */
	2873	+static __always_inline void
	2874	+raw_atomic64_sub(s64 i, atomic64_t *v)
	2875	+{
	2876	+ arch_atomic64_sub(i, v);
	2877	+}
1558	2878
1559		−#ifndef arch_atomic64_fetch_add
	2879	+/**
	2880	+ * raw_atomic64_sub_return() - atomic subtract with full ordering
	2881	+ * @i: s64 value to subtract
	2882	+ * @v: pointer to atomic64_t
	2883	+ *
	2884	+ * Atomically updates @v to (@v - @i) with full ordering.
	2885	+ *
	2886	+ * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
	2887	+ *
	2888	+ * Return: The updated value of @v.
	2889	+ */
1560	2890	static __always_inline s64
1561		−arch_atomic64_fetch_add(s64 i, atomic64_t *v)
	2891	+raw_atomic64_sub_return(s64 i, atomic64_t *v)
1562	2892	{
	2893	+#if defined(arch_atomic64_sub_return)
	2894	+ return arch_atomic64_sub_return(i, v);
	2895	+#elif defined(arch_atomic64_sub_return_relaxed)
1563	2896	s64 ret;
1564	2897	__atomic_pre_full_fence();
1565		− ret = arch_atomic64_fetch_add_relaxed(i, v);
	2898	+ ret = arch_atomic64_sub_return_relaxed(i, v);
1566	2899	__atomic_post_full_fence();
1567	2900	return ret;
1568		−}
1569		−#define arch_atomic64_fetch_add arch_atomic64_fetch_add
	2901	+#else
	2902	+#error "Unable to define raw_atomic64_sub_return"
1570	2903	#endif
	2904	+}
1571	2905
1572		−#endif /* arch_atomic64_fetch_add_relaxed */
1573		−
1574		−#ifndef arch_atomic64_sub_return_relaxed
1575		−#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1576		−#define arch_atomic64_sub_return_release arch_atomic64_sub_return
1577		−#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1578		−#else /* arch_atomic64_sub_return_relaxed */
1579		−
1580		−#ifndef arch_atomic64_sub_return_acquire
	2906	+/**
	2907	+ * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
	2908	+ * @i: s64 value to subtract
	2909	+ * @v: pointer to atomic64_t
	2910	+ *
	2911	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	2912	+ *
	2913	+ * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
	2914	+ *
	2915	+ * Return: The updated value of @v.
	2916	+ */
1581	2917	static __always_inline s64
1582		−arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
	2918	+raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1583	2919	{
	2920	+#if defined(arch_atomic64_sub_return_acquire)
	2921	+ return arch_atomic64_sub_return_acquire(i, v);
	2922	+#elif defined(arch_atomic64_sub_return_relaxed)
1584	2923	s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1585	2924	__atomic_acquire_fence();
1586	2925	return ret;
1587		−}
1588		−#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
	2926	+#elif defined(arch_atomic64_sub_return)
	2927	+ return arch_atomic64_sub_return(i, v);
	2928	+#else
	2929	+#error "Unable to define raw_atomic64_sub_return_acquire"
1589	2930	#endif
	2931	+}
1590	2932
1591		−#ifndef arch_atomic64_sub_return_release
	2933	+/**
	2934	+ * raw_atomic64_sub_return_release() - atomic subtract with release ordering
	2935	+ * @i: s64 value to subtract
	2936	+ * @v: pointer to atomic64_t
	2937	+ *
	2938	+ * Atomically updates @v to (@v - @i) with release ordering.
	2939	+ *
	2940	+ * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
	2941	+ *
	2942	+ * Return: The updated value of @v.
	2943	+ */
1592	2944	static __always_inline s64
1593		−arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
	2945	+raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
1594	2946	{
	2947	+#if defined(arch_atomic64_sub_return_release)
	2948	+ return arch_atomic64_sub_return_release(i, v);
	2949	+#elif defined(arch_atomic64_sub_return_relaxed)
1595	2950	__atomic_release_fence();
1596	2951	return arch_atomic64_sub_return_relaxed(i, v);
	2952	+#elif defined(arch_atomic64_sub_return)
	2953	+ return arch_atomic64_sub_return(i, v);
	2954	+#else
	2955	+#error "Unable to define raw_atomic64_sub_return_release"
	2956	+#endif
1597	2957	}
1598		−#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
	2958	+
	2959	+/**
	2960	+ * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
	2961	+ * @i: s64 value to subtract
	2962	+ * @v: pointer to atomic64_t
	2963	+ *
	2964	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	2965	+ *
	2966	+ * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
	2967	+ *
	2968	+ * Return: The updated value of @v.
	2969	+ */
	2970	+static __always_inline s64
	2971	+raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
	2972	+{
	2973	+#if defined(arch_atomic64_sub_return_relaxed)
	2974	+ return arch_atomic64_sub_return_relaxed(i, v);
	2975	+#elif defined(arch_atomic64_sub_return)
	2976	+ return arch_atomic64_sub_return(i, v);
	2977	+#else
	2978	+#error "Unable to define raw_atomic64_sub_return_relaxed"
1599	2979	#endif
	2980	+}
1600	2981
1601		−#ifndef arch_atomic64_sub_return
	2982	+/**
	2983	+ * raw_atomic64_fetch_sub() - atomic subtract with full ordering
	2984	+ * @i: s64 value to subtract
	2985	+ * @v: pointer to atomic64_t
	2986	+ *
	2987	+ * Atomically updates @v to (@v - @i) with full ordering.
	2988	+ *
	2989	+ * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
	2990	+ *
	2991	+ * Return: The original value of @v.
	2992	+ */
1602	2993	static __always_inline s64
1603		−arch_atomic64_sub_return(s64 i, atomic64_t *v)
	2994	+raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
1604	2995	{
	2996	+#if defined(arch_atomic64_fetch_sub)
	2997	+ return arch_atomic64_fetch_sub(i, v);
	2998	+#elif defined(arch_atomic64_fetch_sub_relaxed)
1605	2999	s64 ret;
1606	3000	__atomic_pre_full_fence();
1607		− ret = arch_atomic64_sub_return_relaxed(i, v);
	3001	+ ret = arch_atomic64_fetch_sub_relaxed(i, v);
1608	3002	__atomic_post_full_fence();
1609	3003	return ret;
1610		−}
1611		−#define arch_atomic64_sub_return arch_atomic64_sub_return
	3004	+#else
	3005	+#error "Unable to define raw_atomic64_fetch_sub"
1612	3006	#endif
	3007	+}
1613	3008
1614		−#endif /* arch_atomic64_sub_return_relaxed */
1615		−
1616		−#ifndef arch_atomic64_fetch_sub_relaxed
1617		−#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1618		−#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1619		−#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1620		−#else /* arch_atomic64_fetch_sub_relaxed */
1621		−
1622		−#ifndef arch_atomic64_fetch_sub_acquire
	3009	+/**
	3010	+ * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
	3011	+ * @i: s64 value to subtract
	3012	+ * @v: pointer to atomic64_t
	3013	+ *
	3014	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	3015	+ *
	3016	+ * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
	3017	+ *
	3018	+ * Return: The original value of @v.
	3019	+ */
1623	3020	static __always_inline s64
1624		−arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
	3021	+raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1625	3022	{
	3023	+#if defined(arch_atomic64_fetch_sub_acquire)
	3024	+ return arch_atomic64_fetch_sub_acquire(i, v);
	3025	+#elif defined(arch_atomic64_fetch_sub_relaxed)
1626	3026	s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1627	3027	__atomic_acquire_fence();
1628	3028	return ret;
1629		−}
1630		−#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
	3029	+#elif defined(arch_atomic64_fetch_sub)
	3030	+ return arch_atomic64_fetch_sub(i, v);
	3031	+#else
	3032	+#error "Unable to define raw_atomic64_fetch_sub_acquire"
1631	3033	#endif
	3034	+}
1632	3035
1633		−#ifndef arch_atomic64_fetch_sub_release
	3036	+/**
	3037	+ * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
	3038	+ * @i: s64 value to subtract
	3039	+ * @v: pointer to atomic64_t
	3040	+ *
	3041	+ * Atomically updates @v to (@v - @i) with release ordering.
	3042	+ *
	3043	+ * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
	3044	+ *
	3045	+ * Return: The original value of @v.
	3046	+ */
1634	3047	static __always_inline s64
1635		−arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
	3048	+raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1636	3049	{
	3050	+#if defined(arch_atomic64_fetch_sub_release)
	3051	+ return arch_atomic64_fetch_sub_release(i, v);
	3052	+#elif defined(arch_atomic64_fetch_sub_relaxed)
1637	3053	__atomic_release_fence();
1638	3054	return arch_atomic64_fetch_sub_relaxed(i, v);
1639		−}
1640		−#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
	3055	+#elif defined(arch_atomic64_fetch_sub)
	3056	+ return arch_atomic64_fetch_sub(i, v);
	3057	+#else
	3058	+#error "Unable to define raw_atomic64_fetch_sub_release"
1641	3059	#endif
	3060	+}
1642	3061
1643		−#ifndef arch_atomic64_fetch_sub
	3062	+/**
	3063	+ * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
	3064	+ * @i: s64 value to subtract
	3065	+ * @v: pointer to atomic64_t
	3066	+ *
	3067	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	3068	+ *
	3069	+ * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
	3070	+ *
	3071	+ * Return: The original value of @v.
	3072	+ */
1644	3073	static __always_inline s64
1645		−arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
	3074	+raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1646	3075	{
1647		− s64 ret;
1648		− __atomic_pre_full_fence();
1649		− ret = arch_atomic64_fetch_sub_relaxed(i, v);
1650		− __atomic_post_full_fence();
1651		− return ret;
1652		−}
1653		−#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
	3076	+#if defined(arch_atomic64_fetch_sub_relaxed)
	3077	+ return arch_atomic64_fetch_sub_relaxed(i, v);
	3078	+#elif defined(arch_atomic64_fetch_sub)
	3079	+ return arch_atomic64_fetch_sub(i, v);
	3080	+#else
	3081	+#error "Unable to define raw_atomic64_fetch_sub_relaxed"
1654	3082	#endif
1655		−
1656		−#endif /* arch_atomic64_fetch_sub_relaxed */
1657		−
1658		−#ifndef arch_atomic64_inc
1659		−static __always_inline void
1660		−arch_atomic64_inc(atomic64_t *v)
1661		−{
1662		− arch_atomic64_add(1, v);
1663	3083	}
1664		−#define arch_atomic64_inc arch_atomic64_inc
1665		−#endif
1666	3084
1667		−#ifndef arch_atomic64_inc_return_relaxed
1668		−#ifdef arch_atomic64_inc_return
1669		−#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1670		−#define arch_atomic64_inc_return_release arch_atomic64_inc_return
1671		−#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1672		−#endif /* arch_atomic64_inc_return */
1673		−
1674		−#ifndef arch_atomic64_inc_return
1675		−static __always_inline s64
1676		−arch_atomic64_inc_return(atomic64_t *v)
	3085	+/**
	3086	+ * raw_atomic64_inc() - atomic increment with relaxed ordering
	3087	+ * @v: pointer to atomic64_t
	3088	+ *
	3089	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	3090	+ *
	3091	+ * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
	3092	+ *
	3093	+ * Return: Nothing.
	3094	+ */
	3095	+static __always_inline void
	3096	+raw_atomic64_inc(atomic64_t *v)
1677	3097	{
1678		− return arch_atomic64_add_return(1, v);
1679		−}
1680		−#define arch_atomic64_inc_return arch_atomic64_inc_return
	3098	+#if defined(arch_atomic64_inc)
	3099	+ arch_atomic64_inc(v);
	3100	+#else
	3101	+ raw_atomic64_add(1, v);
1681	3102	#endif
1682		−
1683		−#ifndef arch_atomic64_inc_return_acquire
1684		−static __always_inline s64
1685		−arch_atomic64_inc_return_acquire(atomic64_t *v)
1686		−{
1687		− return arch_atomic64_add_return_acquire(1, v);
1688	3103	}
1689		−#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1690		−#endif
1691	3104
1692		−#ifndef arch_atomic64_inc_return_release
	3105	+/**
	3106	+ * raw_atomic64_inc_return() - atomic increment with full ordering
	3107	+ * @v: pointer to atomic64_t
	3108	+ *
	3109	+ * Atomically updates @v to (@v + 1) with full ordering.
	3110	+ *
	3111	+ * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
	3112	+ *
	3113	+ * Return: The updated value of @v.
	3114	+ */
1693	3115	static __always_inline s64
1694		−arch_atomic64_inc_return_release(atomic64_t *v)
	3116	+raw_atomic64_inc_return(atomic64_t *v)
1695	3117	{
1696		− return arch_atomic64_add_return_release(1, v);
1697		−}
1698		−#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
	3118	+#if defined(arch_atomic64_inc_return)
	3119	+ return arch_atomic64_inc_return(v);
	3120	+#elif defined(arch_atomic64_inc_return_relaxed)
	3121	+ s64 ret;
	3122	+ __atomic_pre_full_fence();
	3123	+ ret = arch_atomic64_inc_return_relaxed(v);
	3124	+ __atomic_post_full_fence();
	3125	+ return ret;
	3126	+#else
	3127	+ return raw_atomic64_add_return(1, v);
1699	3128	#endif
1700		−
1701		−#ifndef arch_atomic64_inc_return_relaxed
1702		−static __always_inline s64
1703		−arch_atomic64_inc_return_relaxed(atomic64_t *v)
1704		−{
1705		− return arch_atomic64_add_return_relaxed(1, v);
1706	3129	}
1707		−#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1708		−#endif
1709	3130
1710		−#else /* arch_atomic64_inc_return_relaxed */
1711		−
1712		−#ifndef arch_atomic64_inc_return_acquire
	3131	+/**
	3132	+ * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
	3133	+ * @v: pointer to atomic64_t
	3134	+ *
	3135	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	3136	+ *
	3137	+ * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
	3138	+ *
	3139	+ * Return: The updated value of @v.
	3140	+ */
1713	3141	static __always_inline s64
1714		−arch_atomic64_inc_return_acquire(atomic64_t *v)
	3142	+raw_atomic64_inc_return_acquire(atomic64_t *v)
1715	3143	{
	3144	+#if defined(arch_atomic64_inc_return_acquire)
	3145	+ return arch_atomic64_inc_return_acquire(v);
	3146	+#elif defined(arch_atomic64_inc_return_relaxed)
1716	3147	s64 ret = arch_atomic64_inc_return_relaxed(v);
1717	3148	__atomic_acquire_fence();
1718	3149	return ret;
1719		−}
1720		−#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
	3150	+#elif defined(arch_atomic64_inc_return)
	3151	+ return arch_atomic64_inc_return(v);
	3152	+#else
	3153	+ return raw_atomic64_add_return_acquire(1, v);
1721	3154	#endif
	3155	+}
1722	3156
1723		−#ifndef arch_atomic64_inc_return_release
	3157	+/**
	3158	+ * raw_atomic64_inc_return_release() - atomic increment with release ordering
	3159	+ * @v: pointer to atomic64_t
	3160	+ *
	3161	+ * Atomically updates @v to (@v + 1) with release ordering.
	3162	+ *
	3163	+ * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
	3164	+ *
	3165	+ * Return: The updated value of @v.
	3166	+ */
1724	3167	static __always_inline s64
1725		−arch_atomic64_inc_return_release(atomic64_t *v)
	3168	+raw_atomic64_inc_return_release(atomic64_t *v)
1726	3169	{
	3170	+#if defined(arch_atomic64_inc_return_release)
	3171	+ return arch_atomic64_inc_return_release(v);
	3172	+#elif defined(arch_atomic64_inc_return_relaxed)
1727	3173	__atomic_release_fence();
1728	3174	return arch_atomic64_inc_return_relaxed(v);
1729		−}
1730		−#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
	3175	+#elif defined(arch_atomic64_inc_return)
	3176	+ return arch_atomic64_inc_return(v);
	3177	+#else
	3178	+ return raw_atomic64_add_return_release(1, v);
1731	3179	#endif
1732		−
1733		−#ifndef arch_atomic64_inc_return
1734		−static __always_inline s64
1735		−arch_atomic64_inc_return(atomic64_t *v)
1736		−{
1737		− s64 ret;
1738		− __atomic_pre_full_fence();
1739		− ret = arch_atomic64_inc_return_relaxed(v);
1740		− __atomic_post_full_fence();
1741		− return ret;
1742	3180	}
1743		−#define arch_atomic64_inc_return arch_atomic64_inc_return
1744		−#endif
1745		−
1746		−#endif /* arch_atomic64_inc_return_relaxed */
1747	3181
1748		−#ifndef arch_atomic64_fetch_inc_relaxed
1749		−#ifdef arch_atomic64_fetch_inc
1750		−#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1751		−#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1752		−#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1753		−#endif /* arch_atomic64_fetch_inc */
1754		−
1755		−#ifndef arch_atomic64_fetch_inc
	3182	+/**
	3183	+ * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
	3184	+ * @v: pointer to atomic64_t
	3185	+ *
	3186	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	3187	+ *
	3188	+ * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
	3189	+ *
	3190	+ * Return: The updated value of @v.
	3191	+ */
1756	3192	static __always_inline s64
1757		−arch_atomic64_fetch_inc(atomic64_t *v)
	3193	+raw_atomic64_inc_return_relaxed(atomic64_t *v)
1758	3194	{
1759		− return arch_atomic64_fetch_add(1, v);
1760		−}
1761		−#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
	3195	+#if defined(arch_atomic64_inc_return_relaxed)
	3196	+ return arch_atomic64_inc_return_relaxed(v);
	3197	+#elif defined(arch_atomic64_inc_return)
	3198	+ return arch_atomic64_inc_return(v);
	3199	+#else
	3200	+ return raw_atomic64_add_return_relaxed(1, v);
1762	3201	#endif
1763		−
1764		−#ifndef arch_atomic64_fetch_inc_acquire
1765		−static __always_inline s64
1766		−arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1767		−{
1768		− return arch_atomic64_fetch_add_acquire(1, v);
1769	3202	}
1770		−#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1771		−#endif
1772	3203
1773		−#ifndef arch_atomic64_fetch_inc_release
	3204	+/**
	3205	+ * raw_atomic64_fetch_inc() - atomic increment with full ordering
	3206	+ * @v: pointer to atomic64_t
	3207	+ *
	3208	+ * Atomically updates @v to (@v + 1) with full ordering.
	3209	+ *
	3210	+ * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
	3211	+ *
	3212	+ * Return: The original value of @v.
	3213	+ */
1774	3214	static __always_inline s64
1775		−arch_atomic64_fetch_inc_release(atomic64_t *v)
	3215	+raw_atomic64_fetch_inc(atomic64_t *v)
1776	3216	{
1777		− return arch_atomic64_fetch_add_release(1, v);
1778		−}
1779		−#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
	3217	+#if defined(arch_atomic64_fetch_inc)
	3218	+ return arch_atomic64_fetch_inc(v);
	3219	+#elif defined(arch_atomic64_fetch_inc_relaxed)
	3220	+ s64 ret;
	3221	+ __atomic_pre_full_fence();
	3222	+ ret = arch_atomic64_fetch_inc_relaxed(v);
	3223	+ __atomic_post_full_fence();
	3224	+ return ret;
	3225	+#else
	3226	+ return raw_atomic64_fetch_add(1, v);
1780	3227	#endif
1781		−
1782		−#ifndef arch_atomic64_fetch_inc_relaxed
1783		−static __always_inline s64
1784		−arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1785		−{
1786		− return arch_atomic64_fetch_add_relaxed(1, v);
1787	3228	}
1788		−#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1789		−#endif
1790		−
1791		−#else /* arch_atomic64_fetch_inc_relaxed */
1792	3229
1793		−#ifndef arch_atomic64_fetch_inc_acquire
	3230	+/**
	3231	+ * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
	3232	+ * @v: pointer to atomic64_t
	3233	+ *
	3234	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	3235	+ *
	3236	+ * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
	3237	+ *
	3238	+ * Return: The original value of @v.
	3239	+ */
1794	3240	static __always_inline s64
1795		−arch_atomic64_fetch_inc_acquire(atomic64_t *v)
	3241	+raw_atomic64_fetch_inc_acquire(atomic64_t *v)
1796	3242	{
	3243	+#if defined(arch_atomic64_fetch_inc_acquire)
	3244	+ return arch_atomic64_fetch_inc_acquire(v);
	3245	+#elif defined(arch_atomic64_fetch_inc_relaxed)
1797	3246	s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1798	3247	__atomic_acquire_fence();
1799	3248	return ret;
1800		−}
1801		−#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
	3249	+#elif defined(arch_atomic64_fetch_inc)
	3250	+ return arch_atomic64_fetch_inc(v);
	3251	+#else
	3252	+ return raw_atomic64_fetch_add_acquire(1, v);
1802	3253	#endif
	3254	+}
1803	3255
1804		−#ifndef arch_atomic64_fetch_inc_release
	3256	+/**
	3257	+ * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
	3258	+ * @v: pointer to atomic64_t
	3259	+ *
	3260	+ * Atomically updates @v to (@v + 1) with release ordering.
	3261	+ *
	3262	+ * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
	3263	+ *
	3264	+ * Return: The original value of @v.
	3265	+ */
1805	3266	static __always_inline s64
1806		−arch_atomic64_fetch_inc_release(atomic64_t *v)
	3267	+raw_atomic64_fetch_inc_release(atomic64_t *v)
1807	3268	{
	3269	+#if defined(arch_atomic64_fetch_inc_release)
	3270	+ return arch_atomic64_fetch_inc_release(v);
	3271	+#elif defined(arch_atomic64_fetch_inc_relaxed)
1808	3272	__atomic_release_fence();
1809	3273	return arch_atomic64_fetch_inc_relaxed(v);
1810		−}
1811		−#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1812		−#endif
1813		−
1814		−#ifndef arch_atomic64_fetch_inc
1815		−static __always_inline s64
1816		−arch_atomic64_fetch_inc(atomic64_t *v)
1817		−{
1818		− s64 ret;
1819		− __atomic_pre_full_fence();
1820		− ret = arch_atomic64_fetch_inc_relaxed(v);
1821		− __atomic_post_full_fence();
1822		− return ret;
1823		−}
1824		−#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
	3274	+#elif defined(arch_atomic64_fetch_inc)
	3275	+ return arch_atomic64_fetch_inc(v);
	3276	+#else
	3277	+ return raw_atomic64_fetch_add_release(1, v);
1825	3278	#endif
1826		−
1827		−#endif /* arch_atomic64_fetch_inc_relaxed */
1828		−
1829		−#ifndef arch_atomic64_dec
1830		−static __always_inline void
1831		−arch_atomic64_dec(atomic64_t *v)
1832		−{
1833		− arch_atomic64_sub(1, v);
1834	3279	}
1835		−#define arch_atomic64_dec arch_atomic64_dec
1836		−#endif
1837	3280
1838		−#ifndef arch_atomic64_dec_return_relaxed
1839		−#ifdef arch_atomic64_dec_return
1840		−#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1841		−#define arch_atomic64_dec_return_release arch_atomic64_dec_return
1842		−#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1843		−#endif /* arch_atomic64_dec_return */
1844		−
1845		−#ifndef arch_atomic64_dec_return
	3281	+/**
	3282	+ * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
	3283	+ * @v: pointer to atomic64_t
	3284	+ *
	3285	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	3286	+ *
	3287	+ * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
	3288	+ *
	3289	+ * Return: The original value of @v.
	3290	+ */
1846	3291	static __always_inline s64
1847		−arch_atomic64_dec_return(atomic64_t *v)
	3292	+raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
1848	3293	{
1849		− return arch_atomic64_sub_return(1, v);
1850		−}
1851		−#define arch_atomic64_dec_return arch_atomic64_dec_return
	3294	+#if defined(arch_atomic64_fetch_inc_relaxed)
	3295	+ return arch_atomic64_fetch_inc_relaxed(v);
	3296	+#elif defined(arch_atomic64_fetch_inc)
	3297	+ return arch_atomic64_fetch_inc(v);
	3298	+#else
	3299	+ return raw_atomic64_fetch_add_relaxed(1, v);
1852	3300	#endif
	3301	+}
1853	3302
1854		−#ifndef arch_atomic64_dec_return_acquire
1855		−static __always_inline s64
1856		−arch_atomic64_dec_return_acquire(atomic64_t *v)
	3303	+/**
	3304	+ * raw_atomic64_dec() - atomic decrement with relaxed ordering
	3305	+ * @v: pointer to atomic64_t
	3306	+ *
	3307	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	3308	+ *
	3309	+ * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
	3310	+ *
	3311	+ * Return: Nothing.
	3312	+ */
	3313	+static __always_inline void
	3314	+raw_atomic64_dec(atomic64_t *v)
1857	3315	{
1858		− return arch_atomic64_sub_return_acquire(1, v);
1859		−}
1860		−#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
	3316	+#if defined(arch_atomic64_dec)
	3317	+ arch_atomic64_dec(v);
	3318	+#else
	3319	+ raw_atomic64_sub(1, v);
1861	3320	#endif
1862		−
1863		−#ifndef arch_atomic64_dec_return_release
1864		−static __always_inline s64
1865		−arch_atomic64_dec_return_release(atomic64_t *v)
1866		−{
1867		− return arch_atomic64_sub_return_release(1, v);
1868	3321	}
1869		−#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1870		−#endif
1871	3322
1872		−#ifndef arch_atomic64_dec_return_relaxed
	3323	+/**
	3324	+ * raw_atomic64_dec_return() - atomic decrement with full ordering
	3325	+ * @v: pointer to atomic64_t
	3326	+ *
	3327	+ * Atomically updates @v to (@v - 1) with full ordering.
	3328	+ *
	3329	+ * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
	3330	+ *
	3331	+ * Return: The updated value of @v.
	3332	+ */
1873	3333	static __always_inline s64
1874		−arch_atomic64_dec_return_relaxed(atomic64_t *v)
	3334	+raw_atomic64_dec_return(atomic64_t *v)
1875	3335	{
1876		− return arch_atomic64_sub_return_relaxed(1, v);
1877		−}
1878		−#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
	3336	+#if defined(arch_atomic64_dec_return)
	3337	+ return arch_atomic64_dec_return(v);
	3338	+#elif defined(arch_atomic64_dec_return_relaxed)
	3339	+ s64 ret;
	3340	+ __atomic_pre_full_fence();
	3341	+ ret = arch_atomic64_dec_return_relaxed(v);
	3342	+ __atomic_post_full_fence();
	3343	+ return ret;
	3344	+#else
	3345	+ return raw_atomic64_sub_return(1, v);
1879	3346	#endif
	3347	+}
1880	3348
1881		−#else /* arch_atomic64_dec_return_relaxed */
1882		−
1883		−#ifndef arch_atomic64_dec_return_acquire
	3349	+/**
	3350	+ * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
	3351	+ * @v: pointer to atomic64_t
	3352	+ *
	3353	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	3354	+ *
	3355	+ * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
	3356	+ *
	3357	+ * Return: The updated value of @v.
	3358	+ */
1884	3359	static __always_inline s64
1885		−arch_atomic64_dec_return_acquire(atomic64_t *v)
	3360	+raw_atomic64_dec_return_acquire(atomic64_t *v)
1886	3361	{
	3362	+#if defined(arch_atomic64_dec_return_acquire)
	3363	+ return arch_atomic64_dec_return_acquire(v);
	3364	+#elif defined(arch_atomic64_dec_return_relaxed)
1887	3365	s64 ret = arch_atomic64_dec_return_relaxed(v);
1888	3366	__atomic_acquire_fence();
1889	3367	return ret;
1890		−}
1891		−#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
	3368	+#elif defined(arch_atomic64_dec_return)
	3369	+ return arch_atomic64_dec_return(v);
	3370	+#else
	3371	+ return raw_atomic64_sub_return_acquire(1, v);
1892	3372	#endif
	3373	+}
1893	3374
1894		−#ifndef arch_atomic64_dec_return_release
	3375	+/**
	3376	+ * raw_atomic64_dec_return_release() - atomic decrement with release ordering
	3377	+ * @v: pointer to atomic64_t
	3378	+ *
	3379	+ * Atomically updates @v to (@v - 1) with release ordering.
	3380	+ *
	3381	+ * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
	3382	+ *
	3383	+ * Return: The updated value of @v.
	3384	+ */
1895	3385	static __always_inline s64
1896		−arch_atomic64_dec_return_release(atomic64_t *v)
	3386	+raw_atomic64_dec_return_release(atomic64_t *v)
1897	3387	{
	3388	+#if defined(arch_atomic64_dec_return_release)
	3389	+ return arch_atomic64_dec_return_release(v);
	3390	+#elif defined(arch_atomic64_dec_return_relaxed)
1898	3391	__atomic_release_fence();
1899	3392	return arch_atomic64_dec_return_relaxed(v);
	3393	+#elif defined(arch_atomic64_dec_return)
	3394	+ return arch_atomic64_dec_return(v);
	3395	+#else
	3396	+ return raw_atomic64_sub_return_release(1, v);
	3397	+#endif
1900	3398	}
1901		−#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
	3399	+
	3400	+/**
	3401	+ * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
	3402	+ * @v: pointer to atomic64_t
	3403	+ *
	3404	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	3405	+ *
	3406	+ * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
	3407	+ *
	3408	+ * Return: The updated value of @v.
	3409	+ */
	3410	+static __always_inline s64
	3411	+raw_atomic64_dec_return_relaxed(atomic64_t *v)
	3412	+{
	3413	+#if defined(arch_atomic64_dec_return_relaxed)
	3414	+ return arch_atomic64_dec_return_relaxed(v);
	3415	+#elif defined(arch_atomic64_dec_return)
	3416	+ return arch_atomic64_dec_return(v);
	3417	+#else
	3418	+ return raw_atomic64_sub_return_relaxed(1, v);
1902	3419	#endif
	3420	+}
1903	3421
1904		−#ifndef arch_atomic64_dec_return
	3422	+/**
	3423	+ * raw_atomic64_fetch_dec() - atomic decrement with full ordering
	3424	+ * @v: pointer to atomic64_t
	3425	+ *
	3426	+ * Atomically updates @v to (@v - 1) with full ordering.
	3427	+ *
	3428	+ * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
	3429	+ *
	3430	+ * Return: The original value of @v.
	3431	+ */
1905	3432	static __always_inline s64
1906		−arch_atomic64_dec_return(atomic64_t *v)
	3433	+raw_atomic64_fetch_dec(atomic64_t *v)
1907	3434	{
	3435	+#if defined(arch_atomic64_fetch_dec)
	3436	+ return arch_atomic64_fetch_dec(v);
	3437	+#elif defined(arch_atomic64_fetch_dec_relaxed)
1908	3438	s64 ret;
1909	3439	__atomic_pre_full_fence();
1910		− ret = arch_atomic64_dec_return_relaxed(v);
	3440	+ ret = arch_atomic64_fetch_dec_relaxed(v);
1911	3441	__atomic_post_full_fence();
1912	3442	return ret;
1913		−}
1914		−#define arch_atomic64_dec_return arch_atomic64_dec_return
	3443	+#else
	3444	+ return raw_atomic64_fetch_sub(1, v);
1915	3445	#endif
1916		−
1917		−#endif /* arch_atomic64_dec_return_relaxed */
1918		−
1919		−#ifndef arch_atomic64_fetch_dec_relaxed
1920		−#ifdef arch_atomic64_fetch_dec
1921		−#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1922		−#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1923		−#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1924		−#endif /* arch_atomic64_fetch_dec */
1925		−
1926		−#ifndef arch_atomic64_fetch_dec
1927		−static __always_inline s64
1928		−arch_atomic64_fetch_dec(atomic64_t *v)
1929		−{
1930		− return arch_atomic64_fetch_sub(1, v);
1931	3446	}
1932		−#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1933		−#endif
1934	3447
1935		−#ifndef arch_atomic64_fetch_dec_acquire
	3448	+/**
	3449	+ * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
	3450	+ * @v: pointer to atomic64_t
	3451	+ *
	3452	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	3453	+ *
	3454	+ * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
	3455	+ *
	3456	+ * Return: The original value of @v.
	3457	+ */
1936	3458	static __always_inline s64
1937		−arch_atomic64_fetch_dec_acquire(atomic64_t *v)
	3459	+raw_atomic64_fetch_dec_acquire(atomic64_t *v)
1938	3460	{
1939		− return arch_atomic64_fetch_sub_acquire(1, v);
1940		−}
1941		−#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
	3461	+#if defined(arch_atomic64_fetch_dec_acquire)
	3462	+ return arch_atomic64_fetch_dec_acquire(v);
	3463	+#elif defined(arch_atomic64_fetch_dec_relaxed)
	3464	+ s64 ret = arch_atomic64_fetch_dec_relaxed(v);
	3465	+ __atomic_acquire_fence();
	3466	+ return ret;
	3467	+#elif defined(arch_atomic64_fetch_dec)
	3468	+ return arch_atomic64_fetch_dec(v);
	3469	+#else
	3470	+ return raw_atomic64_fetch_sub_acquire(1, v);
1942	3471	#endif
1943		−
1944		−#ifndef arch_atomic64_fetch_dec_release
1945		−static __always_inline s64
1946		−arch_atomic64_fetch_dec_release(atomic64_t *v)
1947		−{
1948		− return arch_atomic64_fetch_sub_release(1, v);
1949	3472	}
1950		−#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1951		−#endif
1952	3473
1953		−#ifndef arch_atomic64_fetch_dec_relaxed
	3474	+/**
	3475	+ * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
	3476	+ * @v: pointer to atomic64_t
	3477	+ *
	3478	+ * Atomically updates @v to (@v - 1) with release ordering.
	3479	+ *
	3480	+ * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
	3481	+ *
	3482	+ * Return: The original value of @v.
	3483	+ */
1954	3484	static __always_inline s64
1955		−arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
	3485	+raw_atomic64_fetch_dec_release(atomic64_t *v)
1956	3486	{
1957		− return arch_atomic64_fetch_sub_relaxed(1, v);
1958		−}
1959		−#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
	3487	+#if defined(arch_atomic64_fetch_dec_release)
	3488	+ return arch_atomic64_fetch_dec_release(v);
	3489	+#elif defined(arch_atomic64_fetch_dec_relaxed)
	3490	+ __atomic_release_fence();
	3491	+ return arch_atomic64_fetch_dec_relaxed(v);
	3492	+#elif defined(arch_atomic64_fetch_dec)
	3493	+ return arch_atomic64_fetch_dec(v);
	3494	+#else
	3495	+ return raw_atomic64_fetch_sub_release(1, v);
1960	3496	#endif
	3497	+}
1961	3498
1962		−#else /* arch_atomic64_fetch_dec_relaxed */
1963		−
1964		−#ifndef arch_atomic64_fetch_dec_acquire
	3499	+/**
	3500	+ * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
	3501	+ * @v: pointer to atomic64_t
	3502	+ *
	3503	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	3504	+ *
	3505	+ * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
	3506	+ *
	3507	+ * Return: The original value of @v.
	3508	+ */
1965	3509	static __always_inline s64
1966		−arch_atomic64_fetch_dec_acquire(atomic64_t *v)
	3510	+raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
1967	3511	{
1968		− s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1969		− __atomic_acquire_fence();
1970		− return ret;
1971		−}
1972		−#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
	3512	+#if defined(arch_atomic64_fetch_dec_relaxed)
	3513	+ return arch_atomic64_fetch_dec_relaxed(v);
	3514	+#elif defined(arch_atomic64_fetch_dec)
	3515	+ return arch_atomic64_fetch_dec(v);
	3516	+#else
	3517	+ return raw_atomic64_fetch_sub_relaxed(1, v);
1973	3518	#endif
	3519	+}
1974	3520
1975		−#ifndef arch_atomic64_fetch_dec_release
1976		−static __always_inline s64
1977		−arch_atomic64_fetch_dec_release(atomic64_t *v)
	3521	+/**
	3522	+ * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
	3523	+ * @i: s64 value
	3524	+ * @v: pointer to atomic64_t
	3525	+ *
	3526	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	3527	+ *
	3528	+ * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
	3529	+ *
	3530	+ * Return: Nothing.
	3531	+ */
	3532	+static __always_inline void
	3533	+raw_atomic64_and(s64 i, atomic64_t *v)
1978	3534	{
1979		− __atomic_release_fence();
1980		− return arch_atomic64_fetch_dec_relaxed(v);
	3535	+ arch_atomic64_and(i, v);
1981	3536	}
1982		−#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1983		−#endif
1984	3537
1985		−#ifndef arch_atomic64_fetch_dec
	3538	+/**
	3539	+ * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
	3540	+ * @i: s64 value
	3541	+ * @v: pointer to atomic64_t
	3542	+ *
	3543	+ * Atomically updates @v to (@v & @i) with full ordering.
	3544	+ *
	3545	+ * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
	3546	+ *
	3547	+ * Return: The original value of @v.
	3548	+ */
1986	3549	static __always_inline s64
1987		−arch_atomic64_fetch_dec(atomic64_t *v)
	3550	+raw_atomic64_fetch_and(s64 i, atomic64_t *v)
1988	3551	{
	3552	+#if defined(arch_atomic64_fetch_and)
	3553	+ return arch_atomic64_fetch_and(i, v);
	3554	+#elif defined(arch_atomic64_fetch_and_relaxed)
1989	3555	s64 ret;
1990	3556	__atomic_pre_full_fence();
1991		− ret = arch_atomic64_fetch_dec_relaxed(v);
	3557	+ ret = arch_atomic64_fetch_and_relaxed(i, v);
1992	3558	__atomic_post_full_fence();
1993	3559	return ret;
1994		−}
1995		−#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
	3560	+#else
	3561	+#error "Unable to define raw_atomic64_fetch_and"
1996	3562	#endif
	3563	+}
1997	3564
1998		−#endif /* arch_atomic64_fetch_dec_relaxed */
1999		−
2000		−#ifndef arch_atomic64_fetch_and_relaxed
2001		−#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
2002		−#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
2003		−#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
2004		−#else /* arch_atomic64_fetch_and_relaxed */
2005		−
2006		−#ifndef arch_atomic64_fetch_and_acquire
	3565	+/**
	3566	+ * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
	3567	+ * @i: s64 value
	3568	+ * @v: pointer to atomic64_t
	3569	+ *
	3570	+ * Atomically updates @v to (@v & @i) with acquire ordering.
	3571	+ *
	3572	+ * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
	3573	+ *
	3574	+ * Return: The original value of @v.
	3575	+ */
2007	3576	static __always_inline s64
2008		−arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
	3577	+raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2009	3578	{
	3579	+#if defined(arch_atomic64_fetch_and_acquire)
	3580	+ return arch_atomic64_fetch_and_acquire(i, v);
	3581	+#elif defined(arch_atomic64_fetch_and_relaxed)
2010	3582	s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
2011	3583	__atomic_acquire_fence();
2012	3584	return ret;
2013		−}
2014		−#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
	3585	+#elif defined(arch_atomic64_fetch_and)
	3586	+ return arch_atomic64_fetch_and(i, v);
	3587	+#else
	3588	+#error "Unable to define raw_atomic64_fetch_and_acquire"
2015	3589	#endif
	3590	+}
2016	3591
2017		−#ifndef arch_atomic64_fetch_and_release
	3592	+/**
	3593	+ * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
	3594	+ * @i: s64 value
	3595	+ * @v: pointer to atomic64_t
	3596	+ *
	3597	+ * Atomically updates @v to (@v & @i) with release ordering.
	3598	+ *
	3599	+ * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
	3600	+ *
	3601	+ * Return: The original value of @v.
	3602	+ */
2018	3603	static __always_inline s64
2019		−arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
	3604	+raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
2020	3605	{
	3606	+#if defined(arch_atomic64_fetch_and_release)
	3607	+ return arch_atomic64_fetch_and_release(i, v);
	3608	+#elif defined(arch_atomic64_fetch_and_relaxed)
2021	3609	__atomic_release_fence();
2022	3610	return arch_atomic64_fetch_and_relaxed(i, v);
2023		−}
2024		−#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
	3611	+#elif defined(arch_atomic64_fetch_and)
	3612	+ return arch_atomic64_fetch_and(i, v);
	3613	+#else
	3614	+#error "Unable to define raw_atomic64_fetch_and_release"
2025	3615	#endif
	3616	+}
2026	3617
2027		−#ifndef arch_atomic64_fetch_and
	3618	+/**
	3619	+ * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
	3620	+ * @i: s64 value
	3621	+ * @v: pointer to atomic64_t
	3622	+ *
	3623	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	3624	+ *
	3625	+ * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
	3626	+ *
	3627	+ * Return: The original value of @v.
	3628	+ */
2028	3629	static __always_inline s64
2029		−arch_atomic64_fetch_and(s64 i, atomic64_t *v)
	3630	+raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
2030	3631	{
2031		− s64 ret;
2032		− __atomic_pre_full_fence();
2033		− ret = arch_atomic64_fetch_and_relaxed(i, v);
2034		− __atomic_post_full_fence();
2035		− return ret;
2036		−}
2037		−#define arch_atomic64_fetch_and arch_atomic64_fetch_and
	3632	+#if defined(arch_atomic64_fetch_and_relaxed)
	3633	+ return arch_atomic64_fetch_and_relaxed(i, v);
	3634	+#elif defined(arch_atomic64_fetch_and)
	3635	+ return arch_atomic64_fetch_and(i, v);
	3636	+#else
	3637	+#error "Unable to define raw_atomic64_fetch_and_relaxed"
2038	3638	#endif
2039		−
2040		−#endif /* arch_atomic64_fetch_and_relaxed */
2041		−
2042		−#ifndef arch_atomic64_andnot
2043		−static __always_inline void
2044		−arch_atomic64_andnot(s64 i, atomic64_t *v)
2045		−{
2046		− arch_atomic64_and(~i, v);
2047	3639	}
2048		−#define arch_atomic64_andnot arch_atomic64_andnot
2049		−#endif
2050		−
2051		−#ifndef arch_atomic64_fetch_andnot_relaxed
2052		−#ifdef arch_atomic64_fetch_andnot
2053		−#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
2054		−#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
2055		−#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
2056		−#endif /* arch_atomic64_fetch_andnot */
2057	3640
2058		−#ifndef arch_atomic64_fetch_andnot
2059		−static __always_inline s64
2060		−arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
	3641	+/**
	3642	+ * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
	3643	+ * @i: s64 value
	3644	+ * @v: pointer to atomic64_t
	3645	+ *
	3646	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	3647	+ *
	3648	+ * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
	3649	+ *
	3650	+ * Return: Nothing.
	3651	+ */
	3652	+static __always_inline void
	3653	+raw_atomic64_andnot(s64 i, atomic64_t *v)
2061	3654	{
2062		− return arch_atomic64_fetch_and(~i, v);
2063		−}
2064		−#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
	3655	+#if defined(arch_atomic64_andnot)
	3656	+ arch_atomic64_andnot(i, v);
	3657	+#else
	3658	+ raw_atomic64_and(~i, v);
2065	3659	#endif
2066		−
2067		−#ifndef arch_atomic64_fetch_andnot_acquire
2068		−static __always_inline s64
2069		−arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2070		−{
2071		− return arch_atomic64_fetch_and_acquire(~i, v);
2072	3660	}
2073		−#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
2074		−#endif
2075	3661
2076		−#ifndef arch_atomic64_fetch_andnot_release
	3662	+/**
	3663	+ * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
	3664	+ * @i: s64 value
	3665	+ * @v: pointer to atomic64_t
	3666	+ *
	3667	+ * Atomically updates @v to (@v & ~@i) with full ordering.
	3668	+ *
	3669	+ * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
	3670	+ *
	3671	+ * Return: The original value of @v.
	3672	+ */
2077	3673	static __always_inline s64
2078		−arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
	3674	+raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
2079	3675	{
2080		− return arch_atomic64_fetch_and_release(~i, v);
2081		−}
2082		−#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
	3676	+#if defined(arch_atomic64_fetch_andnot)
	3677	+ return arch_atomic64_fetch_andnot(i, v);
	3678	+#elif defined(arch_atomic64_fetch_andnot_relaxed)
	3679	+ s64 ret;
	3680	+ __atomic_pre_full_fence();
	3681	+ ret = arch_atomic64_fetch_andnot_relaxed(i, v);
	3682	+ __atomic_post_full_fence();
	3683	+ return ret;
	3684	+#else
	3685	+ return raw_atomic64_fetch_and(~i, v);
2083	3686	#endif
2084		−
2085		−#ifndef arch_atomic64_fetch_andnot_relaxed
2086		−static __always_inline s64
2087		−arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2088		−{
2089		− return arch_atomic64_fetch_and_relaxed(~i, v);
2090	3687	}
2091		−#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
2092		−#endif
2093		−
2094		−#else /* arch_atomic64_fetch_andnot_relaxed */
2095	3688
2096		−#ifndef arch_atomic64_fetch_andnot_acquire
	3689	+/**
	3690	+ * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
	3691	+ * @i: s64 value
	3692	+ * @v: pointer to atomic64_t
	3693	+ *
	3694	+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
	3695	+ *
	3696	+ * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
	3697	+ *
	3698	+ * Return: The original value of @v.
	3699	+ */
2097	3700	static __always_inline s64
2098		−arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
	3701	+raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2099	3702	{
	3703	+#if defined(arch_atomic64_fetch_andnot_acquire)
	3704	+ return arch_atomic64_fetch_andnot_acquire(i, v);
	3705	+#elif defined(arch_atomic64_fetch_andnot_relaxed)
2100	3706	s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
2101	3707	__atomic_acquire_fence();
2102	3708	return ret;
2103		−}
2104		−#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
	3709	+#elif defined(arch_atomic64_fetch_andnot)
	3710	+ return arch_atomic64_fetch_andnot(i, v);
	3711	+#else
	3712	+ return raw_atomic64_fetch_and_acquire(~i, v);
2105	3713	#endif
	3714	+}
2106	3715
2107		−#ifndef arch_atomic64_fetch_andnot_release
	3716	+/**
	3717	+ * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
	3718	+ * @i: s64 value
	3719	+ * @v: pointer to atomic64_t
	3720	+ *
	3721	+ * Atomically updates @v to (@v & ~@i) with release ordering.
	3722	+ *
	3723	+ * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
	3724	+ *
	3725	+ * Return: The original value of @v.
	3726	+ */
2108	3727	static __always_inline s64
2109		−arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
	3728	+raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2110	3729	{
	3730	+#if defined(arch_atomic64_fetch_andnot_release)
	3731	+ return arch_atomic64_fetch_andnot_release(i, v);
	3732	+#elif defined(arch_atomic64_fetch_andnot_relaxed)
2111	3733	__atomic_release_fence();
2112	3734	return arch_atomic64_fetch_andnot_relaxed(i, v);
	3735	+#elif defined(arch_atomic64_fetch_andnot)
	3736	+ return arch_atomic64_fetch_andnot(i, v);
	3737	+#else
	3738	+ return raw_atomic64_fetch_and_release(~i, v);
	3739	+#endif
2113	3740	}
2114		−#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
	3741	+
	3742	+/**
	3743	+ * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering

1241 more lines aren’t shown here.

include/linux/atomic/atomic-instrumented.h+3194−361

@@ -4,15 +4,10 @@ · 3 unmodified lines
4	4	// DO NOT MODIFY THIS FILE DIRECTLY
5	5
6	6	/*
7		− * This file provides wrappers with KASAN instrumentation for atomic operations.
8		− * To use this functionality an arch's atomic.h file needs to define all
9		− * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10		− * this file at the end. This file provides atomic_read() that forwards to
11		− * arch_atomic_read() for actual atomic operation.
12		− * Note: if an arch atomic operation is implemented by means of other atomic
13		− * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14		− * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15		− * double instrumentation.
	7	+ * This file provoides atomic operations with explicit instrumentation (e.g.
	8	+ * KASAN, KCSAN), which should be used unless it is necessary to avoid
	9	+ * instrumentation. Where it is necessary to aovid instrumenation, the
	10	+ * raw_atomic*() operations should be used.
16	11	*/
17	12	#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18	13	#define _LINUX_ATOMIC_INSTRUMENTED_H
@@ -21,1927 +16,4696 @@ · 2 unmodified lines
21	16	#include <linux/compiler.h>
22	17	#include <linux/instrumented.h>
23	18
	19	+/**
	20	+ * atomic_read() - atomic load with relaxed ordering
	21	+ * @v: pointer to atomic_t
	22	+ *
	23	+ * Atomically loads the value of @v with relaxed ordering.
	24	+ *
	25	+ * Unsafe to use in noinstr code; use raw_atomic_read() there.
	26	+ *
	27	+ * Return: The value loaded from @v.
	28	+ */
24	29	static __always_inline int
25	30	atomic_read(const atomic_t *v)
26	31	{
27	32	instrument_atomic_read(v, sizeof(*v));
28		− return arch_atomic_read(v);
29		−}
30		−
	33	+ return raw_atomic_read(v);
	34	+}
	35	+
	36	+/**
	37	+ * atomic_read_acquire() - atomic load with acquire ordering
	38	+ * @v: pointer to atomic_t
	39	+ *
	40	+ * Atomically loads the value of @v with acquire ordering.
	41	+ *
	42	+ * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there.
	43	+ *
	44	+ * Return: The value loaded from @v.
	45	+ */
31	46	static __always_inline int
32	47	atomic_read_acquire(const atomic_t *v)
33	48	{
34	49	instrument_atomic_read(v, sizeof(*v));
35		− return arch_atomic_read_acquire(v);
36		−}
37		−
	50	+ return raw_atomic_read_acquire(v);
	51	+}
	52	+
	53	+/**
	54	+ * atomic_set() - atomic set with relaxed ordering
	55	+ * @v: pointer to atomic_t
	56	+ * @i: int value to assign
	57	+ *
	58	+ * Atomically sets @v to @i with relaxed ordering.
	59	+ *
	60	+ * Unsafe to use in noinstr code; use raw_atomic_set() there.
	61	+ *
	62	+ * Return: Nothing.
	63	+ */
38	64	static __always_inline void
39	65	atomic_set(atomic_t *v, int i)
40	66	{
41	67	instrument_atomic_write(v, sizeof(*v));
42		− arch_atomic_set(v, i);
43		−}
44		−
	68	+ raw_atomic_set(v, i);
	69	+}
	70	+
	71	+/**
	72	+ * atomic_set_release() - atomic set with release ordering
	73	+ * @v: pointer to atomic_t
	74	+ * @i: int value to assign
	75	+ *
	76	+ * Atomically sets @v to @i with release ordering.
	77	+ *
	78	+ * Unsafe to use in noinstr code; use raw_atomic_set_release() there.
	79	+ *
	80	+ * Return: Nothing.
	81	+ */
45	82	static __always_inline void
46	83	atomic_set_release(atomic_t *v, int i)
47	84	{
48	85	kcsan_release();
49	86	instrument_atomic_write(v, sizeof(*v));
50		− arch_atomic_set_release(v, i);
51		−}
52		−
	87	+ raw_atomic_set_release(v, i);
	88	+}
	89	+
	90	+/**
	91	+ * atomic_add() - atomic add with relaxed ordering
	92	+ * @i: int value to add
	93	+ * @v: pointer to atomic_t
	94	+ *
	95	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	96	+ *
	97	+ * Unsafe to use in noinstr code; use raw_atomic_add() there.
	98	+ *
	99	+ * Return: Nothing.
	100	+ */
53	101	static __always_inline void
54	102	atomic_add(int i, atomic_t *v)
55	103	{
56	104	instrument_atomic_read_write(v, sizeof(*v));
57		− arch_atomic_add(i, v);
	105	+ raw_atomic_add(i, v);
58	106	}
59	107
	108	+/**
	109	+ * atomic_add_return() - atomic add with full ordering
	110	+ * @i: int value to add
	111	+ * @v: pointer to atomic_t
	112	+ *
	113	+ * Atomically updates @v to (@v + @i) with full ordering.
	114	+ *
	115	+ * Unsafe to use in noinstr code; use raw_atomic_add_return() there.
	116	+ *
	117	+ * Return: The updated value of @v.
	118	+ */
60	119	static __always_inline int
61	120	atomic_add_return(int i, atomic_t *v)
62	121	{
63	122	kcsan_mb();
64	123	instrument_atomic_read_write(v, sizeof(*v));
65		− return arch_atomic_add_return(i, v);
	124	+ return raw_atomic_add_return(i, v);
66	125	}
67	126
	127	+/**
	128	+ * atomic_add_return_acquire() - atomic add with acquire ordering
	129	+ * @i: int value to add
	130	+ * @v: pointer to atomic_t
	131	+ *
	132	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	133	+ *
	134	+ * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there.
	135	+ *
	136	+ * Return: The updated value of @v.
	137	+ */
68	138	static __always_inline int
69	139	atomic_add_return_acquire(int i, atomic_t *v)
70	140	{
71	141	instrument_atomic_read_write(v, sizeof(*v));
72		− return arch_atomic_add_return_acquire(i, v);
	142	+ return raw_atomic_add_return_acquire(i, v);
73	143	}
74	144
	145	+/**
	146	+ * atomic_add_return_release() - atomic add with release ordering
	147	+ * @i: int value to add
	148	+ * @v: pointer to atomic_t
	149	+ *
	150	+ * Atomically updates @v to (@v + @i) with release ordering.
	151	+ *
	152	+ * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there.
	153	+ *
	154	+ * Return: The updated value of @v.
	155	+ */
75	156	static __always_inline int
76	157	atomic_add_return_release(int i, atomic_t *v)
77	158	{
78	159	kcsan_release();
79	160	instrument_atomic_read_write(v, sizeof(*v));
80		− return arch_atomic_add_return_release(i, v);
	161	+ return raw_atomic_add_return_release(i, v);
81	162	}
82	163
	164	+/**
	165	+ * atomic_add_return_relaxed() - atomic add with relaxed ordering
	166	+ * @i: int value to add
	167	+ * @v: pointer to atomic_t
	168	+ *
	169	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	170	+ *
	171	+ * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there.
	172	+ *
	173	+ * Return: The updated value of @v.
	174	+ */
83	175	static __always_inline int
84	176	atomic_add_return_relaxed(int i, atomic_t *v)
85	177	{
86	178	instrument_atomic_read_write(v, sizeof(*v));
87		− return arch_atomic_add_return_relaxed(i, v);
	179	+ return raw_atomic_add_return_relaxed(i, v);
88	180	}
89	181
	182	+/**
	183	+ * atomic_fetch_add() - atomic add with full ordering
	184	+ * @i: int value to add
	185	+ * @v: pointer to atomic_t
	186	+ *
	187	+ * Atomically updates @v to (@v + @i) with full ordering.
	188	+ *
	189	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there.
	190	+ *
	191	+ * Return: The original value of @v.
	192	+ */
90	193	static __always_inline int
91	194	atomic_fetch_add(int i, atomic_t *v)
92	195	{
93	196	kcsan_mb();
94	197	instrument_atomic_read_write(v, sizeof(*v));
95		− return arch_atomic_fetch_add(i, v);
	198	+ return raw_atomic_fetch_add(i, v);
96	199	}
97	200
	201	+/**
	202	+ * atomic_fetch_add_acquire() - atomic add with acquire ordering
	203	+ * @i: int value to add
	204	+ * @v: pointer to atomic_t
	205	+ *
	206	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	207	+ *
	208	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there.
	209	+ *
	210	+ * Return: The original value of @v.
	211	+ */
98	212	static __always_inline int
99	213	atomic_fetch_add_acquire(int i, atomic_t *v)
100	214	{
101	215	instrument_atomic_read_write(v, sizeof(*v));
102		− return arch_atomic_fetch_add_acquire(i, v);
	216	+ return raw_atomic_fetch_add_acquire(i, v);
103	217	}
104	218
	219	+/**
	220	+ * atomic_fetch_add_release() - atomic add with release ordering
	221	+ * @i: int value to add
	222	+ * @v: pointer to atomic_t
	223	+ *
	224	+ * Atomically updates @v to (@v + @i) with release ordering.
	225	+ *
	226	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there.
	227	+ *
	228	+ * Return: The original value of @v.
	229	+ */
105	230	static __always_inline int
106	231	atomic_fetch_add_release(int i, atomic_t *v)
107	232	{
108	233	kcsan_release();
109	234	instrument_atomic_read_write(v, sizeof(*v));
110		− return arch_atomic_fetch_add_release(i, v);
	235	+ return raw_atomic_fetch_add_release(i, v);
111	236	}
112	237
	238	+/**
	239	+ * atomic_fetch_add_relaxed() - atomic add with relaxed ordering
	240	+ * @i: int value to add
	241	+ * @v: pointer to atomic_t
	242	+ *
	243	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	244	+ *
	245	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there.
	246	+ *
	247	+ * Return: The original value of @v.
	248	+ */
113	249	static __always_inline int
114	250	atomic_fetch_add_relaxed(int i, atomic_t *v)
115	251	{
116	252	instrument_atomic_read_write(v, sizeof(*v));
117		− return arch_atomic_fetch_add_relaxed(i, v);
	253	+ return raw_atomic_fetch_add_relaxed(i, v);
118	254	}
119	255
	256	+/**
	257	+ * atomic_sub() - atomic subtract with relaxed ordering
	258	+ * @i: int value to subtract
	259	+ * @v: pointer to atomic_t
	260	+ *
	261	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	262	+ *
	263	+ * Unsafe to use in noinstr code; use raw_atomic_sub() there.
	264	+ *
	265	+ * Return: Nothing.
	266	+ */
120	267	static __always_inline void
121	268	atomic_sub(int i, atomic_t *v)
122	269	{
123	270	instrument_atomic_read_write(v, sizeof(*v));
124		− arch_atomic_sub(i, v);
	271	+ raw_atomic_sub(i, v);
125	272	}
126	273
	274	+/**
	275	+ * atomic_sub_return() - atomic subtract with full ordering
	276	+ * @i: int value to subtract
	277	+ * @v: pointer to atomic_t
	278	+ *
	279	+ * Atomically updates @v to (@v - @i) with full ordering.
	280	+ *
	281	+ * Unsafe to use in noinstr code; use raw_atomic_sub_return() there.
	282	+ *
	283	+ * Return: The updated value of @v.
	284	+ */
127	285	static __always_inline int
128	286	atomic_sub_return(int i, atomic_t *v)
129	287	{
130	288	kcsan_mb();
131	289	instrument_atomic_read_write(v, sizeof(*v));
132		− return arch_atomic_sub_return(i, v);
	290	+ return raw_atomic_sub_return(i, v);
133	291	}
134	292
	293	+/**
	294	+ * atomic_sub_return_acquire() - atomic subtract with acquire ordering
	295	+ * @i: int value to subtract
	296	+ * @v: pointer to atomic_t
	297	+ *
	298	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	299	+ *
	300	+ * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there.
	301	+ *
	302	+ * Return: The updated value of @v.
	303	+ */
135	304	static __always_inline int
136	305	atomic_sub_return_acquire(int i, atomic_t *v)
137	306	{
138	307	instrument_atomic_read_write(v, sizeof(*v));
139		− return arch_atomic_sub_return_acquire(i, v);
	308	+ return raw_atomic_sub_return_acquire(i, v);
140	309	}
141	310
	311	+/**
	312	+ * atomic_sub_return_release() - atomic subtract with release ordering
	313	+ * @i: int value to subtract
	314	+ * @v: pointer to atomic_t
	315	+ *
	316	+ * Atomically updates @v to (@v - @i) with release ordering.
	317	+ *
	318	+ * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there.
	319	+ *
	320	+ * Return: The updated value of @v.
	321	+ */
142	322	static __always_inline int
143	323	atomic_sub_return_release(int i, atomic_t *v)
144	324	{
145	325	kcsan_release();
146	326	instrument_atomic_read_write(v, sizeof(*v));
147		− return arch_atomic_sub_return_release(i, v);
	327	+ return raw_atomic_sub_return_release(i, v);
148	328	}
149	329
	330	+/**
	331	+ * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
	332	+ * @i: int value to subtract
	333	+ * @v: pointer to atomic_t
	334	+ *
	335	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	336	+ *
	337	+ * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there.
	338	+ *
	339	+ * Return: The updated value of @v.
	340	+ */
150	341	static __always_inline int
151	342	atomic_sub_return_relaxed(int i, atomic_t *v)
152	343	{
153	344	instrument_atomic_read_write(v, sizeof(*v));
154		− return arch_atomic_sub_return_relaxed(i, v);
	345	+ return raw_atomic_sub_return_relaxed(i, v);
155	346	}
156	347
	348	+/**
	349	+ * atomic_fetch_sub() - atomic subtract with full ordering
	350	+ * @i: int value to subtract
	351	+ * @v: pointer to atomic_t
	352	+ *
	353	+ * Atomically updates @v to (@v - @i) with full ordering.
	354	+ *
	355	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there.
	356	+ *
	357	+ * Return: The original value of @v.
	358	+ */
157	359	static __always_inline int
158	360	atomic_fetch_sub(int i, atomic_t *v)
159	361	{
160	362	kcsan_mb();
161	363	instrument_atomic_read_write(v, sizeof(*v));
162		− return arch_atomic_fetch_sub(i, v);
	364	+ return raw_atomic_fetch_sub(i, v);
163	365	}
164	366
	367	+/**
	368	+ * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
	369	+ * @i: int value to subtract
	370	+ * @v: pointer to atomic_t
	371	+ *
	372	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	373	+ *
	374	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there.
	375	+ *
	376	+ * Return: The original value of @v.
	377	+ */
165	378	static __always_inline int
166	379	atomic_fetch_sub_acquire(int i, atomic_t *v)
167	380	{
168	381	instrument_atomic_read_write(v, sizeof(*v));
169		− return arch_atomic_fetch_sub_acquire(i, v);
	382	+ return raw_atomic_fetch_sub_acquire(i, v);
170	383	}
171	384
	385	+/**
	386	+ * atomic_fetch_sub_release() - atomic subtract with release ordering
	387	+ * @i: int value to subtract
	388	+ * @v: pointer to atomic_t
	389	+ *
	390	+ * Atomically updates @v to (@v - @i) with release ordering.
	391	+ *
	392	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there.
	393	+ *
	394	+ * Return: The original value of @v.
	395	+ */
172	396	static __always_inline int
173	397	atomic_fetch_sub_release(int i, atomic_t *v)
174	398	{
175	399	kcsan_release();
176	400	instrument_atomic_read_write(v, sizeof(*v));
177		− return arch_atomic_fetch_sub_release(i, v);
	401	+ return raw_atomic_fetch_sub_release(i, v);
178	402	}
179	403
	404	+/**
	405	+ * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
	406	+ * @i: int value to subtract
	407	+ * @v: pointer to atomic_t
	408	+ *
	409	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	410	+ *
	411	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there.
	412	+ *
	413	+ * Return: The original value of @v.
	414	+ */
180	415	static __always_inline int
181	416	atomic_fetch_sub_relaxed(int i, atomic_t *v)
182	417	{
183	418	instrument_atomic_read_write(v, sizeof(*v));
184		− return arch_atomic_fetch_sub_relaxed(i, v);
	419	+ return raw_atomic_fetch_sub_relaxed(i, v);
185	420	}
186	421
	422	+/**
	423	+ * atomic_inc() - atomic increment with relaxed ordering
	424	+ * @v: pointer to atomic_t
	425	+ *
	426	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	427	+ *
	428	+ * Unsafe to use in noinstr code; use raw_atomic_inc() there.
	429	+ *
	430	+ * Return: Nothing.
	431	+ */
187	432	static __always_inline void
188	433	atomic_inc(atomic_t *v)
189	434	{
190	435	instrument_atomic_read_write(v, sizeof(*v));
191		− arch_atomic_inc(v);
	436	+ raw_atomic_inc(v);
192	437	}
193	438
	439	+/**
	440	+ * atomic_inc_return() - atomic increment with full ordering
	441	+ * @v: pointer to atomic_t
	442	+ *
	443	+ * Atomically updates @v to (@v + 1) with full ordering.
	444	+ *
	445	+ * Unsafe to use in noinstr code; use raw_atomic_inc_return() there.
	446	+ *
	447	+ * Return: The updated value of @v.
	448	+ */
194	449	static __always_inline int
195	450	atomic_inc_return(atomic_t *v)
196	451	{
197	452	kcsan_mb();
198	453	instrument_atomic_read_write(v, sizeof(*v));
199		− return arch_atomic_inc_return(v);
	454	+ return raw_atomic_inc_return(v);
200	455	}
201	456
	457	+/**
	458	+ * atomic_inc_return_acquire() - atomic increment with acquire ordering
	459	+ * @v: pointer to atomic_t
	460	+ *
	461	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	462	+ *
	463	+ * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there.
	464	+ *
	465	+ * Return: The updated value of @v.
	466	+ */
202	467	static __always_inline int
203	468	atomic_inc_return_acquire(atomic_t *v)
204	469	{
205	470	instrument_atomic_read_write(v, sizeof(*v));
206		− return arch_atomic_inc_return_acquire(v);
	471	+ return raw_atomic_inc_return_acquire(v);
207	472	}
208	473
	474	+/**
	475	+ * atomic_inc_return_release() - atomic increment with release ordering
	476	+ * @v: pointer to atomic_t
	477	+ *
	478	+ * Atomically updates @v to (@v + 1) with release ordering.
	479	+ *
	480	+ * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there.
	481	+ *
	482	+ * Return: The updated value of @v.
	483	+ */
209	484	static __always_inline int
210	485	atomic_inc_return_release(atomic_t *v)
211	486	{
212	487	kcsan_release();
213	488	instrument_atomic_read_write(v, sizeof(*v));
214		− return arch_atomic_inc_return_release(v);
	489	+ return raw_atomic_inc_return_release(v);
215	490	}
216	491
	492	+/**
	493	+ * atomic_inc_return_relaxed() - atomic increment with relaxed ordering
	494	+ * @v: pointer to atomic_t
	495	+ *
	496	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	497	+ *
	498	+ * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there.
	499	+ *
	500	+ * Return: The updated value of @v.
	501	+ */
217	502	static __always_inline int
218	503	atomic_inc_return_relaxed(atomic_t *v)
219	504	{
220	505	instrument_atomic_read_write(v, sizeof(*v));
221		− return arch_atomic_inc_return_relaxed(v);
	506	+ return raw_atomic_inc_return_relaxed(v);
222	507	}
223	508
	509	+/**
	510	+ * atomic_fetch_inc() - atomic increment with full ordering
	511	+ * @v: pointer to atomic_t
	512	+ *
	513	+ * Atomically updates @v to (@v + 1) with full ordering.
	514	+ *
	515	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there.
	516	+ *
	517	+ * Return: The original value of @v.
	518	+ */
224	519	static __always_inline int
225	520	atomic_fetch_inc(atomic_t *v)
226	521	{
227	522	kcsan_mb();
228	523	instrument_atomic_read_write(v, sizeof(*v));
229		− return arch_atomic_fetch_inc(v);
	524	+ return raw_atomic_fetch_inc(v);
230	525	}
231	526
	527	+/**
	528	+ * atomic_fetch_inc_acquire() - atomic increment with acquire ordering
	529	+ * @v: pointer to atomic_t
	530	+ *
	531	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	532	+ *
	533	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there.
	534	+ *
	535	+ * Return: The original value of @v.
	536	+ */
232	537	static __always_inline int
233	538	atomic_fetch_inc_acquire(atomic_t *v)
234	539	{
235	540	instrument_atomic_read_write(v, sizeof(*v));
236		− return arch_atomic_fetch_inc_acquire(v);
	541	+ return raw_atomic_fetch_inc_acquire(v);
237	542	}
238	543
	544	+/**
	545	+ * atomic_fetch_inc_release() - atomic increment with release ordering
	546	+ * @v: pointer to atomic_t
	547	+ *
	548	+ * Atomically updates @v to (@v + 1) with release ordering.
	549	+ *
	550	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there.
	551	+ *
	552	+ * Return: The original value of @v.
	553	+ */
239	554	static __always_inline int
240	555	atomic_fetch_inc_release(atomic_t *v)
241	556	{
242	557	kcsan_release();
243	558	instrument_atomic_read_write(v, sizeof(*v));
244		− return arch_atomic_fetch_inc_release(v);
	559	+ return raw_atomic_fetch_inc_release(v);
245	560	}
246	561
	562	+/**
	563	+ * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
	564	+ * @v: pointer to atomic_t
	565	+ *
	566	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	567	+ *
	568	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there.
	569	+ *
	570	+ * Return: The original value of @v.
	571	+ */
247	572	static __always_inline int
248	573	atomic_fetch_inc_relaxed(atomic_t *v)
249	574	{
250	575	instrument_atomic_read_write(v, sizeof(*v));
251		− return arch_atomic_fetch_inc_relaxed(v);
	576	+ return raw_atomic_fetch_inc_relaxed(v);
252	577	}
253	578
	579	+/**
	580	+ * atomic_dec() - atomic decrement with relaxed ordering
	581	+ * @v: pointer to atomic_t
	582	+ *
	583	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	584	+ *
	585	+ * Unsafe to use in noinstr code; use raw_atomic_dec() there.
	586	+ *
	587	+ * Return: Nothing.
	588	+ */
254	589	static __always_inline void
255	590	atomic_dec(atomic_t *v)
256	591	{
257	592	instrument_atomic_read_write(v, sizeof(*v));
258		− arch_atomic_dec(v);
	593	+ raw_atomic_dec(v);
259	594	}
260	595
	596	+/**
	597	+ * atomic_dec_return() - atomic decrement with full ordering
	598	+ * @v: pointer to atomic_t
	599	+ *
	600	+ * Atomically updates @v to (@v - 1) with full ordering.
	601	+ *
	602	+ * Unsafe to use in noinstr code; use raw_atomic_dec_return() there.
	603	+ *
	604	+ * Return: The updated value of @v.
	605	+ */
261	606	static __always_inline int
262	607	atomic_dec_return(atomic_t *v)
263	608	{
264	609	kcsan_mb();
265	610	instrument_atomic_read_write(v, sizeof(*v));
266		− return arch_atomic_dec_return(v);
	611	+ return raw_atomic_dec_return(v);
267	612	}
268	613
	614	+/**
	615	+ * atomic_dec_return_acquire() - atomic decrement with acquire ordering
	616	+ * @v: pointer to atomic_t
	617	+ *
	618	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	619	+ *
	620	+ * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there.
	621	+ *
	622	+ * Return: The updated value of @v.
	623	+ */
269	624	static __always_inline int
270	625	atomic_dec_return_acquire(atomic_t *v)
271	626	{
272	627	instrument_atomic_read_write(v, sizeof(*v));
273		− return arch_atomic_dec_return_acquire(v);
	628	+ return raw_atomic_dec_return_acquire(v);
274	629	}
275	630
	631	+/**
	632	+ * atomic_dec_return_release() - atomic decrement with release ordering
	633	+ * @v: pointer to atomic_t
	634	+ *
	635	+ * Atomically updates @v to (@v - 1) with release ordering.
	636	+ *
	637	+ * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there.
	638	+ *
	639	+ * Return: The updated value of @v.
	640	+ */
276	641	static __always_inline int
277	642	atomic_dec_return_release(atomic_t *v)
278	643	{
279	644	kcsan_release();
280	645	instrument_atomic_read_write(v, sizeof(*v));
281		− return arch_atomic_dec_return_release(v);
	646	+ return raw_atomic_dec_return_release(v);
282	647	}
283	648
	649	+/**
	650	+ * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
	651	+ * @v: pointer to atomic_t
	652	+ *
	653	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	654	+ *
	655	+ * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there.
	656	+ *
	657	+ * Return: The updated value of @v.
	658	+ */
284	659	static __always_inline int
285	660	atomic_dec_return_relaxed(atomic_t *v)
286	661	{
287	662	instrument_atomic_read_write(v, sizeof(*v));
288		− return arch_atomic_dec_return_relaxed(v);
	663	+ return raw_atomic_dec_return_relaxed(v);
289	664	}
290	665
	666	+/**
	667	+ * atomic_fetch_dec() - atomic decrement with full ordering
	668	+ * @v: pointer to atomic_t
	669	+ *
	670	+ * Atomically updates @v to (@v - 1) with full ordering.
	671	+ *
	672	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there.
	673	+ *
	674	+ * Return: The original value of @v.
	675	+ */
291	676	static __always_inline int
292	677	atomic_fetch_dec(atomic_t *v)
293	678	{
294	679	kcsan_mb();
295	680	instrument_atomic_read_write(v, sizeof(*v));
296		− return arch_atomic_fetch_dec(v);
	681	+ return raw_atomic_fetch_dec(v);
297	682	}
298	683
	684	+/**
	685	+ * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
	686	+ * @v: pointer to atomic_t
	687	+ *
	688	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	689	+ *
	690	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there.
	691	+ *
	692	+ * Return: The original value of @v.
	693	+ */
299	694	static __always_inline int
300	695	atomic_fetch_dec_acquire(atomic_t *v)
301	696	{
302	697	instrument_atomic_read_write(v, sizeof(*v));
303		− return arch_atomic_fetch_dec_acquire(v);
	698	+ return raw_atomic_fetch_dec_acquire(v);
304	699	}
305	700
	701	+/**
	702	+ * atomic_fetch_dec_release() - atomic decrement with release ordering
	703	+ * @v: pointer to atomic_t
	704	+ *
	705	+ * Atomically updates @v to (@v - 1) with release ordering.
	706	+ *
	707	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there.
	708	+ *
	709	+ * Return: The original value of @v.
	710	+ */
306	711	static __always_inline int
307	712	atomic_fetch_dec_release(atomic_t *v)
308	713	{
309	714	kcsan_release();
310	715	instrument_atomic_read_write(v, sizeof(*v));
311		− return arch_atomic_fetch_dec_release(v);
	716	+ return raw_atomic_fetch_dec_release(v);
312	717	}
313	718
	719	+/**
	720	+ * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
	721	+ * @v: pointer to atomic_t
	722	+ *
	723	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	724	+ *
	725	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there.
	726	+ *
	727	+ * Return: The original value of @v.
	728	+ */
314	729	static __always_inline int
315	730	atomic_fetch_dec_relaxed(atomic_t *v)
316	731	{
317	732	instrument_atomic_read_write(v, sizeof(*v));
318		− return arch_atomic_fetch_dec_relaxed(v);
	733	+ return raw_atomic_fetch_dec_relaxed(v);
319	734	}
320	735
	736	+/**
	737	+ * atomic_and() - atomic bitwise AND with relaxed ordering
	738	+ * @i: int value
	739	+ * @v: pointer to atomic_t
	740	+ *
	741	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	742	+ *
	743	+ * Unsafe to use in noinstr code; use raw_atomic_and() there.
	744	+ *
	745	+ * Return: Nothing.
	746	+ */
321	747	static __always_inline void
322	748	atomic_and(int i, atomic_t *v)
323	749	{
324	750	instrument_atomic_read_write(v, sizeof(*v));
325		− arch_atomic_and(i, v);
	751	+ raw_atomic_and(i, v);
326	752	}
327	753
	754	+/**
	755	+ * atomic_fetch_and() - atomic bitwise AND with full ordering
	756	+ * @i: int value
	757	+ * @v: pointer to atomic_t
	758	+ *
	759	+ * Atomically updates @v to (@v & @i) with full ordering.
	760	+ *
	761	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there.
	762	+ *
	763	+ * Return: The original value of @v.
	764	+ */
328	765	static __always_inline int
329	766	atomic_fetch_and(int i, atomic_t *v)
330	767	{
331	768	kcsan_mb();
332	769	instrument_atomic_read_write(v, sizeof(*v));
333		− return arch_atomic_fetch_and(i, v);
	770	+ return raw_atomic_fetch_and(i, v);
334	771	}
335	772
	773	+/**
	774	+ * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
	775	+ * @i: int value
	776	+ * @v: pointer to atomic_t
	777	+ *
	778	+ * Atomically updates @v to (@v & @i) with acquire ordering.
	779	+ *
	780	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there.
	781	+ *
	782	+ * Return: The original value of @v.
	783	+ */
336	784	static __always_inline int
337	785	atomic_fetch_and_acquire(int i, atomic_t *v)
338	786	{
339	787	instrument_atomic_read_write(v, sizeof(*v));
340		− return arch_atomic_fetch_and_acquire(i, v);
	788	+ return raw_atomic_fetch_and_acquire(i, v);
341	789	}
342	790
	791	+/**
	792	+ * atomic_fetch_and_release() - atomic bitwise AND with release ordering
	793	+ * @i: int value
	794	+ * @v: pointer to atomic_t
	795	+ *
	796	+ * Atomically updates @v to (@v & @i) with release ordering.
	797	+ *
	798	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there.
	799	+ *
	800	+ * Return: The original value of @v.
	801	+ */
343	802	static __always_inline int
344	803	atomic_fetch_and_release(int i, atomic_t *v)
345	804	{
346	805	kcsan_release();
347	806	instrument_atomic_read_write(v, sizeof(*v));
348		− return arch_atomic_fetch_and_release(i, v);
	807	+ return raw_atomic_fetch_and_release(i, v);
349	808	}
350	809
	810	+/**
	811	+ * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
	812	+ * @i: int value
	813	+ * @v: pointer to atomic_t
	814	+ *
	815	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	816	+ *
	817	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there.
	818	+ *
	819	+ * Return: The original value of @v.
	820	+ */
351	821	static __always_inline int
352	822	atomic_fetch_and_relaxed(int i, atomic_t *v)
353	823	{
354	824	instrument_atomic_read_write(v, sizeof(*v));
355		− return arch_atomic_fetch_and_relaxed(i, v);
	825	+ return raw_atomic_fetch_and_relaxed(i, v);
356	826	}
357	827
	828	+/**
	829	+ * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
	830	+ * @i: int value
	831	+ * @v: pointer to atomic_t
	832	+ *
	833	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	834	+ *
	835	+ * Unsafe to use in noinstr code; use raw_atomic_andnot() there.
	836	+ *
	837	+ * Return: Nothing.
	838	+ */
358	839	static __always_inline void
359	840	atomic_andnot(int i, atomic_t *v)
360	841	{
361	842	instrument_atomic_read_write(v, sizeof(*v));
362		− arch_atomic_andnot(i, v);
	843	+ raw_atomic_andnot(i, v);
363	844	}
364	845
	846	+/**
	847	+ * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
	848	+ * @i: int value
	849	+ * @v: pointer to atomic_t
	850	+ *
	851	+ * Atomically updates @v to (@v & ~@i) with full ordering.
	852	+ *
	853	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there.
	854	+ *
	855	+ * Return: The original value of @v.
	856	+ */
365	857	static __always_inline int
366	858	atomic_fetch_andnot(int i, atomic_t *v)
367	859	{
368	860	kcsan_mb();
369	861	instrument_atomic_read_write(v, sizeof(*v));
370		− return arch_atomic_fetch_andnot(i, v);
	862	+ return raw_atomic_fetch_andnot(i, v);
371	863	}
372	864
	865	+/**
	866	+ * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
	867	+ * @i: int value
	868	+ * @v: pointer to atomic_t
	869	+ *
	870	+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
	871	+ *
	872	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there.
	873	+ *
	874	+ * Return: The original value of @v.
	875	+ */
373	876	static __always_inline int
374	877	atomic_fetch_andnot_acquire(int i, atomic_t *v)
375	878	{
376	879	instrument_atomic_read_write(v, sizeof(*v));
377		− return arch_atomic_fetch_andnot_acquire(i, v);
	880	+ return raw_atomic_fetch_andnot_acquire(i, v);
378	881	}
379	882
	883	+/**
	884	+ * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
	885	+ * @i: int value
	886	+ * @v: pointer to atomic_t
	887	+ *
	888	+ * Atomically updates @v to (@v & ~@i) with release ordering.
	889	+ *
	890	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there.
	891	+ *
	892	+ * Return: The original value of @v.
	893	+ */
380	894	static __always_inline int
381	895	atomic_fetch_andnot_release(int i, atomic_t *v)
382	896	{
383	897	kcsan_release();
384	898	instrument_atomic_read_write(v, sizeof(*v));
385		− return arch_atomic_fetch_andnot_release(i, v);
	899	+ return raw_atomic_fetch_andnot_release(i, v);
386	900	}
387	901
	902	+/**
	903	+ * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
	904	+ * @i: int value
	905	+ * @v: pointer to atomic_t
	906	+ *
	907	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	908	+ *
	909	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there.
	910	+ *
	911	+ * Return: The original value of @v.
	912	+ */
388	913	static __always_inline int
389	914	atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390	915	{
391	916	instrument_atomic_read_write(v, sizeof(*v));
392		− return arch_atomic_fetch_andnot_relaxed(i, v);
	917	+ return raw_atomic_fetch_andnot_relaxed(i, v);
393	918	}
394	919
	920	+/**
	921	+ * atomic_or() - atomic bitwise OR with relaxed ordering
	922	+ * @i: int value
	923	+ * @v: pointer to atomic_t
	924	+ *
	925	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	926	+ *
	927	+ * Unsafe to use in noinstr code; use raw_atomic_or() there.
	928	+ *
	929	+ * Return: Nothing.
	930	+ */
395	931	static __always_inline void
396	932	atomic_or(int i, atomic_t *v)
397	933	{
398	934	instrument_atomic_read_write(v, sizeof(*v));
399		− arch_atomic_or(i, v);
	935	+ raw_atomic_or(i, v);
400	936	}
401	937
	938	+/**
	939	+ * atomic_fetch_or() - atomic bitwise OR with full ordering
	940	+ * @i: int value
	941	+ * @v: pointer to atomic_t
	942	+ *
	943	+ * Atomically updates @v to (@v \| @i) with full ordering.
	944	+ *
	945	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there.
	946	+ *
	947	+ * Return: The original value of @v.
	948	+ */
402	949	static __always_inline int
403	950	atomic_fetch_or(int i, atomic_t *v)
404	951	{
405	952	kcsan_mb();
406	953	instrument_atomic_read_write(v, sizeof(*v));
407		− return arch_atomic_fetch_or(i, v);
	954	+ return raw_atomic_fetch_or(i, v);
408	955	}
409	956
	957	+/**
	958	+ * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
	959	+ * @i: int value
	960	+ * @v: pointer to atomic_t
	961	+ *
	962	+ * Atomically updates @v to (@v \| @i) with acquire ordering.
	963	+ *
	964	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there.
	965	+ *
	966	+ * Return: The original value of @v.
	967	+ */
410	968	static __always_inline int
411	969	atomic_fetch_or_acquire(int i, atomic_t *v)
412	970	{
413	971	instrument_atomic_read_write(v, sizeof(*v));
414		− return arch_atomic_fetch_or_acquire(i, v);
	972	+ return raw_atomic_fetch_or_acquire(i, v);
415	973	}
416	974
	975	+/**
	976	+ * atomic_fetch_or_release() - atomic bitwise OR with release ordering
	977	+ * @i: int value
	978	+ * @v: pointer to atomic_t
	979	+ *
	980	+ * Atomically updates @v to (@v \| @i) with release ordering.
	981	+ *
	982	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there.
	983	+ *
	984	+ * Return: The original value of @v.
	985	+ */
417	986	static __always_inline int
418	987	atomic_fetch_or_release(int i, atomic_t *v)
419	988	{
420	989	kcsan_release();
421	990	instrument_atomic_read_write(v, sizeof(*v));
422		− return arch_atomic_fetch_or_release(i, v);
	991	+ return raw_atomic_fetch_or_release(i, v);
423	992	}
424	993
	994	+/**
	995	+ * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
	996	+ * @i: int value
	997	+ * @v: pointer to atomic_t
	998	+ *
	999	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	1000	+ *
	1001	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there.
	1002	+ *
	1003	+ * Return: The original value of @v.
	1004	+ */
425	1005	static __always_inline int
426	1006	atomic_fetch_or_relaxed(int i, atomic_t *v)
427	1007	{
428	1008	instrument_atomic_read_write(v, sizeof(*v));
429		− return arch_atomic_fetch_or_relaxed(i, v);
	1009	+ return raw_atomic_fetch_or_relaxed(i, v);
430	1010	}
431	1011
	1012	+/**
	1013	+ * atomic_xor() - atomic bitwise XOR with relaxed ordering
	1014	+ * @i: int value
	1015	+ * @v: pointer to atomic_t
	1016	+ *
	1017	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	1018	+ *
	1019	+ * Unsafe to use in noinstr code; use raw_atomic_xor() there.
	1020	+ *
	1021	+ * Return: Nothing.
	1022	+ */
432	1023	static __always_inline void
433	1024	atomic_xor(int i, atomic_t *v)
434	1025	{
435	1026	instrument_atomic_read_write(v, sizeof(*v));
436		− arch_atomic_xor(i, v);
	1027	+ raw_atomic_xor(i, v);
437	1028	}
438	1029
	1030	+/**
	1031	+ * atomic_fetch_xor() - atomic bitwise XOR with full ordering
	1032	+ * @i: int value
	1033	+ * @v: pointer to atomic_t
	1034	+ *
	1035	+ * Atomically updates @v to (@v ^ @i) with full ordering.
	1036	+ *
	1037	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there.
	1038	+ *
	1039	+ * Return: The original value of @v.
	1040	+ */
439	1041	static __always_inline int
440	1042	atomic_fetch_xor(int i, atomic_t *v)
441	1043	{
442	1044	kcsan_mb();
443	1045	instrument_atomic_read_write(v, sizeof(*v));
444		− return arch_atomic_fetch_xor(i, v);
	1046	+ return raw_atomic_fetch_xor(i, v);
445	1047	}
446	1048
	1049	+/**
	1050	+ * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
	1051	+ * @i: int value
	1052	+ * @v: pointer to atomic_t
	1053	+ *
	1054	+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
	1055	+ *
	1056	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there.
	1057	+ *
	1058	+ * Return: The original value of @v.
	1059	+ */
447	1060	static __always_inline int
448	1061	atomic_fetch_xor_acquire(int i, atomic_t *v)
449	1062	{
450	1063	instrument_atomic_read_write(v, sizeof(*v));
451		− return arch_atomic_fetch_xor_acquire(i, v);
	1064	+ return raw_atomic_fetch_xor_acquire(i, v);
452	1065	}
453	1066
	1067	+/**
	1068	+ * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
	1069	+ * @i: int value
	1070	+ * @v: pointer to atomic_t
	1071	+ *
	1072	+ * Atomically updates @v to (@v ^ @i) with release ordering.
	1073	+ *
	1074	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there.
	1075	+ *
	1076	+ * Return: The original value of @v.
	1077	+ */
454	1078	static __always_inline int
455	1079	atomic_fetch_xor_release(int i, atomic_t *v)
456	1080	{
457	1081	kcsan_release();
458	1082	instrument_atomic_read_write(v, sizeof(*v));
459		− return arch_atomic_fetch_xor_release(i, v);
	1083	+ return raw_atomic_fetch_xor_release(i, v);
460	1084	}
461	1085
	1086	+/**
	1087	+ * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
	1088	+ * @i: int value
	1089	+ * @v: pointer to atomic_t
	1090	+ *
	1091	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	1092	+ *
	1093	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there.
	1094	+ *
	1095	+ * Return: The original value of @v.
	1096	+ */
462	1097	static __always_inline int
463	1098	atomic_fetch_xor_relaxed(int i, atomic_t *v)
464	1099	{
465	1100	instrument_atomic_read_write(v, sizeof(*v));
466		− return arch_atomic_fetch_xor_relaxed(i, v);
	1101	+ return raw_atomic_fetch_xor_relaxed(i, v);
467	1102	}
468	1103
	1104	+/**
	1105	+ * atomic_xchg() - atomic exchange with full ordering
	1106	+ * @v: pointer to atomic_t
	1107	+ * @new: int value to assign
	1108	+ *
	1109	+ * Atomically updates @v to @new with full ordering.
	1110	+ *
	1111	+ * Unsafe to use in noinstr code; use raw_atomic_xchg() there.
	1112	+ *
	1113	+ * Return: The original value of @v.
	1114	+ */
469	1115	static __always_inline int
470		−atomic_xchg(atomic_t *v, int i)
	1116	+atomic_xchg(atomic_t *v, int new)
471	1117	{
472	1118	kcsan_mb();
473	1119	instrument_atomic_read_write(v, sizeof(*v));
474		− return arch_atomic_xchg(v, i);
	1120	+ return raw_atomic_xchg(v, new);
475	1121	}
476	1122
	1123	+/**
	1124	+ * atomic_xchg_acquire() - atomic exchange with acquire ordering
	1125	+ * @v: pointer to atomic_t
	1126	+ * @new: int value to assign
	1127	+ *
	1128	+ * Atomically updates @v to @new with acquire ordering.
	1129	+ *
	1130	+ * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there.
	1131	+ *
	1132	+ * Return: The original value of @v.
	1133	+ */
477	1134	static __always_inline int
478		−atomic_xchg_acquire(atomic_t *v, int i)
	1135	+atomic_xchg_acquire(atomic_t *v, int new)
479	1136	{
480	1137	instrument_atomic_read_write(v, sizeof(*v));
481		− return arch_atomic_xchg_acquire(v, i);
	1138	+ return raw_atomic_xchg_acquire(v, new);
482	1139	}
483	1140
	1141	+/**
	1142	+ * atomic_xchg_release() - atomic exchange with release ordering
	1143	+ * @v: pointer to atomic_t
	1144	+ * @new: int value to assign
	1145	+ *
	1146	+ * Atomically updates @v to @new with release ordering.
	1147	+ *
	1148	+ * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there.
	1149	+ *
	1150	+ * Return: The original value of @v.
	1151	+ */
484	1152	static __always_inline int
485		−atomic_xchg_release(atomic_t *v, int i)
	1153	+atomic_xchg_release(atomic_t *v, int new)
486	1154	{
487	1155	kcsan_release();
488	1156	instrument_atomic_read_write(v, sizeof(*v));
489		− return arch_atomic_xchg_release(v, i);
	1157	+ return raw_atomic_xchg_release(v, new);
490	1158	}
491	1159
	1160	+/**
	1161	+ * atomic_xchg_relaxed() - atomic exchange with relaxed ordering
	1162	+ * @v: pointer to atomic_t
	1163	+ * @new: int value to assign
	1164	+ *
	1165	+ * Atomically updates @v to @new with relaxed ordering.
	1166	+ *
	1167	+ * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there.
	1168	+ *
	1169	+ * Return: The original value of @v.
	1170	+ */
492	1171	static __always_inline int
493		−atomic_xchg_relaxed(atomic_t *v, int i)
	1172	+atomic_xchg_relaxed(atomic_t *v, int new)
494	1173	{
495	1174	instrument_atomic_read_write(v, sizeof(*v));
496		− return arch_atomic_xchg_relaxed(v, i);
	1175	+ return raw_atomic_xchg_relaxed(v, new);
497	1176	}
498	1177
	1178	+/**
	1179	+ * atomic_cmpxchg() - atomic compare and exchange with full ordering
	1180	+ * @v: pointer to atomic_t
	1181	+ * @old: int value to compare with
	1182	+ * @new: int value to assign
	1183	+ *
	1184	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	1185	+ *
	1186	+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there.
	1187	+ *
	1188	+ * Return: The original value of @v.
	1189	+ */
499	1190	static __always_inline int
500	1191	atomic_cmpxchg(atomic_t *v, int old, int new)
501	1192	{
502	1193	kcsan_mb();
503	1194	instrument_atomic_read_write(v, sizeof(*v));
504		− return arch_atomic_cmpxchg(v, old, new);
	1195	+ return raw_atomic_cmpxchg(v, old, new);
505	1196	}
506	1197
	1198	+/**
	1199	+ * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	1200	+ * @v: pointer to atomic_t
	1201	+ * @old: int value to compare with
	1202	+ * @new: int value to assign
	1203	+ *
	1204	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	1205	+ *
	1206	+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there.
	1207	+ *
	1208	+ * Return: The original value of @v.
	1209	+ */
507	1210	static __always_inline int
508	1211	atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509	1212	{
510	1213	instrument_atomic_read_write(v, sizeof(*v));
511		− return arch_atomic_cmpxchg_acquire(v, old, new);
	1214	+ return raw_atomic_cmpxchg_acquire(v, old, new);
512	1215	}
513	1216
	1217	+/**
	1218	+ * atomic_cmpxchg_release() - atomic compare and exchange with release ordering
	1219	+ * @v: pointer to atomic_t
	1220	+ * @old: int value to compare with
	1221	+ * @new: int value to assign
	1222	+ *
	1223	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	1224	+ *
	1225	+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there.
	1226	+ *
	1227	+ * Return: The original value of @v.
	1228	+ */
514	1229	static __always_inline int
515	1230	atomic_cmpxchg_release(atomic_t *v, int old, int new)
516	1231	{
517	1232	kcsan_release();
518	1233	instrument_atomic_read_write(v, sizeof(*v));
519		− return arch_atomic_cmpxchg_release(v, old, new);
	1234	+ return raw_atomic_cmpxchg_release(v, old, new);
520	1235	}
521	1236
	1237	+/**
	1238	+ * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	1239	+ * @v: pointer to atomic_t
	1240	+ * @old: int value to compare with
	1241	+ * @new: int value to assign
	1242	+ *
	1243	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	1244	+ *
	1245	+ * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there.
	1246	+ *
	1247	+ * Return: The original value of @v.
	1248	+ */
522	1249	static __always_inline int
523	1250	atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524	1251	{
525	1252	instrument_atomic_read_write(v, sizeof(*v));
526		− return arch_atomic_cmpxchg_relaxed(v, old, new);
	1253	+ return raw_atomic_cmpxchg_relaxed(v, old, new);
527	1254	}
528	1255
	1256	+/**
	1257	+ * atomic_try_cmpxchg() - atomic compare and exchange with full ordering
	1258	+ * @v: pointer to atomic_t
	1259	+ * @old: pointer to int value to compare with
	1260	+ * @new: int value to assign
	1261	+ *
	1262	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	1263	+ * Otherwise, updates @old to the current value of @v.
	1264	+ *
	1265	+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there.
	1266	+ *
	1267	+ * Return: @true if the exchange occured, @false otherwise.
	1268	+ */
529	1269	static __always_inline bool
530	1270	atomic_try_cmpxchg(atomic_t v, int old, int new)
531	1271	{
532	1272	kcsan_mb();
533	1273	instrument_atomic_read_write(v, sizeof(*v));
534	1274	instrument_atomic_read_write(old, sizeof(*old));
535		− return arch_atomic_try_cmpxchg(v, old, new);
536		−}
537		−
	1275	+ return raw_atomic_try_cmpxchg(v, old, new);
	1276	+}
	1277	+
	1278	+/**
	1279	+ * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	1280	+ * @v: pointer to atomic_t
	1281	+ * @old: pointer to int value to compare with
	1282	+ * @new: int value to assign
	1283	+ *
	1284	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	1285	+ * Otherwise, updates @old to the current value of @v.
	1286	+ *
	1287	+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there.
	1288	+ *
	1289	+ * Return: @true if the exchange occured, @false otherwise.
	1290	+ */
538	1291	static __always_inline bool
539	1292	atomic_try_cmpxchg_acquire(atomic_t v, int old, int new)
540	1293	{
541	1294	instrument_atomic_read_write(v, sizeof(*v));
542	1295	instrument_atomic_read_write(old, sizeof(*old));
543		− return arch_atomic_try_cmpxchg_acquire(v, old, new);
544		−}
545		−
	1296	+ return raw_atomic_try_cmpxchg_acquire(v, old, new);
	1297	+}
	1298	+
	1299	+/**
	1300	+ * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
	1301	+ * @v: pointer to atomic_t
	1302	+ * @old: pointer to int value to compare with
	1303	+ * @new: int value to assign
	1304	+ *
	1305	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	1306	+ * Otherwise, updates @old to the current value of @v.
	1307	+ *
	1308	+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there.
	1309	+ *
	1310	+ * Return: @true if the exchange occured, @false otherwise.
	1311	+ */
546	1312	static __always_inline bool
547	1313	atomic_try_cmpxchg_release(atomic_t v, int old, int new)
548	1314	{
549	1315	kcsan_release();
550	1316	instrument_atomic_read_write(v, sizeof(*v));
551	1317	instrument_atomic_read_write(old, sizeof(*old));
552		− return arch_atomic_try_cmpxchg_release(v, old, new);
553		−}
554		−
	1318	+ return raw_atomic_try_cmpxchg_release(v, old, new);
	1319	+}
	1320	+
	1321	+/**
	1322	+ * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	1323	+ * @v: pointer to atomic_t
	1324	+ * @old: pointer to int value to compare with
	1325	+ * @new: int value to assign
	1326	+ *
	1327	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	1328	+ * Otherwise, updates @old to the current value of @v.
	1329	+ *
	1330	+ * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there.
	1331	+ *
	1332	+ * Return: @true if the exchange occured, @false otherwise.
	1333	+ */
555	1334	static __always_inline bool
556	1335	atomic_try_cmpxchg_relaxed(atomic_t v, int old, int new)
557	1336	{
558	1337	instrument_atomic_read_write(v, sizeof(*v));
559	1338	instrument_atomic_read_write(old, sizeof(*old));
560		− return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561		−}
562		−
	1339	+ return raw_atomic_try_cmpxchg_relaxed(v, old, new);
	1340	+}
	1341	+
	1342	+/**
	1343	+ * atomic_sub_and_test() - atomic subtract and test if zero with full ordering
	1344	+ * @i: int value to add
	1345	+ * @v: pointer to atomic_t
	1346	+ *
	1347	+ * Atomically updates @v to (@v - @i) with full ordering.
	1348	+ *
	1349	+ * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there.
	1350	+ *
	1351	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	1352	+ */
563	1353	static __always_inline bool
564	1354	atomic_sub_and_test(int i, atomic_t *v)
565	1355	{
566	1356	kcsan_mb();
567	1357	instrument_atomic_read_write(v, sizeof(*v));
568		− return arch_atomic_sub_and_test(i, v);
	1358	+ return raw_atomic_sub_and_test(i, v);
569	1359	}
570	1360
	1361	+/**
	1362	+ * atomic_dec_and_test() - atomic decrement and test if zero with full ordering
	1363	+ * @v: pointer to atomic_t
	1364	+ *
	1365	+ * Atomically updates @v to (@v - 1) with full ordering.
	1366	+ *
	1367	+ * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there.
	1368	+ *
	1369	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	1370	+ */
571	1371	static __always_inline bool
572	1372	atomic_dec_and_test(atomic_t *v)
573	1373	{
574	1374	kcsan_mb();
575	1375	instrument_atomic_read_write(v, sizeof(*v));
576		− return arch_atomic_dec_and_test(v);
	1376	+ return raw_atomic_dec_and_test(v);
577	1377	}
578	1378
	1379	+/**
	1380	+ * atomic_inc_and_test() - atomic increment and test if zero with full ordering
	1381	+ * @v: pointer to atomic_t
	1382	+ *
	1383	+ * Atomically updates @v to (@v + 1) with full ordering.
	1384	+ *
	1385	+ * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there.
	1386	+ *
	1387	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	1388	+ */
579	1389	static __always_inline bool
580	1390	atomic_inc_and_test(atomic_t *v)
581	1391	{
582	1392	kcsan_mb();
583	1393	instrument_atomic_read_write(v, sizeof(*v));
584		− return arch_atomic_inc_and_test(v);
	1394	+ return raw_atomic_inc_and_test(v);
585	1395	}
586	1396
	1397	+/**
	1398	+ * atomic_add_negative() - atomic add and test if negative with full ordering
	1399	+ * @i: int value to add
	1400	+ * @v: pointer to atomic_t
	1401	+ *
	1402	+ * Atomically updates @v to (@v + @i) with full ordering.
	1403	+ *
	1404	+ * Unsafe to use in noinstr code; use raw_atomic_add_negative() there.
	1405	+ *
	1406	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1407	+ */
587	1408	static __always_inline bool
588	1409	atomic_add_negative(int i, atomic_t *v)
589	1410	{
590	1411	kcsan_mb();
591	1412	instrument_atomic_read_write(v, sizeof(*v));
592		− return arch_atomic_add_negative(i, v);
	1413	+ return raw_atomic_add_negative(i, v);
593	1414	}
594	1415
	1416	+/**
	1417	+ * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
	1418	+ * @i: int value to add
	1419	+ * @v: pointer to atomic_t
	1420	+ *
	1421	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	1422	+ *
	1423	+ * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there.
	1424	+ *
	1425	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1426	+ */
595	1427	static __always_inline bool
596	1428	atomic_add_negative_acquire(int i, atomic_t *v)
597	1429	{
598	1430	instrument_atomic_read_write(v, sizeof(*v));
599		− return arch_atomic_add_negative_acquire(i, v);
	1431	+ return raw_atomic_add_negative_acquire(i, v);
600	1432	}
601	1433
	1434	+/**
	1435	+ * atomic_add_negative_release() - atomic add and test if negative with release ordering
	1436	+ * @i: int value to add
	1437	+ * @v: pointer to atomic_t
	1438	+ *
	1439	+ * Atomically updates @v to (@v + @i) with release ordering.
	1440	+ *
	1441	+ * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there.
	1442	+ *
	1443	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1444	+ */
602	1445	static __always_inline bool
603	1446	atomic_add_negative_release(int i, atomic_t *v)
604	1447	{
605	1448	kcsan_release();
606	1449	instrument_atomic_read_write(v, sizeof(*v));
607		− return arch_atomic_add_negative_release(i, v);
	1450	+ return raw_atomic_add_negative_release(i, v);
608	1451	}
609	1452
	1453	+/**
	1454	+ * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
	1455	+ * @i: int value to add
	1456	+ * @v: pointer to atomic_t
	1457	+ *
	1458	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	1459	+ *
	1460	+ * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there.
	1461	+ *
	1462	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1463	+ */
610	1464	static __always_inline bool
611	1465	atomic_add_negative_relaxed(int i, atomic_t *v)
612	1466	{
613	1467	instrument_atomic_read_write(v, sizeof(*v));
614		− return arch_atomic_add_negative_relaxed(i, v);
	1468	+ return raw_atomic_add_negative_relaxed(i, v);
615	1469	}
616	1470
	1471	+/**
	1472	+ * atomic_fetch_add_unless() - atomic add unless value with full ordering
	1473	+ * @v: pointer to atomic_t
	1474	+ * @a: int value to add
	1475	+ * @u: int value to compare with
	1476	+ *
	1477	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1478	+ *
	1479	+ * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there.
	1480	+ *
	1481	+ * Return: The original value of @v.
	1482	+ */
617	1483	static __always_inline int
618	1484	atomic_fetch_add_unless(atomic_t *v, int a, int u)
619	1485	{
620	1486	kcsan_mb();
621	1487	instrument_atomic_read_write(v, sizeof(*v));
622		− return arch_atomic_fetch_add_unless(v, a, u);
	1488	+ return raw_atomic_fetch_add_unless(v, a, u);
623	1489	}
624	1490
	1491	+/**
	1492	+ * atomic_add_unless() - atomic add unless value with full ordering
	1493	+ * @v: pointer to atomic_t
	1494	+ * @a: int value to add
	1495	+ * @u: int value to compare with
	1496	+ *
	1497	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1498	+ *
	1499	+ * Unsafe to use in noinstr code; use raw_atomic_add_unless() there.
	1500	+ *
	1501	+ * Return: @true if @v was updated, @false otherwise.
	1502	+ */
625	1503	static __always_inline bool
626	1504	atomic_add_unless(atomic_t *v, int a, int u)
627	1505	{
628	1506	kcsan_mb();
629	1507	instrument_atomic_read_write(v, sizeof(*v));
630		− return arch_atomic_add_unless(v, a, u);
	1508	+ return raw_atomic_add_unless(v, a, u);
631	1509	}
632	1510
	1511	+/**
	1512	+ * atomic_inc_not_zero() - atomic increment unless zero with full ordering
	1513	+ * @v: pointer to atomic_t
	1514	+ *
	1515	+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	1516	+ *
	1517	+ * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there.
	1518	+ *
	1519	+ * Return: @true if @v was updated, @false otherwise.
	1520	+ */
633	1521	static __always_inline bool
634	1522	atomic_inc_not_zero(atomic_t *v)
635	1523	{
636	1524	kcsan_mb();
637	1525	instrument_atomic_read_write(v, sizeof(*v));
638		− return arch_atomic_inc_not_zero(v);
	1526	+ return raw_atomic_inc_not_zero(v);
639	1527	}
640	1528
	1529	+/**
	1530	+ * atomic_inc_unless_negative() - atomic increment unless negative with full ordering
	1531	+ * @v: pointer to atomic_t
	1532	+ *
	1533	+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	1534	+ *
	1535	+ * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there.
	1536	+ *
	1537	+ * Return: @true if @v was updated, @false otherwise.
	1538	+ */
641	1539	static __always_inline bool
642	1540	atomic_inc_unless_negative(atomic_t *v)
643	1541	{
644	1542	kcsan_mb();
645	1543	instrument_atomic_read_write(v, sizeof(*v));
646		− return arch_atomic_inc_unless_negative(v);
	1544	+ return raw_atomic_inc_unless_negative(v);
647	1545	}
648	1546
	1547	+/**
	1548	+ * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
	1549	+ * @v: pointer to atomic_t
	1550	+ *
	1551	+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	1552	+ *
	1553	+ * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there.
	1554	+ *
	1555	+ * Return: @true if @v was updated, @false otherwise.
	1556	+ */
649	1557	static __always_inline bool
650	1558	atomic_dec_unless_positive(atomic_t *v)
651	1559	{
652	1560	kcsan_mb();
653	1561	instrument_atomic_read_write(v, sizeof(*v));
654		− return arch_atomic_dec_unless_positive(v);
	1562	+ return raw_atomic_dec_unless_positive(v);
655	1563	}
656	1564
	1565	+/**
	1566	+ * atomic_dec_if_positive() - atomic decrement if positive with full ordering
	1567	+ * @v: pointer to atomic_t
	1568	+ *
	1569	+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	1570	+ *
	1571	+ * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there.
	1572	+ *
	1573	+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
	1574	+ */
657	1575	static __always_inline int
658	1576	atomic_dec_if_positive(atomic_t *v)
659	1577	{
660	1578	kcsan_mb();
661	1579	instrument_atomic_read_write(v, sizeof(*v));
662		− return arch_atomic_dec_if_positive(v);
	1580	+ return raw_atomic_dec_if_positive(v);
663	1581	}
664	1582
	1583	+/**
	1584	+ * atomic64_read() - atomic load with relaxed ordering
	1585	+ * @v: pointer to atomic64_t
	1586	+ *
	1587	+ * Atomically loads the value of @v with relaxed ordering.
	1588	+ *
	1589	+ * Unsafe to use in noinstr code; use raw_atomic64_read() there.
	1590	+ *
	1591	+ * Return: The value loaded from @v.
	1592	+ */
665	1593	static __always_inline s64
666	1594	atomic64_read(const atomic64_t *v)
667	1595	{
668	1596	instrument_atomic_read(v, sizeof(*v));
669		− return arch_atomic64_read(v);
670		−}
671		−
	1597	+ return raw_atomic64_read(v);
	1598	+}
	1599	+
	1600	+/**
	1601	+ * atomic64_read_acquire() - atomic load with acquire ordering
	1602	+ * @v: pointer to atomic64_t
	1603	+ *
	1604	+ * Atomically loads the value of @v with acquire ordering.
	1605	+ *
	1606	+ * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there.
	1607	+ *
	1608	+ * Return: The value loaded from @v.
	1609	+ */
672	1610	static __always_inline s64
673	1611	atomic64_read_acquire(const atomic64_t *v)
674	1612	{
675	1613	instrument_atomic_read(v, sizeof(*v));
676		− return arch_atomic64_read_acquire(v);
677		−}
678		−
	1614	+ return raw_atomic64_read_acquire(v);
	1615	+}
	1616	+
	1617	+/**
	1618	+ * atomic64_set() - atomic set with relaxed ordering
	1619	+ * @v: pointer to atomic64_t
	1620	+ * @i: s64 value to assign
	1621	+ *
	1622	+ * Atomically sets @v to @i with relaxed ordering.
	1623	+ *
	1624	+ * Unsafe to use in noinstr code; use raw_atomic64_set() there.
	1625	+ *
	1626	+ * Return: Nothing.
	1627	+ */
679	1628	static __always_inline void
680	1629	atomic64_set(atomic64_t *v, s64 i)
681	1630	{
682	1631	instrument_atomic_write(v, sizeof(*v));
683		− arch_atomic64_set(v, i);
684		−}
685		−
	1632	+ raw_atomic64_set(v, i);
	1633	+}
	1634	+
	1635	+/**
	1636	+ * atomic64_set_release() - atomic set with release ordering
	1637	+ * @v: pointer to atomic64_t
	1638	+ * @i: s64 value to assign
	1639	+ *
	1640	+ * Atomically sets @v to @i with release ordering.
	1641	+ *
	1642	+ * Unsafe to use in noinstr code; use raw_atomic64_set_release() there.
	1643	+ *
	1644	+ * Return: Nothing.
	1645	+ */
686	1646	static __always_inline void
687	1647	atomic64_set_release(atomic64_t *v, s64 i)
688	1648	{
689	1649	kcsan_release();
690	1650	instrument_atomic_write(v, sizeof(*v));
691		− arch_atomic64_set_release(v, i);
692		−}
693		−
	1651	+ raw_atomic64_set_release(v, i);
	1652	+}
	1653	+
	1654	+/**
	1655	+ * atomic64_add() - atomic add with relaxed ordering
	1656	+ * @i: s64 value to add
	1657	+ * @v: pointer to atomic64_t
	1658	+ *
	1659	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	1660	+ *
	1661	+ * Unsafe to use in noinstr code; use raw_atomic64_add() there.
	1662	+ *
	1663	+ * Return: Nothing.
	1664	+ */
694	1665	static __always_inline void
695	1666	atomic64_add(s64 i, atomic64_t *v)
696	1667	{
697	1668	instrument_atomic_read_write(v, sizeof(*v));
698		− arch_atomic64_add(i, v);
	1669	+ raw_atomic64_add(i, v);
699	1670	}
700	1671
	1672	+/**
	1673	+ * atomic64_add_return() - atomic add with full ordering
	1674	+ * @i: s64 value to add
	1675	+ * @v: pointer to atomic64_t
	1676	+ *
	1677	+ * Atomically updates @v to (@v + @i) with full ordering.
	1678	+ *
	1679	+ * Unsafe to use in noinstr code; use raw_atomic64_add_return() there.
	1680	+ *
	1681	+ * Return: The updated value of @v.
	1682	+ */
701	1683	static __always_inline s64
702	1684	atomic64_add_return(s64 i, atomic64_t *v)
703	1685	{
704	1686	kcsan_mb();
705	1687	instrument_atomic_read_write(v, sizeof(*v));
706		− return arch_atomic64_add_return(i, v);
	1688	+ return raw_atomic64_add_return(i, v);
707	1689	}
708	1690
	1691	+/**
	1692	+ * atomic64_add_return_acquire() - atomic add with acquire ordering
	1693	+ * @i: s64 value to add
	1694	+ * @v: pointer to atomic64_t
	1695	+ *
	1696	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	1697	+ *
	1698	+ * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there.
	1699	+ *
	1700	+ * Return: The updated value of @v.
	1701	+ */
709	1702	static __always_inline s64
710	1703	atomic64_add_return_acquire(s64 i, atomic64_t *v)
711	1704	{
712	1705	instrument_atomic_read_write(v, sizeof(*v));
713		− return arch_atomic64_add_return_acquire(i, v);
	1706	+ return raw_atomic64_add_return_acquire(i, v);
714	1707	}
715	1708
	1709	+/**
	1710	+ * atomic64_add_return_release() - atomic add with release ordering
	1711	+ * @i: s64 value to add
	1712	+ * @v: pointer to atomic64_t
	1713	+ *
	1714	+ * Atomically updates @v to (@v + @i) with release ordering.
	1715	+ *
	1716	+ * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there.
	1717	+ *
	1718	+ * Return: The updated value of @v.
	1719	+ */
716	1720	static __always_inline s64
717	1721	atomic64_add_return_release(s64 i, atomic64_t *v)
718	1722	{
719	1723	kcsan_release();
720	1724	instrument_atomic_read_write(v, sizeof(*v));
721		− return arch_atomic64_add_return_release(i, v);
	1725	+ return raw_atomic64_add_return_release(i, v);
722	1726	}
723	1727
	1728	+/**
	1729	+ * atomic64_add_return_relaxed() - atomic add with relaxed ordering
	1730	+ * @i: s64 value to add
	1731	+ * @v: pointer to atomic64_t
	1732	+ *
	1733	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	1734	+ *
	1735	+ * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there.
	1736	+ *
	1737	+ * Return: The updated value of @v.
	1738	+ */
724	1739	static __always_inline s64
725	1740	atomic64_add_return_relaxed(s64 i, atomic64_t *v)
726	1741	{
727	1742	instrument_atomic_read_write(v, sizeof(*v));
728		− return arch_atomic64_add_return_relaxed(i, v);
	1743	+ return raw_atomic64_add_return_relaxed(i, v);
729	1744	}
730	1745
	1746	+/**
	1747	+ * atomic64_fetch_add() - atomic add with full ordering
	1748	+ * @i: s64 value to add
	1749	+ * @v: pointer to atomic64_t
	1750	+ *
	1751	+ * Atomically updates @v to (@v + @i) with full ordering.
	1752	+ *
	1753	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there.
	1754	+ *
	1755	+ * Return: The original value of @v.
	1756	+ */
731	1757	static __always_inline s64
732	1758	atomic64_fetch_add(s64 i, atomic64_t *v)
733	1759	{
734	1760	kcsan_mb();
735	1761	instrument_atomic_read_write(v, sizeof(*v));
736		− return arch_atomic64_fetch_add(i, v);
	1762	+ return raw_atomic64_fetch_add(i, v);
737	1763	}
738	1764
	1765	+/**
	1766	+ * atomic64_fetch_add_acquire() - atomic add with acquire ordering
	1767	+ * @i: s64 value to add
	1768	+ * @v: pointer to atomic64_t
	1769	+ *
	1770	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	1771	+ *
	1772	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there.
	1773	+ *
	1774	+ * Return: The original value of @v.
	1775	+ */
739	1776	static __always_inline s64
740	1777	atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
741	1778	{
742	1779	instrument_atomic_read_write(v, sizeof(*v));
743		− return arch_atomic64_fetch_add_acquire(i, v);
	1780	+ return raw_atomic64_fetch_add_acquire(i, v);
744	1781	}
745	1782
	1783	+/**
	1784	+ * atomic64_fetch_add_release() - atomic add with release ordering
	1785	+ * @i: s64 value to add
	1786	+ * @v: pointer to atomic64_t
	1787	+ *
	1788	+ * Atomically updates @v to (@v + @i) with release ordering.
	1789	+ *
	1790	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there.
	1791	+ *
	1792	+ * Return: The original value of @v.
	1793	+ */
746	1794	static __always_inline s64
747	1795	atomic64_fetch_add_release(s64 i, atomic64_t *v)
748	1796	{
749	1797	kcsan_release();
750	1798	instrument_atomic_read_write(v, sizeof(*v));
751		− return arch_atomic64_fetch_add_release(i, v);
	1799	+ return raw_atomic64_fetch_add_release(i, v);
752	1800	}
753	1801
	1802	+/**
	1803	+ * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
	1804	+ * @i: s64 value to add
	1805	+ * @v: pointer to atomic64_t
	1806	+ *
	1807	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	1808	+ *
	1809	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there.
	1810	+ *
	1811	+ * Return: The original value of @v.
	1812	+ */
754	1813	static __always_inline s64
755	1814	atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
756	1815	{
757	1816	instrument_atomic_read_write(v, sizeof(*v));
758		− return arch_atomic64_fetch_add_relaxed(i, v);
	1817	+ return raw_atomic64_fetch_add_relaxed(i, v);
759	1818	}
760	1819
	1820	+/**
	1821	+ * atomic64_sub() - atomic subtract with relaxed ordering
	1822	+ * @i: s64 value to subtract
	1823	+ * @v: pointer to atomic64_t
	1824	+ *
	1825	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	1826	+ *
	1827	+ * Unsafe to use in noinstr code; use raw_atomic64_sub() there.
	1828	+ *
	1829	+ * Return: Nothing.
	1830	+ */
761	1831	static __always_inline void
762	1832	atomic64_sub(s64 i, atomic64_t *v)
763	1833	{
764	1834	instrument_atomic_read_write(v, sizeof(*v));
765		− arch_atomic64_sub(i, v);
	1835	+ raw_atomic64_sub(i, v);
766	1836	}
767	1837
	1838	+/**
	1839	+ * atomic64_sub_return() - atomic subtract with full ordering
	1840	+ * @i: s64 value to subtract
	1841	+ * @v: pointer to atomic64_t
	1842	+ *
	1843	+ * Atomically updates @v to (@v - @i) with full ordering.
	1844	+ *
	1845	+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there.
	1846	+ *
	1847	+ * Return: The updated value of @v.
	1848	+ */
768	1849	static __always_inline s64
769	1850	atomic64_sub_return(s64 i, atomic64_t *v)
770	1851	{
771	1852	kcsan_mb();
772	1853	instrument_atomic_read_write(v, sizeof(*v));
773		− return arch_atomic64_sub_return(i, v);
	1854	+ return raw_atomic64_sub_return(i, v);
774	1855	}
775	1856
	1857	+/**
	1858	+ * atomic64_sub_return_acquire() - atomic subtract with acquire ordering
	1859	+ * @i: s64 value to subtract
	1860	+ * @v: pointer to atomic64_t
	1861	+ *
	1862	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	1863	+ *
	1864	+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there.
	1865	+ *
	1866	+ * Return: The updated value of @v.
	1867	+ */
776	1868	static __always_inline s64
777	1869	atomic64_sub_return_acquire(s64 i, atomic64_t *v)
778	1870	{
779	1871	instrument_atomic_read_write(v, sizeof(*v));
780		− return arch_atomic64_sub_return_acquire(i, v);
	1872	+ return raw_atomic64_sub_return_acquire(i, v);
781	1873	}
782	1874
	1875	+/**
	1876	+ * atomic64_sub_return_release() - atomic subtract with release ordering
	1877	+ * @i: s64 value to subtract
	1878	+ * @v: pointer to atomic64_t
	1879	+ *
	1880	+ * Atomically updates @v to (@v - @i) with release ordering.
	1881	+ *
	1882	+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there.
	1883	+ *
	1884	+ * Return: The updated value of @v.
	1885	+ */
783	1886	static __always_inline s64
784	1887	atomic64_sub_return_release(s64 i, atomic64_t *v)
785	1888	{
786	1889	kcsan_release();
787	1890	instrument_atomic_read_write(v, sizeof(*v));
788		− return arch_atomic64_sub_return_release(i, v);
	1891	+ return raw_atomic64_sub_return_release(i, v);
789	1892	}
790	1893
	1894	+/**
	1895	+ * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
	1896	+ * @i: s64 value to subtract
	1897	+ * @v: pointer to atomic64_t
	1898	+ *
	1899	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	1900	+ *
	1901	+ * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there.
	1902	+ *
	1903	+ * Return: The updated value of @v.
	1904	+ */
791	1905	static __always_inline s64
792	1906	atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
793	1907	{
794	1908	instrument_atomic_read_write(v, sizeof(*v));
795		− return arch_atomic64_sub_return_relaxed(i, v);
	1909	+ return raw_atomic64_sub_return_relaxed(i, v);
796	1910	}
797	1911
	1912	+/**
	1913	+ * atomic64_fetch_sub() - atomic subtract with full ordering
	1914	+ * @i: s64 value to subtract
	1915	+ * @v: pointer to atomic64_t
	1916	+ *
	1917	+ * Atomically updates @v to (@v - @i) with full ordering.
	1918	+ *
	1919	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there.
	1920	+ *
	1921	+ * Return: The original value of @v.
	1922	+ */
798	1923	static __always_inline s64
799	1924	atomic64_fetch_sub(s64 i, atomic64_t *v)
800	1925	{
801	1926	kcsan_mb();
802	1927	instrument_atomic_read_write(v, sizeof(*v));
803		− return arch_atomic64_fetch_sub(i, v);
	1928	+ return raw_atomic64_fetch_sub(i, v);
804	1929	}
805	1930
	1931	+/**
	1932	+ * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
	1933	+ * @i: s64 value to subtract
	1934	+ * @v: pointer to atomic64_t
	1935	+ *
	1936	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	1937	+ *
	1938	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there.
	1939	+ *
	1940	+ * Return: The original value of @v.
	1941	+ */
806	1942	static __always_inline s64
807	1943	atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
808	1944	{
809	1945	instrument_atomic_read_write(v, sizeof(*v));
810		− return arch_atomic64_fetch_sub_acquire(i, v);
	1946	+ return raw_atomic64_fetch_sub_acquire(i, v);
811	1947	}
812	1948
	1949	+/**
	1950	+ * atomic64_fetch_sub_release() - atomic subtract with release ordering
	1951	+ * @i: s64 value to subtract
	1952	+ * @v: pointer to atomic64_t
	1953	+ *
	1954	+ * Atomically updates @v to (@v - @i) with release ordering.
	1955	+ *
	1956	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there.
	1957	+ *
	1958	+ * Return: The original value of @v.
	1959	+ */
813	1960	static __always_inline s64
814	1961	atomic64_fetch_sub_release(s64 i, atomic64_t *v)
815	1962	{
816	1963	kcsan_release();
817	1964	instrument_atomic_read_write(v, sizeof(*v));
818		− return arch_atomic64_fetch_sub_release(i, v);
	1965	+ return raw_atomic64_fetch_sub_release(i, v);
819	1966	}
820	1967
	1968	+/**
	1969	+ * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
	1970	+ * @i: s64 value to subtract
	1971	+ * @v: pointer to atomic64_t
	1972	+ *
	1973	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	1974	+ *
	1975	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there.
	1976	+ *
	1977	+ * Return: The original value of @v.
	1978	+ */
821	1979	static __always_inline s64
822	1980	atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
823	1981	{
824	1982	instrument_atomic_read_write(v, sizeof(*v));
825		− return arch_atomic64_fetch_sub_relaxed(i, v);
	1983	+ return raw_atomic64_fetch_sub_relaxed(i, v);
826	1984	}
827	1985
	1986	+/**
	1987	+ * atomic64_inc() - atomic increment with relaxed ordering
	1988	+ * @v: pointer to atomic64_t
	1989	+ *
	1990	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	1991	+ *
	1992	+ * Unsafe to use in noinstr code; use raw_atomic64_inc() there.
	1993	+ *
	1994	+ * Return: Nothing.
	1995	+ */
828	1996	static __always_inline void
829	1997	atomic64_inc(atomic64_t *v)
830	1998	{
831	1999	instrument_atomic_read_write(v, sizeof(*v));
832		− arch_atomic64_inc(v);
	2000	+ raw_atomic64_inc(v);
833	2001	}
834	2002
	2003	+/**
	2004	+ * atomic64_inc_return() - atomic increment with full ordering
	2005	+ * @v: pointer to atomic64_t
	2006	+ *
	2007	+ * Atomically updates @v to (@v + 1) with full ordering.
	2008	+ *
	2009	+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there.
	2010	+ *
	2011	+ * Return: The updated value of @v.
	2012	+ */
835	2013	static __always_inline s64
836	2014	atomic64_inc_return(atomic64_t *v)
837	2015	{
838	2016	kcsan_mb();
839	2017	instrument_atomic_read_write(v, sizeof(*v));
840		− return arch_atomic64_inc_return(v);
	2018	+ return raw_atomic64_inc_return(v);
841	2019	}
842	2020
	2021	+/**
	2022	+ * atomic64_inc_return_acquire() - atomic increment with acquire ordering
	2023	+ * @v: pointer to atomic64_t
	2024	+ *
	2025	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	2026	+ *
	2027	+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there.
	2028	+ *
	2029	+ * Return: The updated value of @v.
	2030	+ */
843	2031	static __always_inline s64
844	2032	atomic64_inc_return_acquire(atomic64_t *v)
845	2033	{
846	2034	instrument_atomic_read_write(v, sizeof(*v));
847		− return arch_atomic64_inc_return_acquire(v);
	2035	+ return raw_atomic64_inc_return_acquire(v);
848	2036	}
849	2037
	2038	+/**
	2039	+ * atomic64_inc_return_release() - atomic increment with release ordering
	2040	+ * @v: pointer to atomic64_t
	2041	+ *
	2042	+ * Atomically updates @v to (@v + 1) with release ordering.
	2043	+ *
	2044	+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there.
	2045	+ *
	2046	+ * Return: The updated value of @v.
	2047	+ */
850	2048	static __always_inline s64
851	2049	atomic64_inc_return_release(atomic64_t *v)
852	2050	{
853	2051	kcsan_release();
854	2052	instrument_atomic_read_write(v, sizeof(*v));
855		− return arch_atomic64_inc_return_release(v);
	2053	+ return raw_atomic64_inc_return_release(v);
856	2054	}
857	2055
	2056	+/**
	2057	+ * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
	2058	+ * @v: pointer to atomic64_t
	2059	+ *
	2060	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	2061	+ *
	2062	+ * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there.
	2063	+ *
	2064	+ * Return: The updated value of @v.
	2065	+ */
858	2066	static __always_inline s64
859	2067	atomic64_inc_return_relaxed(atomic64_t *v)
860	2068	{
861	2069	instrument_atomic_read_write(v, sizeof(*v));
862		− return arch_atomic64_inc_return_relaxed(v);
	2070	+ return raw_atomic64_inc_return_relaxed(v);
863	2071	}
864	2072
	2073	+/**
	2074	+ * atomic64_fetch_inc() - atomic increment with full ordering
	2075	+ * @v: pointer to atomic64_t
	2076	+ *
	2077	+ * Atomically updates @v to (@v + 1) with full ordering.
	2078	+ *
	2079	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there.
	2080	+ *
	2081	+ * Return: The original value of @v.
	2082	+ */
865	2083	static __always_inline s64
866	2084	atomic64_fetch_inc(atomic64_t *v)
867	2085	{
868	2086	kcsan_mb();
869	2087	instrument_atomic_read_write(v, sizeof(*v));
870		− return arch_atomic64_fetch_inc(v);
	2088	+ return raw_atomic64_fetch_inc(v);
871	2089	}
872	2090
	2091	+/**
	2092	+ * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
	2093	+ * @v: pointer to atomic64_t
	2094	+ *
	2095	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	2096	+ *
	2097	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there.
	2098	+ *
	2099	+ * Return: The original value of @v.
	2100	+ */
873	2101	static __always_inline s64
874	2102	atomic64_fetch_inc_acquire(atomic64_t *v)
875	2103	{
876	2104	instrument_atomic_read_write(v, sizeof(*v));
877		− return arch_atomic64_fetch_inc_acquire(v);
	2105	+ return raw_atomic64_fetch_inc_acquire(v);
878	2106	}
879	2107
	2108	+/**
	2109	+ * atomic64_fetch_inc_release() - atomic increment with release ordering
	2110	+ * @v: pointer to atomic64_t
	2111	+ *
	2112	+ * Atomically updates @v to (@v + 1) with release ordering.
	2113	+ *
	2114	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there.
	2115	+ *
	2116	+ * Return: The original value of @v.
	2117	+ */
880	2118	static __always_inline s64
881	2119	atomic64_fetch_inc_release(atomic64_t *v)
882	2120	{
883	2121	kcsan_release();
884	2122	instrument_atomic_read_write(v, sizeof(*v));
885		− return arch_atomic64_fetch_inc_release(v);
	2123	+ return raw_atomic64_fetch_inc_release(v);
886	2124	}
887	2125
	2126	+/**
	2127	+ * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
	2128	+ * @v: pointer to atomic64_t
	2129	+ *
	2130	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	2131	+ *
	2132	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there.
	2133	+ *
	2134	+ * Return: The original value of @v.
	2135	+ */
888	2136	static __always_inline s64
889	2137	atomic64_fetch_inc_relaxed(atomic64_t *v)
890	2138	{
891	2139	instrument_atomic_read_write(v, sizeof(*v));
892		− return arch_atomic64_fetch_inc_relaxed(v);
	2140	+ return raw_atomic64_fetch_inc_relaxed(v);
893	2141	}
894	2142
	2143	+/**
	2144	+ * atomic64_dec() - atomic decrement with relaxed ordering
	2145	+ * @v: pointer to atomic64_t
	2146	+ *
	2147	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	2148	+ *
	2149	+ * Unsafe to use in noinstr code; use raw_atomic64_dec() there.
	2150	+ *
	2151	+ * Return: Nothing.
	2152	+ */
895	2153	static __always_inline void
896	2154	atomic64_dec(atomic64_t *v)
897	2155	{
898	2156	instrument_atomic_read_write(v, sizeof(*v));
899		− arch_atomic64_dec(v);
	2157	+ raw_atomic64_dec(v);
900	2158	}
901	2159
	2160	+/**
	2161	+ * atomic64_dec_return() - atomic decrement with full ordering
	2162	+ * @v: pointer to atomic64_t
	2163	+ *
	2164	+ * Atomically updates @v to (@v - 1) with full ordering.
	2165	+ *
	2166	+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there.
	2167	+ *
	2168	+ * Return: The updated value of @v.
	2169	+ */
902	2170	static __always_inline s64
903	2171	atomic64_dec_return(atomic64_t *v)
904	2172	{
905	2173	kcsan_mb();
906	2174	instrument_atomic_read_write(v, sizeof(*v));
907		− return arch_atomic64_dec_return(v);
	2175	+ return raw_atomic64_dec_return(v);
908	2176	}
909	2177
	2178	+/**
	2179	+ * atomic64_dec_return_acquire() - atomic decrement with acquire ordering
	2180	+ * @v: pointer to atomic64_t
	2181	+ *
	2182	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	2183	+ *
	2184	+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there.
	2185	+ *
	2186	+ * Return: The updated value of @v.
	2187	+ */
910	2188	static __always_inline s64
911	2189	atomic64_dec_return_acquire(atomic64_t *v)
912	2190	{
913	2191	instrument_atomic_read_write(v, sizeof(*v));
914		− return arch_atomic64_dec_return_acquire(v);
	2192	+ return raw_atomic64_dec_return_acquire(v);
915	2193	}
916	2194
	2195	+/**
	2196	+ * atomic64_dec_return_release() - atomic decrement with release ordering
	2197	+ * @v: pointer to atomic64_t
	2198	+ *
	2199	+ * Atomically updates @v to (@v - 1) with release ordering.
	2200	+ *
	2201	+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there.
	2202	+ *
	2203	+ * Return: The updated value of @v.
	2204	+ */
917	2205	static __always_inline s64
918	2206	atomic64_dec_return_release(atomic64_t *v)
919	2207	{
920	2208	kcsan_release();
921	2209	instrument_atomic_read_write(v, sizeof(*v));
922		− return arch_atomic64_dec_return_release(v);
	2210	+ return raw_atomic64_dec_return_release(v);
923	2211	}
924	2212
	2213	+/**
	2214	+ * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
	2215	+ * @v: pointer to atomic64_t
	2216	+ *
	2217	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	2218	+ *
	2219	+ * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there.
	2220	+ *
	2221	+ * Return: The updated value of @v.
	2222	+ */
925	2223	static __always_inline s64
926	2224	atomic64_dec_return_relaxed(atomic64_t *v)
927	2225	{
928	2226	instrument_atomic_read_write(v, sizeof(*v));
929		− return arch_atomic64_dec_return_relaxed(v);
	2227	+ return raw_atomic64_dec_return_relaxed(v);
930	2228	}
931	2229
	2230	+/**
	2231	+ * atomic64_fetch_dec() - atomic decrement with full ordering
	2232	+ * @v: pointer to atomic64_t
	2233	+ *
	2234	+ * Atomically updates @v to (@v - 1) with full ordering.
	2235	+ *
	2236	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there.
	2237	+ *
	2238	+ * Return: The original value of @v.
	2239	+ */
932	2240	static __always_inline s64
933	2241	atomic64_fetch_dec(atomic64_t *v)
934	2242	{
935	2243	kcsan_mb();
936	2244	instrument_atomic_read_write(v, sizeof(*v));
937		− return arch_atomic64_fetch_dec(v);
	2245	+ return raw_atomic64_fetch_dec(v);
938	2246	}
939	2247
	2248	+/**
	2249	+ * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
	2250	+ * @v: pointer to atomic64_t
	2251	+ *
	2252	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	2253	+ *
	2254	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there.
	2255	+ *
	2256	+ * Return: The original value of @v.
	2257	+ */
940	2258	static __always_inline s64
941	2259	atomic64_fetch_dec_acquire(atomic64_t *v)
942	2260	{
943	2261	instrument_atomic_read_write(v, sizeof(*v));
944		− return arch_atomic64_fetch_dec_acquire(v);
	2262	+ return raw_atomic64_fetch_dec_acquire(v);
945	2263	}
946	2264
	2265	+/**
	2266	+ * atomic64_fetch_dec_release() - atomic decrement with release ordering
	2267	+ * @v: pointer to atomic64_t
	2268	+ *
	2269	+ * Atomically updates @v to (@v - 1) with release ordering.
	2270	+ *
	2271	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there.
	2272	+ *
	2273	+ * Return: The original value of @v.
	2274	+ */
947	2275	static __always_inline s64
948	2276	atomic64_fetch_dec_release(atomic64_t *v)
949	2277	{
950	2278	kcsan_release();
951	2279	instrument_atomic_read_write(v, sizeof(*v));
952		− return arch_atomic64_fetch_dec_release(v);
	2280	+ return raw_atomic64_fetch_dec_release(v);
953	2281	}
954	2282
	2283	+/**
	2284	+ * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
	2285	+ * @v: pointer to atomic64_t
	2286	+ *
	2287	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	2288	+ *
	2289	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there.
	2290	+ *
	2291	+ * Return: The original value of @v.
	2292	+ */
955	2293	static __always_inline s64
956	2294	atomic64_fetch_dec_relaxed(atomic64_t *v)
957	2295	{
958	2296	instrument_atomic_read_write(v, sizeof(*v));
959		− return arch_atomic64_fetch_dec_relaxed(v);
	2297	+ return raw_atomic64_fetch_dec_relaxed(v);
960	2298	}
961	2299
	2300	+/**
	2301	+ * atomic64_and() - atomic bitwise AND with relaxed ordering
	2302	+ * @i: s64 value
	2303	+ * @v: pointer to atomic64_t
	2304	+ *
	2305	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	2306	+ *
	2307	+ * Unsafe to use in noinstr code; use raw_atomic64_and() there.
	2308	+ *
	2309	+ * Return: Nothing.
	2310	+ */
962	2311	static __always_inline void
963	2312	atomic64_and(s64 i, atomic64_t *v)
964	2313	{
965	2314	instrument_atomic_read_write(v, sizeof(*v));
966		− arch_atomic64_and(i, v);
	2315	+ raw_atomic64_and(i, v);
967	2316	}
968	2317
	2318	+/**
	2319	+ * atomic64_fetch_and() - atomic bitwise AND with full ordering
	2320	+ * @i: s64 value
	2321	+ * @v: pointer to atomic64_t
	2322	+ *
	2323	+ * Atomically updates @v to (@v & @i) with full ordering.
	2324	+ *
	2325	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there.
	2326	+ *
	2327	+ * Return: The original value of @v.
	2328	+ */
969	2329	static __always_inline s64
970	2330	atomic64_fetch_and(s64 i, atomic64_t *v)
971	2331	{
972	2332	kcsan_mb();
973	2333	instrument_atomic_read_write(v, sizeof(*v));
974		− return arch_atomic64_fetch_and(i, v);
	2334	+ return raw_atomic64_fetch_and(i, v);
975	2335	}
976	2336
	2337	+/**
	2338	+ * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
	2339	+ * @i: s64 value
	2340	+ * @v: pointer to atomic64_t
	2341	+ *
	2342	+ * Atomically updates @v to (@v & @i) with acquire ordering.
	2343	+ *
	2344	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there.
	2345	+ *
	2346	+ * Return: The original value of @v.
	2347	+ */
977	2348	static __always_inline s64
978	2349	atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
979	2350	{
980	2351	instrument_atomic_read_write(v, sizeof(*v));
981		− return arch_atomic64_fetch_and_acquire(i, v);
	2352	+ return raw_atomic64_fetch_and_acquire(i, v);
982	2353	}
983	2354
	2355	+/**
	2356	+ * atomic64_fetch_and_release() - atomic bitwise AND with release ordering
	2357	+ * @i: s64 value
	2358	+ * @v: pointer to atomic64_t
	2359	+ *
	2360	+ * Atomically updates @v to (@v & @i) with release ordering.
	2361	+ *
	2362	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there.
	2363	+ *
	2364	+ * Return: The original value of @v.
	2365	+ */
984	2366	static __always_inline s64
985	2367	atomic64_fetch_and_release(s64 i, atomic64_t *v)
986	2368	{
987	2369	kcsan_release();
988	2370	instrument_atomic_read_write(v, sizeof(*v));
989		− return arch_atomic64_fetch_and_release(i, v);
	2371	+ return raw_atomic64_fetch_and_release(i, v);
990	2372	}
991	2373
	2374	+/**
	2375	+ * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
	2376	+ * @i: s64 value
	2377	+ * @v: pointer to atomic64_t
	2378	+ *
	2379	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	2380	+ *
	2381	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there.
	2382	+ *
	2383	+ * Return: The original value of @v.
	2384	+ */
992	2385	static __always_inline s64
993	2386	atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
994	2387	{
995	2388	instrument_atomic_read_write(v, sizeof(*v));
996		− return arch_atomic64_fetch_and_relaxed(i, v);
	2389	+ return raw_atomic64_fetch_and_relaxed(i, v);
997	2390	}
998	2391
	2392	+/**
	2393	+ * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
	2394	+ * @i: s64 value
	2395	+ * @v: pointer to atomic64_t
	2396	+ *
	2397	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	2398	+ *
	2399	+ * Unsafe to use in noinstr code; use raw_atomic64_andnot() there.
	2400	+ *
	2401	+ * Return: Nothing.
	2402	+ */
999	2403	static __always_inline void
1000	2404	atomic64_andnot(s64 i, atomic64_t *v)
1001	2405	{
1002	2406	instrument_atomic_read_write(v, sizeof(*v));
1003		− arch_atomic64_andnot(i, v);
	2407	+ raw_atomic64_andnot(i, v);
1004	2408	}
1005	2409
	2410	+/**
	2411	+ * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
	2412	+ * @i: s64 value
	2413	+ * @v: pointer to atomic64_t
	2414	+ *
	2415	+ * Atomically updates @v to (@v & ~@i) with full ordering.
	2416	+ *
	2417	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there.
	2418	+ *
	2419	+ * Return: The original value of @v.
	2420	+ */
1006	2421	static __always_inline s64
1007	2422	atomic64_fetch_andnot(s64 i, atomic64_t *v)
1008	2423	{
1009	2424	kcsan_mb();
1010	2425	instrument_atomic_read_write(v, sizeof(*v));
1011		− return arch_atomic64_fetch_andnot(i, v);
	2426	+ return raw_atomic64_fetch_andnot(i, v);
1012	2427	}
1013	2428
	2429	+/**
	2430	+ * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
	2431	+ * @i: s64 value
	2432	+ * @v: pointer to atomic64_t
	2433	+ *
	2434	+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
	2435	+ *
	2436	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there.
	2437	+ *
	2438	+ * Return: The original value of @v.
	2439	+ */
1014	2440	static __always_inline s64
1015	2441	atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1016	2442	{
1017	2443	instrument_atomic_read_write(v, sizeof(*v));
1018		− return arch_atomic64_fetch_andnot_acquire(i, v);
	2444	+ return raw_atomic64_fetch_andnot_acquire(i, v);
1019	2445	}
1020	2446
	2447	+/**
	2448	+ * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
	2449	+ * @i: s64 value
	2450	+ * @v: pointer to atomic64_t
	2451	+ *
	2452	+ * Atomically updates @v to (@v & ~@i) with release ordering.
	2453	+ *
	2454	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there.
	2455	+ *
	2456	+ * Return: The original value of @v.
	2457	+ */
1021	2458	static __always_inline s64
1022	2459	atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1023	2460	{
1024	2461	kcsan_release();
1025	2462	instrument_atomic_read_write(v, sizeof(*v));
1026		− return arch_atomic64_fetch_andnot_release(i, v);
	2463	+ return raw_atomic64_fetch_andnot_release(i, v);
1027	2464	}
1028	2465
	2466	+/**
	2467	+ * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
	2468	+ * @i: s64 value
	2469	+ * @v: pointer to atomic64_t
	2470	+ *
	2471	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	2472	+ *
	2473	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there.
	2474	+ *
	2475	+ * Return: The original value of @v.
	2476	+ */
1029	2477	static __always_inline s64
1030	2478	atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1031	2479	{
1032	2480	instrument_atomic_read_write(v, sizeof(*v));
1033		− return arch_atomic64_fetch_andnot_relaxed(i, v);
	2481	+ return raw_atomic64_fetch_andnot_relaxed(i, v);
1034	2482	}
1035	2483
	2484	+/**
	2485	+ * atomic64_or() - atomic bitwise OR with relaxed ordering
	2486	+ * @i: s64 value
	2487	+ * @v: pointer to atomic64_t
	2488	+ *
	2489	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	2490	+ *
	2491	+ * Unsafe to use in noinstr code; use raw_atomic64_or() there.
	2492	+ *
	2493	+ * Return: Nothing.
	2494	+ */
1036	2495	static __always_inline void
1037	2496	atomic64_or(s64 i, atomic64_t *v)
1038	2497	{
1039	2498	instrument_atomic_read_write(v, sizeof(*v));
1040		− arch_atomic64_or(i, v);
	2499	+ raw_atomic64_or(i, v);
1041	2500	}
1042	2501
	2502	+/**
	2503	+ * atomic64_fetch_or() - atomic bitwise OR with full ordering
	2504	+ * @i: s64 value
	2505	+ * @v: pointer to atomic64_t
	2506	+ *
	2507	+ * Atomically updates @v to (@v \| @i) with full ordering.
	2508	+ *
	2509	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there.
	2510	+ *
	2511	+ * Return: The original value of @v.
	2512	+ */
1043	2513	static __always_inline s64
1044	2514	atomic64_fetch_or(s64 i, atomic64_t *v)
1045	2515	{
1046	2516	kcsan_mb();
1047	2517	instrument_atomic_read_write(v, sizeof(*v));
1048		− return arch_atomic64_fetch_or(i, v);
	2518	+ return raw_atomic64_fetch_or(i, v);
1049	2519	}
1050	2520
	2521	+/**
	2522	+ * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
	2523	+ * @i: s64 value
	2524	+ * @v: pointer to atomic64_t
	2525	+ *
	2526	+ * Atomically updates @v to (@v \| @i) with acquire ordering.
	2527	+ *
	2528	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there.
	2529	+ *
	2530	+ * Return: The original value of @v.
	2531	+ */
1051	2532	static __always_inline s64
1052	2533	atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1053	2534	{
1054	2535	instrument_atomic_read_write(v, sizeof(*v));
1055		− return arch_atomic64_fetch_or_acquire(i, v);
	2536	+ return raw_atomic64_fetch_or_acquire(i, v);
1056	2537	}
1057	2538
	2539	+/**
	2540	+ * atomic64_fetch_or_release() - atomic bitwise OR with release ordering
	2541	+ * @i: s64 value
	2542	+ * @v: pointer to atomic64_t
	2543	+ *
	2544	+ * Atomically updates @v to (@v \| @i) with release ordering.
	2545	+ *
	2546	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there.
	2547	+ *
	2548	+ * Return: The original value of @v.
	2549	+ */
1058	2550	static __always_inline s64
1059	2551	atomic64_fetch_or_release(s64 i, atomic64_t *v)
1060	2552	{
1061	2553	kcsan_release();
1062	2554	instrument_atomic_read_write(v, sizeof(*v));
1063		− return arch_atomic64_fetch_or_release(i, v);
	2555	+ return raw_atomic64_fetch_or_release(i, v);
1064	2556	}
1065	2557
	2558	+/**
	2559	+ * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
	2560	+ * @i: s64 value
	2561	+ * @v: pointer to atomic64_t
	2562	+ *
	2563	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	2564	+ *
	2565	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there.
	2566	+ *
	2567	+ * Return: The original value of @v.
	2568	+ */
1066	2569	static __always_inline s64
1067	2570	atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1068	2571	{
1069	2572	instrument_atomic_read_write(v, sizeof(*v));
1070		− return arch_atomic64_fetch_or_relaxed(i, v);
	2573	+ return raw_atomic64_fetch_or_relaxed(i, v);
1071	2574	}
1072	2575
	2576	+/**
	2577	+ * atomic64_xor() - atomic bitwise XOR with relaxed ordering
	2578	+ * @i: s64 value
	2579	+ * @v: pointer to atomic64_t
	2580	+ *
	2581	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	2582	+ *
	2583	+ * Unsafe to use in noinstr code; use raw_atomic64_xor() there.
	2584	+ *
	2585	+ * Return: Nothing.
	2586	+ */
1073	2587	static __always_inline void
1074	2588	atomic64_xor(s64 i, atomic64_t *v)
1075	2589	{
1076	2590	instrument_atomic_read_write(v, sizeof(*v));
1077		− arch_atomic64_xor(i, v);
	2591	+ raw_atomic64_xor(i, v);
1078	2592	}
1079	2593
	2594	+/**
	2595	+ * atomic64_fetch_xor() - atomic bitwise XOR with full ordering
	2596	+ * @i: s64 value
	2597	+ * @v: pointer to atomic64_t
	2598	+ *
	2599	+ * Atomically updates @v to (@v ^ @i) with full ordering.
	2600	+ *
	2601	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there.
	2602	+ *
	2603	+ * Return: The original value of @v.
	2604	+ */
1080	2605	static __always_inline s64
1081	2606	atomic64_fetch_xor(s64 i, atomic64_t *v)
1082	2607	{
1083	2608	kcsan_mb();
1084	2609	instrument_atomic_read_write(v, sizeof(*v));
1085		− return arch_atomic64_fetch_xor(i, v);
	2610	+ return raw_atomic64_fetch_xor(i, v);
1086	2611	}
1087	2612
	2613	+/**
	2614	+ * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
	2615	+ * @i: s64 value
	2616	+ * @v: pointer to atomic64_t
	2617	+ *
	2618	+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
	2619	+ *
	2620	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there.
	2621	+ *
	2622	+ * Return: The original value of @v.
	2623	+ */
1088	2624	static __always_inline s64
1089	2625	atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1090	2626	{
1091	2627	instrument_atomic_read_write(v, sizeof(*v));
1092		− return arch_atomic64_fetch_xor_acquire(i, v);
	2628	+ return raw_atomic64_fetch_xor_acquire(i, v);
1093	2629	}
1094	2630
	2631	+/**
	2632	+ * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
	2633	+ * @i: s64 value
	2634	+ * @v: pointer to atomic64_t
	2635	+ *
	2636	+ * Atomically updates @v to (@v ^ @i) with release ordering.
	2637	+ *
	2638	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there.
	2639	+ *
	2640	+ * Return: The original value of @v.
	2641	+ */
1095	2642	static __always_inline s64
1096	2643	atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1097	2644	{
1098	2645	kcsan_release();
1099	2646	instrument_atomic_read_write(v, sizeof(*v));
1100		− return arch_atomic64_fetch_xor_release(i, v);
	2647	+ return raw_atomic64_fetch_xor_release(i, v);
1101	2648	}
1102	2649
	2650	+/**
	2651	+ * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
	2652	+ * @i: s64 value
	2653	+ * @v: pointer to atomic64_t
	2654	+ *
	2655	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	2656	+ *
	2657	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there.
	2658	+ *
	2659	+ * Return: The original value of @v.
	2660	+ */
1103	2661	static __always_inline s64
1104	2662	atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1105	2663	{
1106	2664	instrument_atomic_read_write(v, sizeof(*v));
1107		− return arch_atomic64_fetch_xor_relaxed(i, v);
	2665	+ return raw_atomic64_fetch_xor_relaxed(i, v);
1108	2666	}
1109	2667
	2668	+/**
	2669	+ * atomic64_xchg() - atomic exchange with full ordering
	2670	+ * @v: pointer to atomic64_t
	2671	+ * @new: s64 value to assign
	2672	+ *
	2673	+ * Atomically updates @v to @new with full ordering.
	2674	+ *
	2675	+ * Unsafe to use in noinstr code; use raw_atomic64_xchg() there.
	2676	+ *
	2677	+ * Return: The original value of @v.
	2678	+ */
1110	2679	static __always_inline s64
1111		−atomic64_xchg(atomic64_t *v, s64 i)
	2680	+atomic64_xchg(atomic64_t *v, s64 new)
1112	2681	{
1113	2682	kcsan_mb();
1114	2683	instrument_atomic_read_write(v, sizeof(*v));
1115		− return arch_atomic64_xchg(v, i);
	2684	+ return raw_atomic64_xchg(v, new);
1116	2685	}
1117	2686
	2687	+/**
	2688	+ * atomic64_xchg_acquire() - atomic exchange with acquire ordering
	2689	+ * @v: pointer to atomic64_t
	2690	+ * @new: s64 value to assign
	2691	+ *
	2692	+ * Atomically updates @v to @new with acquire ordering.
	2693	+ *
	2694	+ * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there.
	2695	+ *
	2696	+ * Return: The original value of @v.
	2697	+ */
1118	2698	static __always_inline s64
1119		−atomic64_xchg_acquire(atomic64_t *v, s64 i)
	2699	+atomic64_xchg_acquire(atomic64_t *v, s64 new)
1120	2700	{
1121	2701	instrument_atomic_read_write(v, sizeof(*v));
1122		− return arch_atomic64_xchg_acquire(v, i);
	2702	+ return raw_atomic64_xchg_acquire(v, new);
1123	2703	}
1124	2704
	2705	+/**
	2706	+ * atomic64_xchg_release() - atomic exchange with release ordering
	2707	+ * @v: pointer to atomic64_t
	2708	+ * @new: s64 value to assign
	2709	+ *
	2710	+ * Atomically updates @v to @new with release ordering.
	2711	+ *
	2712	+ * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there.
	2713	+ *
	2714	+ * Return: The original value of @v.
	2715	+ */
1125	2716	static __always_inline s64
1126		−atomic64_xchg_release(atomic64_t *v, s64 i)
	2717	+atomic64_xchg_release(atomic64_t *v, s64 new)
1127	2718	{
1128	2719	kcsan_release();
1129	2720	instrument_atomic_read_write(v, sizeof(*v));
1130		− return arch_atomic64_xchg_release(v, i);
	2721	+ return raw_atomic64_xchg_release(v, new);
1131	2722	}
1132	2723
	2724	+/**
	2725	+ * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
	2726	+ * @v: pointer to atomic64_t
	2727	+ * @new: s64 value to assign
	2728	+ *
	2729	+ * Atomically updates @v to @new with relaxed ordering.
	2730	+ *
	2731	+ * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there.
	2732	+ *
	2733	+ * Return: The original value of @v.
	2734	+ */
1133	2735	static __always_inline s64
1134		−atomic64_xchg_relaxed(atomic64_t *v, s64 i)
	2736	+atomic64_xchg_relaxed(atomic64_t *v, s64 new)
1135	2737	{
1136	2738	instrument_atomic_read_write(v, sizeof(*v));
1137		− return arch_atomic64_xchg_relaxed(v, i);
	2739	+ return raw_atomic64_xchg_relaxed(v, new);
1138	2740	}
1139	2741
	2742	+/**
	2743	+ * atomic64_cmpxchg() - atomic compare and exchange with full ordering
	2744	+ * @v: pointer to atomic64_t
	2745	+ * @old: s64 value to compare with
	2746	+ * @new: s64 value to assign
	2747	+ *
	2748	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	2749	+ *
	2750	+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there.
	2751	+ *
	2752	+ * Return: The original value of @v.
	2753	+ */
1140	2754	static __always_inline s64
1141	2755	atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1142	2756	{
1143	2757	kcsan_mb();
1144	2758	instrument_atomic_read_write(v, sizeof(*v));
1145		− return arch_atomic64_cmpxchg(v, old, new);
	2759	+ return raw_atomic64_cmpxchg(v, old, new);
1146	2760	}
1147	2761
	2762	+/**
	2763	+ * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	2764	+ * @v: pointer to atomic64_t
	2765	+ * @old: s64 value to compare with
	2766	+ * @new: s64 value to assign
	2767	+ *
	2768	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	2769	+ *
	2770	+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there.
	2771	+ *
	2772	+ * Return: The original value of @v.
	2773	+ */
1148	2774	static __always_inline s64
1149	2775	atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1150	2776	{
1151	2777	instrument_atomic_read_write(v, sizeof(*v));
1152		− return arch_atomic64_cmpxchg_acquire(v, old, new);
	2778	+ return raw_atomic64_cmpxchg_acquire(v, old, new);
1153	2779	}
1154	2780
	2781	+/**
	2782	+ * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
	2783	+ * @v: pointer to atomic64_t
	2784	+ * @old: s64 value to compare with
	2785	+ * @new: s64 value to assign
	2786	+ *
	2787	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	2788	+ *
	2789	+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there.
	2790	+ *
	2791	+ * Return: The original value of @v.
	2792	+ */
1155	2793	static __always_inline s64
1156	2794	atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1157	2795	{
1158	2796	kcsan_release();
1159	2797	instrument_atomic_read_write(v, sizeof(*v));
1160		− return arch_atomic64_cmpxchg_release(v, old, new);
	2798	+ return raw_atomic64_cmpxchg_release(v, old, new);
1161	2799	}
1162	2800
	2801	+/**
	2802	+ * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	2803	+ * @v: pointer to atomic64_t
	2804	+ * @old: s64 value to compare with
	2805	+ * @new: s64 value to assign
	2806	+ *
	2807	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	2808	+ *
	2809	+ * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there.
	2810	+ *
	2811	+ * Return: The original value of @v.
	2812	+ */
1163	2813	static __always_inline s64
1164	2814	atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1165	2815	{
1166	2816	instrument_atomic_read_write(v, sizeof(*v));
1167		− return arch_atomic64_cmpxchg_relaxed(v, old, new);
	2817	+ return raw_atomic64_cmpxchg_relaxed(v, old, new);
1168	2818	}
1169	2819
	2820	+/**
	2821	+ * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
	2822	+ * @v: pointer to atomic64_t
	2823	+ * @old: pointer to s64 value to compare with
	2824	+ * @new: s64 value to assign
	2825	+ *
	2826	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	2827	+ * Otherwise, updates @old to the current value of @v.
	2828	+ *
	2829	+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there.
	2830	+ *
	2831	+ * Return: @true if the exchange occured, @false otherwise.
	2832	+ */
1170	2833	static __always_inline bool
1171	2834	atomic64_try_cmpxchg(atomic64_t v, s64 old, s64 new)
1172	2835	{
1173	2836	kcsan_mb();
1174	2837	instrument_atomic_read_write(v, sizeof(*v));
1175	2838	instrument_atomic_read_write(old, sizeof(*old));
1176		− return arch_atomic64_try_cmpxchg(v, old, new);
1177		−}
1178		−
	2839	+ return raw_atomic64_try_cmpxchg(v, old, new);
	2840	+}
	2841	+
	2842	+/**
	2843	+ * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	2844	+ * @v: pointer to atomic64_t
	2845	+ * @old: pointer to s64 value to compare with
	2846	+ * @new: s64 value to assign
	2847	+ *
	2848	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	2849	+ * Otherwise, updates @old to the current value of @v.
	2850	+ *
	2851	+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there.
	2852	+ *
	2853	+ * Return: @true if the exchange occured, @false otherwise.
	2854	+ */
1179	2855	static __always_inline bool
1180	2856	atomic64_try_cmpxchg_acquire(atomic64_t v, s64 old, s64 new)
1181	2857	{
1182	2858	instrument_atomic_read_write(v, sizeof(*v));
1183	2859	instrument_atomic_read_write(old, sizeof(*old));
1184		− return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1185		−}
1186		−
	2860	+ return raw_atomic64_try_cmpxchg_acquire(v, old, new);
	2861	+}
	2862	+
	2863	+/**
	2864	+ * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
	2865	+ * @v: pointer to atomic64_t
	2866	+ * @old: pointer to s64 value to compare with
	2867	+ * @new: s64 value to assign
	2868	+ *
	2869	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	2870	+ * Otherwise, updates @old to the current value of @v.
	2871	+ *
	2872	+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there.
	2873	+ *
	2874	+ * Return: @true if the exchange occured, @false otherwise.
	2875	+ */
1187	2876	static __always_inline bool
1188	2877	atomic64_try_cmpxchg_release(atomic64_t v, s64 old, s64 new)
1189	2878	{
1190	2879	kcsan_release();
1191	2880	instrument_atomic_read_write(v, sizeof(*v));
1192	2881	instrument_atomic_read_write(old, sizeof(*old));
1193		− return arch_atomic64_try_cmpxchg_release(v, old, new);
1194		−}
1195		−
	2882	+ return raw_atomic64_try_cmpxchg_release(v, old, new);
	2883	+}
	2884	+
	2885	+/**
	2886	+ * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	2887	+ * @v: pointer to atomic64_t
	2888	+ * @old: pointer to s64 value to compare with
	2889	+ * @new: s64 value to assign
	2890	+ *
	2891	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	2892	+ * Otherwise, updates @old to the current value of @v.
	2893	+ *
	2894	+ * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there.
	2895	+ *
	2896	+ * Return: @true if the exchange occured, @false otherwise.
	2897	+ */
1196	2898	static __always_inline bool
1197	2899	atomic64_try_cmpxchg_relaxed(atomic64_t v, s64 old, s64 new)
1198	2900	{
1199	2901	instrument_atomic_read_write(v, sizeof(*v));
1200	2902	instrument_atomic_read_write(old, sizeof(*old));
1201		− return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1202		−}
1203		−
	2903	+ return raw_atomic64_try_cmpxchg_relaxed(v, old, new);
	2904	+}
	2905	+
	2906	+/**
	2907	+ * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
	2908	+ * @i: s64 value to add
	2909	+ * @v: pointer to atomic64_t
	2910	+ *
	2911	+ * Atomically updates @v to (@v - @i) with full ordering.
	2912	+ *
	2913	+ * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there.
	2914	+ *
	2915	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	2916	+ */
1204	2917	static __always_inline bool
1205	2918	atomic64_sub_and_test(s64 i, atomic64_t *v)
1206	2919	{
1207	2920	kcsan_mb();
1208	2921	instrument_atomic_read_write(v, sizeof(*v));
1209		− return arch_atomic64_sub_and_test(i, v);
	2922	+ return raw_atomic64_sub_and_test(i, v);
1210	2923	}
1211	2924
	2925	+/**
	2926	+ * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
	2927	+ * @v: pointer to atomic64_t
	2928	+ *
	2929	+ * Atomically updates @v to (@v - 1) with full ordering.
	2930	+ *
	2931	+ * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there.
	2932	+ *
	2933	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	2934	+ */
1212	2935	static __always_inline bool
1213	2936	atomic64_dec_and_test(atomic64_t *v)
1214	2937	{
1215	2938	kcsan_mb();
1216	2939	instrument_atomic_read_write(v, sizeof(*v));
1217		− return arch_atomic64_dec_and_test(v);
	2940	+ return raw_atomic64_dec_and_test(v);
1218	2941	}
1219	2942
	2943	+/**
	2944	+ * atomic64_inc_and_test() - atomic increment and test if zero with full ordering
	2945	+ * @v: pointer to atomic64_t
	2946	+ *
	2947	+ * Atomically updates @v to (@v + 1) with full ordering.
	2948	+ *
	2949	+ * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there.
	2950	+ *
	2951	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	2952	+ */
1220	2953	static __always_inline bool
1221	2954	atomic64_inc_and_test(atomic64_t *v)
1222	2955	{
1223	2956	kcsan_mb();
1224	2957	instrument_atomic_read_write(v, sizeof(*v));
1225		− return arch_atomic64_inc_and_test(v);
	2958	+ return raw_atomic64_inc_and_test(v);
1226	2959	}
1227	2960
	2961	+/**
	2962	+ * atomic64_add_negative() - atomic add and test if negative with full ordering
	2963	+ * @i: s64 value to add
	2964	+ * @v: pointer to atomic64_t
	2965	+ *
	2966	+ * Atomically updates @v to (@v + @i) with full ordering.
	2967	+ *
	2968	+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there.
	2969	+ *
	2970	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	2971	+ */
1228	2972	static __always_inline bool
1229	2973	atomic64_add_negative(s64 i, atomic64_t *v)
1230	2974	{
1231	2975	kcsan_mb();
1232	2976	instrument_atomic_read_write(v, sizeof(*v));
1233		− return arch_atomic64_add_negative(i, v);
	2977	+ return raw_atomic64_add_negative(i, v);
1234	2978	}
1235	2979
	2980	+/**
	2981	+ * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
	2982	+ * @i: s64 value to add
	2983	+ * @v: pointer to atomic64_t
	2984	+ *
	2985	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	2986	+ *
	2987	+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there.
	2988	+ *
	2989	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	2990	+ */
1236	2991	static __always_inline bool
1237	2992	atomic64_add_negative_acquire(s64 i, atomic64_t *v)
1238	2993	{
1239	2994	instrument_atomic_read_write(v, sizeof(*v));
1240		− return arch_atomic64_add_negative_acquire(i, v);
	2995	+ return raw_atomic64_add_negative_acquire(i, v);
1241	2996	}
1242	2997
	2998	+/**
	2999	+ * atomic64_add_negative_release() - atomic add and test if negative with release ordering
	3000	+ * @i: s64 value to add
	3001	+ * @v: pointer to atomic64_t
	3002	+ *
	3003	+ * Atomically updates @v to (@v + @i) with release ordering.
	3004	+ *
	3005	+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there.
	3006	+ *
	3007	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	3008	+ */
1243	3009	static __always_inline bool
1244	3010	atomic64_add_negative_release(s64 i, atomic64_t *v)
1245	3011	{
1246	3012	kcsan_release();
1247	3013	instrument_atomic_read_write(v, sizeof(*v));
1248		− return arch_atomic64_add_negative_release(i, v);
	3014	+ return raw_atomic64_add_negative_release(i, v);
1249	3015	}
1250	3016
	3017	+/**
	3018	+ * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
	3019	+ * @i: s64 value to add
	3020	+ * @v: pointer to atomic64_t
	3021	+ *
	3022	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	3023	+ *
	3024	+ * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there.
	3025	+ *
	3026	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	3027	+ */
1251	3028	static __always_inline bool
1252	3029	atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
1253	3030	{
1254	3031	instrument_atomic_read_write(v, sizeof(*v));
1255		− return arch_atomic64_add_negative_relaxed(i, v);
	3032	+ return raw_atomic64_add_negative_relaxed(i, v);
1256	3033	}
1257	3034
	3035	+/**
	3036	+ * atomic64_fetch_add_unless() - atomic add unless value with full ordering
	3037	+ * @v: pointer to atomic64_t
	3038	+ * @a: s64 value to add
	3039	+ * @u: s64 value to compare with
	3040	+ *
	3041	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	3042	+ *
	3043	+ * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there.
	3044	+ *
	3045	+ * Return: The original value of @v.
	3046	+ */
1258	3047	static __always_inline s64
1259	3048	atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1260	3049	{
1261	3050	kcsan_mb();
1262	3051	instrument_atomic_read_write(v, sizeof(*v));
1263		− return arch_atomic64_fetch_add_unless(v, a, u);
	3052	+ return raw_atomic64_fetch_add_unless(v, a, u);
1264	3053	}
1265	3054
	3055	+/**
	3056	+ * atomic64_add_unless() - atomic add unless value with full ordering
	3057	+ * @v: pointer to atomic64_t
	3058	+ * @a: s64 value to add
	3059	+ * @u: s64 value to compare with
	3060	+ *
	3061	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	3062	+ *
	3063	+ * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there.
	3064	+ *
	3065	+ * Return: @true if @v was updated, @false otherwise.
	3066	+ */
1266	3067	static __always_inline bool
1267	3068	atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1268	3069	{
1269	3070	kcsan_mb();
1270	3071	instrument_atomic_read_write(v, sizeof(*v));
1271		− return arch_atomic64_add_unless(v, a, u);
	3072	+ return raw_atomic64_add_unless(v, a, u);
1272	3073	}
1273	3074
	3075	+/**
	3076	+ * atomic64_inc_not_zero() - atomic increment unless zero with full ordering
	3077	+ * @v: pointer to atomic64_t
	3078	+ *
	3079	+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	3080	+ *
	3081	+ * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there.
	3082	+ *
	3083	+ * Return: @true if @v was updated, @false otherwise.
	3084	+ */
1274	3085	static __always_inline bool
1275	3086	atomic64_inc_not_zero(atomic64_t *v)
1276	3087	{
1277	3088	kcsan_mb();
1278	3089	instrument_atomic_read_write(v, sizeof(*v));
1279		− return arch_atomic64_inc_not_zero(v);
	3090	+ return raw_atomic64_inc_not_zero(v);
1280	3091	}
1281	3092
	3093	+/**
	3094	+ * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
	3095	+ * @v: pointer to atomic64_t
	3096	+ *
	3097	+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	3098	+ *
	3099	+ * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there.
	3100	+ *
	3101	+ * Return: @true if @v was updated, @false otherwise.
	3102	+ */
1282	3103	static __always_inline bool
1283	3104	atomic64_inc_unless_negative(atomic64_t *v)
1284	3105	{
1285	3106	kcsan_mb();
1286	3107	instrument_atomic_read_write(v, sizeof(*v));
1287		− return arch_atomic64_inc_unless_negative(v);
	3108	+ return raw_atomic64_inc_unless_negative(v);
1288	3109	}
1289	3110
	3111	+/**
	3112	+ * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
	3113	+ * @v: pointer to atomic64_t
	3114	+ *
	3115	+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	3116	+ *
	3117	+ * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there.
	3118	+ *
	3119	+ * Return: @true if @v was updated, @false otherwise.
	3120	+ */
1290	3121	static __always_inline bool
1291	3122	atomic64_dec_unless_positive(atomic64_t *v)
1292	3123	{
1293	3124	kcsan_mb();
1294	3125	instrument_atomic_read_write(v, sizeof(*v));
1295		− return arch_atomic64_dec_unless_positive(v);
	3126	+ return raw_atomic64_dec_unless_positive(v);
1296	3127	}
1297	3128
	3129	+/**
	3130	+ * atomic64_dec_if_positive() - atomic decrement if positive with full ordering
	3131	+ * @v: pointer to atomic64_t
	3132	+ *
	3133	+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	3134	+ *
	3135	+ * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there.
	3136	+ *
	3137	+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
	3138	+ */
1298	3139	static __always_inline s64
1299	3140	atomic64_dec_if_positive(atomic64_t *v)
1300	3141	{
1301	3142	kcsan_mb();
1302	3143	instrument_atomic_read_write(v, sizeof(*v));
1303		− return arch_atomic64_dec_if_positive(v);
	3144	+ return raw_atomic64_dec_if_positive(v);
1304	3145	}
1305	3146
	3147	+/**
	3148	+ * atomic_long_read() - atomic load with relaxed ordering
	3149	+ * @v: pointer to atomic_long_t
	3150	+ *
	3151	+ * Atomically loads the value of @v with relaxed ordering.
	3152	+ *
	3153	+ * Unsafe to use in noinstr code; use raw_atomic_long_read() there.
	3154	+ *
	3155	+ * Return: The value loaded from @v.
	3156	+ */
1306	3157	static __always_inline long
1307	3158	atomic_long_read(const atomic_long_t *v)
1308	3159	{
1309	3160	instrument_atomic_read(v, sizeof(*v));
1310		− return arch_atomic_long_read(v);
1311		−}
1312		−
	3161	+ return raw_atomic_long_read(v);
	3162	+}
	3163	+
	3164	+/**
	3165	+ * atomic_long_read_acquire() - atomic load with acquire ordering
	3166	+ * @v: pointer to atomic_long_t
	3167	+ *
	3168	+ * Atomically loads the value of @v with acquire ordering.
	3169	+ *
	3170	+ * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there.
	3171	+ *
	3172	+ * Return: The value loaded from @v.
	3173	+ */
1313	3174	static __always_inline long
1314	3175	atomic_long_read_acquire(const atomic_long_t *v)
1315	3176	{
1316	3177	instrument_atomic_read(v, sizeof(*v));
1317		− return arch_atomic_long_read_acquire(v);
1318		−}
1319		−
	3178	+ return raw_atomic_long_read_acquire(v);
	3179	+}
	3180	+
	3181	+/**
	3182	+ * atomic_long_set() - atomic set with relaxed ordering
	3183	+ * @v: pointer to atomic_long_t
	3184	+ * @i: long value to assign
	3185	+ *
	3186	+ * Atomically sets @v to @i with relaxed ordering.
	3187	+ *
	3188	+ * Unsafe to use in noinstr code; use raw_atomic_long_set() there.
	3189	+ *
	3190	+ * Return: Nothing.
	3191	+ */
1320	3192	static __always_inline void
1321	3193	atomic_long_set(atomic_long_t *v, long i)
1322	3194	{
1323	3195	instrument_atomic_write(v, sizeof(*v));
1324		− arch_atomic_long_set(v, i);
1325		−}
1326		−
	3196	+ raw_atomic_long_set(v, i);
	3197	+}
	3198	+
	3199	+/**
	3200	+ * atomic_long_set_release() - atomic set with release ordering
	3201	+ * @v: pointer to atomic_long_t
	3202	+ * @i: long value to assign
	3203	+ *
	3204	+ * Atomically sets @v to @i with release ordering.
	3205	+ *
	3206	+ * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there.
	3207	+ *
	3208	+ * Return: Nothing.
	3209	+ */
1327	3210	static __always_inline void
1328	3211	atomic_long_set_release(atomic_long_t *v, long i)
1329	3212	{
1330	3213	kcsan_release();
1331	3214	instrument_atomic_write(v, sizeof(*v));
1332		− arch_atomic_long_set_release(v, i);
1333		−}
1334		−
	3215	+ raw_atomic_long_set_release(v, i);
	3216	+}
	3217	+
	3218	+/**
	3219	+ * atomic_long_add() - atomic add with relaxed ordering
	3220	+ * @i: long value to add
	3221	+ * @v: pointer to atomic_long_t
	3222	+ *
	3223	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	3224	+ *
	3225	+ * Unsafe to use in noinstr code; use raw_atomic_long_add() there.
	3226	+ *
	3227	+ * Return: Nothing.
	3228	+ */
1335	3229	static __always_inline void
1336	3230	atomic_long_add(long i, atomic_long_t *v)
1337	3231	{
1338	3232	instrument_atomic_read_write(v, sizeof(*v));
1339		− arch_atomic_long_add(i, v);
	3233	+ raw_atomic_long_add(i, v);
1340	3234	}
1341	3235
	3236	+/**
	3237	+ * atomic_long_add_return() - atomic add with full ordering
	3238	+ * @i: long value to add
	3239	+ * @v: pointer to atomic_long_t
	3240	+ *
	3241	+ * Atomically updates @v to (@v + @i) with full ordering.
	3242	+ *
	3243	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there.
	3244	+ *
	3245	+ * Return: The updated value of @v.
	3246	+ */
1342	3247	static __always_inline long
1343	3248	atomic_long_add_return(long i, atomic_long_t *v)
1344	3249	{
1345	3250	kcsan_mb();
1346	3251	instrument_atomic_read_write(v, sizeof(*v));
1347		− return arch_atomic_long_add_return(i, v);
	3252	+ return raw_atomic_long_add_return(i, v);
1348	3253	}
1349	3254
	3255	+/**
	3256	+ * atomic_long_add_return_acquire() - atomic add with acquire ordering
	3257	+ * @i: long value to add
	3258	+ * @v: pointer to atomic_long_t
	3259	+ *
	3260	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	3261	+ *
	3262	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there.
	3263	+ *
	3264	+ * Return: The updated value of @v.
	3265	+ */
1350	3266	static __always_inline long
1351	3267	atomic_long_add_return_acquire(long i, atomic_long_t *v)
1352	3268	{
1353	3269	instrument_atomic_read_write(v, sizeof(*v));
1354		− return arch_atomic_long_add_return_acquire(i, v);
	3270	+ return raw_atomic_long_add_return_acquire(i, v);
1355	3271	}
1356	3272
	3273	+/**
	3274	+ * atomic_long_add_return_release() - atomic add with release ordering
	3275	+ * @i: long value to add
	3276	+ * @v: pointer to atomic_long_t
	3277	+ *
	3278	+ * Atomically updates @v to (@v + @i) with release ordering.
	3279	+ *
	3280	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there.
	3281	+ *
	3282	+ * Return: The updated value of @v.
	3283	+ */
1357	3284	static __always_inline long
1358	3285	atomic_long_add_return_release(long i, atomic_long_t *v)
1359	3286	{
1360	3287	kcsan_release();
1361	3288	instrument_atomic_read_write(v, sizeof(*v));
1362		− return arch_atomic_long_add_return_release(i, v);
	3289	+ return raw_atomic_long_add_return_release(i, v);
1363	3290	}
1364	3291
	3292	+/**
	3293	+ * atomic_long_add_return_relaxed() - atomic add with relaxed ordering
	3294	+ * @i: long value to add
	3295	+ * @v: pointer to atomic_long_t
	3296	+ *
	3297	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	3298	+ *
	3299	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there.
	3300	+ *
	3301	+ * Return: The updated value of @v.
	3302	+ */
1365	3303	static __always_inline long
1366	3304	atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1367	3305	{
1368	3306	instrument_atomic_read_write(v, sizeof(*v));
1369		− return arch_atomic_long_add_return_relaxed(i, v);
	3307	+ return raw_atomic_long_add_return_relaxed(i, v);
1370	3308	}
1371	3309
	3310	+/**
	3311	+ * atomic_long_fetch_add() - atomic add with full ordering
	3312	+ * @i: long value to add
	3313	+ * @v: pointer to atomic_long_t
	3314	+ *
	3315	+ * Atomically updates @v to (@v + @i) with full ordering.
	3316	+ *
	3317	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there.
	3318	+ *
	3319	+ * Return: The original value of @v.
	3320	+ */
1372	3321	static __always_inline long
1373	3322	atomic_long_fetch_add(long i, atomic_long_t *v)
1374	3323	{
1375	3324	kcsan_mb();
1376	3325	instrument_atomic_read_write(v, sizeof(*v));
1377		− return arch_atomic_long_fetch_add(i, v);
	3326	+ return raw_atomic_long_fetch_add(i, v);
1378	3327	}
1379	3328
	3329	+/**
	3330	+ * atomic_long_fetch_add_acquire() - atomic add with acquire ordering
	3331	+ * @i: long value to add
	3332	+ * @v: pointer to atomic_long_t
	3333	+ *
	3334	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	3335	+ *
	3336	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there.
	3337	+ *
	3338	+ * Return: The original value of @v.
	3339	+ */
1380	3340	static __always_inline long
1381	3341	atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1382	3342	{
1383	3343	instrument_atomic_read_write(v, sizeof(*v));
1384		− return arch_atomic_long_fetch_add_acquire(i, v);
	3344	+ return raw_atomic_long_fetch_add_acquire(i, v);
1385	3345	}
1386	3346
	3347	+/**
	3348	+ * atomic_long_fetch_add_release() - atomic add with release ordering
	3349	+ * @i: long value to add
	3350	+ * @v: pointer to atomic_long_t
	3351	+ *
	3352	+ * Atomically updates @v to (@v + @i) with release ordering.
	3353	+ *
	3354	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there.
	3355	+ *
	3356	+ * Return: The original value of @v.
	3357	+ */
1387	3358	static __always_inline long
1388	3359	atomic_long_fetch_add_release(long i, atomic_long_t *v)
1389	3360	{
1390	3361	kcsan_release();
1391	3362	instrument_atomic_read_write(v, sizeof(*v));
1392		− return arch_atomic_long_fetch_add_release(i, v);
	3363	+ return raw_atomic_long_fetch_add_release(i, v);
1393	3364	}
1394	3365
	3366	+/**
	3367	+ * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
	3368	+ * @i: long value to add
	3369	+ * @v: pointer to atomic_long_t
	3370	+ *
	3371	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	3372	+ *
	3373	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there.
	3374	+ *
	3375	+ * Return: The original value of @v.
	3376	+ */
1395	3377	static __always_inline long
1396	3378	atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1397	3379	{
1398	3380	instrument_atomic_read_write(v, sizeof(*v));
1399		− return arch_atomic_long_fetch_add_relaxed(i, v);
	3381	+ return raw_atomic_long_fetch_add_relaxed(i, v);
1400	3382	}
1401	3383
	3384	+/**
	3385	+ * atomic_long_sub() - atomic subtract with relaxed ordering
	3386	+ * @i: long value to subtract
	3387	+ * @v: pointer to atomic_long_t
	3388	+ *
	3389	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	3390	+ *
	3391	+ * Unsafe to use in noinstr code; use raw_atomic_long_sub() there.
	3392	+ *
	3393	+ * Return: Nothing.
	3394	+ */
1402	3395	static __always_inline void
1403	3396	atomic_long_sub(long i, atomic_long_t *v)
1404	3397	{
1405	3398	instrument_atomic_read_write(v, sizeof(*v));
1406		− arch_atomic_long_sub(i, v);
	3399	+ raw_atomic_long_sub(i, v);
1407	3400	}
1408	3401
	3402	+/**
	3403	+ * atomic_long_sub_return() - atomic subtract with full ordering
	3404	+ * @i: long value to subtract
	3405	+ * @v: pointer to atomic_long_t
	3406	+ *
	3407	+ * Atomically updates @v to (@v - @i) with full ordering.
	3408	+ *
	3409	+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there.
	3410	+ *
	3411	+ * Return: The updated value of @v.
	3412	+ */
1409	3413	static __always_inline long
1410	3414	atomic_long_sub_return(long i, atomic_long_t *v)
1411	3415	{
1412	3416	kcsan_mb();
1413	3417	instrument_atomic_read_write(v, sizeof(*v));
1414		− return arch_atomic_long_sub_return(i, v);
	3418	+ return raw_atomic_long_sub_return(i, v);
1415	3419	}
1416	3420
	3421	+/**
	3422	+ * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
	3423	+ * @i: long value to subtract
	3424	+ * @v: pointer to atomic_long_t
	3425	+ *
	3426	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	3427	+ *
	3428	+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there.
	3429	+ *
	3430	+ * Return: The updated value of @v.
	3431	+ */
1417	3432	static __always_inline long
1418	3433	atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1419	3434	{
1420	3435	instrument_atomic_read_write(v, sizeof(*v));
1421		− return arch_atomic_long_sub_return_acquire(i, v);
	3436	+ return raw_atomic_long_sub_return_acquire(i, v);
1422	3437	}
1423	3438
	3439	+/**
	3440	+ * atomic_long_sub_return_release() - atomic subtract with release ordering
	3441	+ * @i: long value to subtract
	3442	+ * @v: pointer to atomic_long_t
	3443	+ *
	3444	+ * Atomically updates @v to (@v - @i) with release ordering.
	3445	+ *
	3446	+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there.
	3447	+ *
	3448	+ * Return: The updated value of @v.
	3449	+ */
1424	3450	static __always_inline long
1425	3451	atomic_long_sub_return_release(long i, atomic_long_t *v)
1426	3452	{
1427	3453	kcsan_release();
1428	3454	instrument_atomic_read_write(v, sizeof(*v));
1429		− return arch_atomic_long_sub_return_release(i, v);
	3455	+ return raw_atomic_long_sub_return_release(i, v);
1430	3456	}
1431	3457
	3458	+/**
	3459	+ * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
	3460	+ * @i: long value to subtract
	3461	+ * @v: pointer to atomic_long_t
	3462	+ *
	3463	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	3464	+ *
	3465	+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there.
	3466	+ *
	3467	+ * Return: The updated value of @v.
	3468	+ */
1432	3469	static __always_inline long
1433	3470	atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1434	3471	{
1435	3472	instrument_atomic_read_write(v, sizeof(*v));
1436		− return arch_atomic_long_sub_return_relaxed(i, v);
	3473	+ return raw_atomic_long_sub_return_relaxed(i, v);
1437	3474	}
1438	3475
	3476	+/**
	3477	+ * atomic_long_fetch_sub() - atomic subtract with full ordering
	3478	+ * @i: long value to subtract
	3479	+ * @v: pointer to atomic_long_t
	3480	+ *
	3481	+ * Atomically updates @v to (@v - @i) with full ordering.
	3482	+ *
	3483	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there.
	3484	+ *
	3485	+ * Return: The original value of @v.
	3486	+ */
1439	3487	static __always_inline long
1440	3488	atomic_long_fetch_sub(long i, atomic_long_t *v)
1441	3489	{
1442	3490	kcsan_mb();
1443	3491	instrument_atomic_read_write(v, sizeof(*v));
1444		− return arch_atomic_long_fetch_sub(i, v);
	3492	+ return raw_atomic_long_fetch_sub(i, v);
1445	3493	}
1446	3494
	3495	+/**
	3496	+ * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
	3497	+ * @i: long value to subtract
	3498	+ * @v: pointer to atomic_long_t
	3499	+ *
	3500	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	3501	+ *
	3502	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there.
	3503	+ *
	3504	+ * Return: The original value of @v.
	3505	+ */
1447	3506	static __always_inline long
1448	3507	atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1449	3508	{
1450	3509	instrument_atomic_read_write(v, sizeof(*v));
1451		− return arch_atomic_long_fetch_sub_acquire(i, v);
	3510	+ return raw_atomic_long_fetch_sub_acquire(i, v);
1452	3511	}
1453	3512
	3513	+/**
	3514	+ * atomic_long_fetch_sub_release() - atomic subtract with release ordering
	3515	+ * @i: long value to subtract
	3516	+ * @v: pointer to atomic_long_t
	3517	+ *
	3518	+ * Atomically updates @v to (@v - @i) with release ordering.
	3519	+ *
	3520	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there.
	3521	+ *
	3522	+ * Return: The original value of @v.
	3523	+ */
1454	3524	static __always_inline long
1455	3525	atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1456	3526	{
1457	3527	kcsan_release();
1458	3528	instrument_atomic_read_write(v, sizeof(*v));
1459		− return arch_atomic_long_fetch_sub_release(i, v);
	3529	+ return raw_atomic_long_fetch_sub_release(i, v);
1460	3530	}
1461	3531
	3532	+/**
	3533	+ * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
	3534	+ * @i: long value to subtract
	3535	+ * @v: pointer to atomic_long_t
	3536	+ *
	3537	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	3538	+ *
	3539	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there.
	3540	+ *
	3541	+ * Return: The original value of @v.
	3542	+ */
1462	3543	static __always_inline long
1463	3544	atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1464	3545	{
1465	3546	instrument_atomic_read_write(v, sizeof(*v));
1466		− return arch_atomic_long_fetch_sub_relaxed(i, v);
	3547	+ return raw_atomic_long_fetch_sub_relaxed(i, v);
1467	3548	}
1468	3549
	3550	+/**
	3551	+ * atomic_long_inc() - atomic increment with relaxed ordering
	3552	+ * @v: pointer to atomic_long_t
	3553	+ *
	3554	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	3555	+ *
	3556	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc() there.
	3557	+ *
	3558	+ * Return: Nothing.
	3559	+ */
1469	3560	static __always_inline void
1470	3561	atomic_long_inc(atomic_long_t *v)
1471	3562	{
1472	3563	instrument_atomic_read_write(v, sizeof(*v));
1473		− arch_atomic_long_inc(v);
	3564	+ raw_atomic_long_inc(v);
1474	3565	}
1475	3566
	3567	+/**
	3568	+ * atomic_long_inc_return() - atomic increment with full ordering
	3569	+ * @v: pointer to atomic_long_t
	3570	+ *
	3571	+ * Atomically updates @v to (@v + 1) with full ordering.
	3572	+ *
	3573	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there.
	3574	+ *
	3575	+ * Return: The updated value of @v.
	3576	+ */
1476	3577	static __always_inline long
1477	3578	atomic_long_inc_return(atomic_long_t *v)
1478	3579	{
1479	3580	kcsan_mb();
1480	3581	instrument_atomic_read_write(v, sizeof(*v));
1481		− return arch_atomic_long_inc_return(v);
	3582	+ return raw_atomic_long_inc_return(v);
1482	3583	}
1483	3584
	3585	+/**
	3586	+ * atomic_long_inc_return_acquire() - atomic increment with acquire ordering
	3587	+ * @v: pointer to atomic_long_t
	3588	+ *
	3589	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	3590	+ *
	3591	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there.
	3592	+ *
	3593	+ * Return: The updated value of @v.
	3594	+ */
1484	3595	static __always_inline long
1485	3596	atomic_long_inc_return_acquire(atomic_long_t *v)
1486	3597	{
1487	3598	instrument_atomic_read_write(v, sizeof(*v));
1488		− return arch_atomic_long_inc_return_acquire(v);
	3599	+ return raw_atomic_long_inc_return_acquire(v);
1489	3600	}
1490	3601
	3602	+/**
	3603	+ * atomic_long_inc_return_release() - atomic increment with release ordering
	3604	+ * @v: pointer to atomic_long_t
	3605	+ *
	3606	+ * Atomically updates @v to (@v + 1) with release ordering.
	3607	+ *
	3608	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there.
	3609	+ *
	3610	+ * Return: The updated value of @v.
	3611	+ */
1491	3612	static __always_inline long
1492	3613	atomic_long_inc_return_release(atomic_long_t *v)
1493	3614	{
1494	3615	kcsan_release();
1495	3616	instrument_atomic_read_write(v, sizeof(*v));
1496		− return arch_atomic_long_inc_return_release(v);
	3617	+ return raw_atomic_long_inc_return_release(v);
1497	3618	}
1498	3619
	3620	+/**
	3621	+ * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
	3622	+ * @v: pointer to atomic_long_t
	3623	+ *
	3624	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	3625	+ *
	3626	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there.
	3627	+ *
	3628	+ * Return: The updated value of @v.
	3629	+ */
1499	3630	static __always_inline long
1500	3631	atomic_long_inc_return_relaxed(atomic_long_t *v)
1501	3632	{
1502	3633	instrument_atomic_read_write(v, sizeof(*v));
1503		− return arch_atomic_long_inc_return_relaxed(v);
	3634	+ return raw_atomic_long_inc_return_relaxed(v);
1504	3635	}
1505	3636
	3637	+/**
	3638	+ * atomic_long_fetch_inc() - atomic increment with full ordering
	3639	+ * @v: pointer to atomic_long_t
	3640	+ *
	3641	+ * Atomically updates @v to (@v + 1) with full ordering.
	3642	+ *
	3643	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there.
	3644	+ *
	3645	+ * Return: The original value of @v.
	3646	+ */
1506	3647	static __always_inline long
1507	3648	atomic_long_fetch_inc(atomic_long_t *v)
1508	3649	{
1509	3650	kcsan_mb();
1510	3651	instrument_atomic_read_write(v, sizeof(*v));
1511		− return arch_atomic_long_fetch_inc(v);
	3652	+ return raw_atomic_long_fetch_inc(v);
1512	3653	}
1513	3654
	3655	+/**
	3656	+ * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
	3657	+ * @v: pointer to atomic_long_t
	3658	+ *
	3659	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	3660	+ *
	3661	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there.
	3662	+ *
	3663	+ * Return: The original value of @v.
	3664	+ */
1514	3665	static __always_inline long
1515	3666	atomic_long_fetch_inc_acquire(atomic_long_t *v)
1516	3667	{
1517	3668	instrument_atomic_read_write(v, sizeof(*v));
1518		− return arch_atomic_long_fetch_inc_acquire(v);
	3669	+ return raw_atomic_long_fetch_inc_acquire(v);
1519	3670	}
1520	3671
	3672	+/**
	3673	+ * atomic_long_fetch_inc_release() - atomic increment with release ordering
	3674	+ * @v: pointer to atomic_long_t
	3675	+ *
	3676	+ * Atomically updates @v to (@v + 1) with release ordering.
	3677	+ *
	3678	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there.
	3679	+ *
	3680	+ * Return: The original value of @v.
	3681	+ */
1521	3682	static __always_inline long
1522	3683	atomic_long_fetch_inc_release(atomic_long_t *v)
1523	3684	{
1524	3685	kcsan_release();
1525	3686	instrument_atomic_read_write(v, sizeof(*v));
1526		− return arch_atomic_long_fetch_inc_release(v);
	3687	+ return raw_atomic_long_fetch_inc_release(v);
1527	3688	}
1528	3689
	3690	+/**
	3691	+ * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
	3692	+ * @v: pointer to atomic_long_t
	3693	+ *
	3694	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	3695	+ *
	3696	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there.
	3697	+ *
	3698	+ * Return: The original value of @v.
	3699	+ */
1529	3700	static __always_inline long
1530	3701	atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1531	3702	{
1532	3703	instrument_atomic_read_write(v, sizeof(*v));
1533		− return arch_atomic_long_fetch_inc_relaxed(v);
	3704	+ return raw_atomic_long_fetch_inc_relaxed(v);
1534	3705	}
1535	3706
	3707	+/**
	3708	+ * atomic_long_dec() - atomic decrement with relaxed ordering
	3709	+ * @v: pointer to atomic_long_t
	3710	+ *
	3711	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	3712	+ *
	3713	+ * Unsafe to use in noinstr code; use raw_atomic_long_dec() there.
	3714	+ *
	3715	+ * Return: Nothing.
	3716	+ */
1536	3717	static __always_inline void
1537	3718	atomic_long_dec(atomic_long_t *v)
1538	3719	{
1539	3720	instrument_atomic_read_write(v, sizeof(*v));
1540		− arch_atomic_long_dec(v);
	3721	+ raw_atomic_long_dec(v);
1541	3722	}
1542	3723
	3724	+/**
	3725	+ * atomic_long_dec_return() - atomic decrement with full ordering
	3726	+ * @v: pointer to atomic_long_t
	3727	+ *
	3728	+ * Atomically updates @v to (@v - 1) with full ordering.
	3729	+ *
	3730	+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there.
	3731	+ *
	3732	+ * Return: The updated value of @v.
	3733	+ */
1543	3734	static __always_inline long
1544	3735	atomic_long_dec_return(atomic_long_t *v)
1545	3736	{
1546	3737	kcsan_mb();
1547	3738	instrument_atomic_read_write(v, sizeof(*v));
1548		− return arch_atomic_long_dec_return(v);
	3739	+ return raw_atomic_long_dec_return(v);
1549	3740	}
1550	3741
	3742	+/**
	3743	+ * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
	3744	+ * @v: pointer to atomic_long_t
	3745	+ *
	3746	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	3747	+ *
	3748	+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there.
	3749	+ *
	3750	+ * Return: The updated value of @v.
	3751	+ */
1551	3752	static __always_inline long
1552	3753	atomic_long_dec_return_acquire(atomic_long_t *v)
1553	3754	{
1554	3755	instrument_atomic_read_write(v, sizeof(*v));
1555		− return arch_atomic_long_dec_return_acquire(v);
	3756	+ return raw_atomic_long_dec_return_acquire(v);
1556	3757	}
1557	3758
	3759	+/**
	3760	+ * atomic_long_dec_return_release() - atomic decrement with release ordering
	3761	+ * @v: pointer to atomic_long_t
	3762	+ *
	3763	+ * Atomically updates @v to (@v - 1) with release ordering.
	3764	+ *
	3765	+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there.
	3766	+ *
	3767	+ * Return: The updated value of @v.
	3768	+ */
1558	3769	static __always_inline long
1559	3770	atomic_long_dec_return_release(atomic_long_t *v)
1560	3771	{
1561	3772	kcsan_release();
1562	3773	instrument_atomic_read_write(v, sizeof(*v));
1563		− return arch_atomic_long_dec_return_release(v);
	3774	+ return raw_atomic_long_dec_return_release(v);
1564	3775	}
1565	3776
	3777	+/**
	3778	+ * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
	3779	+ * @v: pointer to atomic_long_t
	3780	+ *
	3781	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	3782	+ *
	3783	+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there.
	3784	+ *
	3785	+ * Return: The updated value of @v.
	3786	+ */
1566	3787	static __always_inline long
1567	3788	atomic_long_dec_return_relaxed(atomic_long_t *v)
1568	3789	{
1569	3790	instrument_atomic_read_write(v, sizeof(*v));
1570		− return arch_atomic_long_dec_return_relaxed(v);
	3791	+ return raw_atomic_long_dec_return_relaxed(v);
1571	3792	}
1572	3793
	3794	+/**
	3795	+ * atomic_long_fetch_dec() - atomic decrement with full ordering
	3796	+ * @v: pointer to atomic_long_t
	3797	+ *
	3798	+ * Atomically updates @v to (@v - 1) with full ordering.
	3799	+ *
	3800	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there.
	3801	+ *
	3802	+ * Return: The original value of @v.
	3803	+ */
1573	3804	static __always_inline long
1574	3805	atomic_long_fetch_dec(atomic_long_t *v)
1575	3806	{
1576	3807	kcsan_mb();
1577	3808	instrument_atomic_read_write(v, sizeof(*v));
1578		− return arch_atomic_long_fetch_dec(v);
	3809	+ return raw_atomic_long_fetch_dec(v);
1579	3810	}
1580	3811
	3812	+/**
	3813	+ * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
	3814	+ * @v: pointer to atomic_long_t
	3815	+ *
	3816	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	3817	+ *
	3818	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there.
	3819	+ *
	3820	+ * Return: The original value of @v.
	3821	+ */
1581	3822	static __always_inline long
1582	3823	atomic_long_fetch_dec_acquire(atomic_long_t *v)
1583	3824	{
1584	3825	instrument_atomic_read_write(v, sizeof(*v));
1585		− return arch_atomic_long_fetch_dec_acquire(v);
	3826	+ return raw_atomic_long_fetch_dec_acquire(v);
1586	3827	}
1587	3828
	3829	+/**
	3830	+ * atomic_long_fetch_dec_release() - atomic decrement with release ordering
	3831	+ * @v: pointer to atomic_long_t
	3832	+ *
	3833	+ * Atomically updates @v to (@v - 1) with release ordering.
	3834	+ *
	3835	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there.
	3836	+ *
	3837	+ * Return: The original value of @v.
	3838	+ */
1588	3839	static __always_inline long
1589	3840	atomic_long_fetch_dec_release(atomic_long_t *v)
1590	3841	{
1591	3842	kcsan_release();
1592	3843	instrument_atomic_read_write(v, sizeof(*v));
1593		− return arch_atomic_long_fetch_dec_release(v);
	3844	+ return raw_atomic_long_fetch_dec_release(v);
1594	3845	}
1595	3846
	3847	+/**
	3848	+ * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
	3849	+ * @v: pointer to atomic_long_t
	3850	+ *
	3851	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	3852	+ *
	3853	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there.
	3854	+ *
	3855	+ * Return: The original value of @v.
	3856	+ */
1596	3857	static __always_inline long
1597	3858	atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1598	3859	{
1599	3860	instrument_atomic_read_write(v, sizeof(*v));
1600		− return arch_atomic_long_fetch_dec_relaxed(v);
	3861	+ return raw_atomic_long_fetch_dec_relaxed(v);
1601	3862	}
1602	3863
	3864	+/**
	3865	+ * atomic_long_and() - atomic bitwise AND with relaxed ordering
	3866	+ * @i: long value
	3867	+ * @v: pointer to atomic_long_t
	3868	+ *
	3869	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	3870	+ *
	3871	+ * Unsafe to use in noinstr code; use raw_atomic_long_and() there.
	3872	+ *
	3873	+ * Return: Nothing.
	3874	+ */
1603	3875	static __always_inline void
1604	3876	atomic_long_and(long i, atomic_long_t *v)
1605	3877	{
1606	3878	instrument_atomic_read_write(v, sizeof(*v));
1607		− arch_atomic_long_and(i, v);
	3879	+ raw_atomic_long_and(i, v);
1608	3880	}
1609	3881
	3882	+/**
	3883	+ * atomic_long_fetch_and() - atomic bitwise AND with full ordering
	3884	+ * @i: long value
	3885	+ * @v: pointer to atomic_long_t
	3886	+ *
	3887	+ * Atomically updates @v to (@v & @i) with full ordering.
	3888	+ *
	3889	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there.
	3890	+ *
	3891	+ * Return: The original value of @v.
	3892	+ */
1610	3893	static __always_inline long
1611	3894	atomic_long_fetch_and(long i, atomic_long_t *v)
1612	3895	{
1613	3896	kcsan_mb();
1614	3897	instrument_atomic_read_write(v, sizeof(*v));
1615		− return arch_atomic_long_fetch_and(i, v);
	3898	+ return raw_atomic_long_fetch_and(i, v);
1616	3899	}
1617	3900
	3901	+/**
	3902	+ * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
	3903	+ * @i: long value
	3904	+ * @v: pointer to atomic_long_t
	3905	+ *
	3906	+ * Atomically updates @v to (@v & @i) with acquire ordering.
	3907	+ *
	3908	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there.
	3909	+ *
	3910	+ * Return: The original value of @v.
	3911	+ */
1618	3912	static __always_inline long
1619	3913	atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1620	3914	{
1621	3915	instrument_atomic_read_write(v, sizeof(*v));
1622		− return arch_atomic_long_fetch_and_acquire(i, v);
	3916	+ return raw_atomic_long_fetch_and_acquire(i, v);
1623	3917	}
1624	3918
	3919	+/**
	3920	+ * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
	3921	+ * @i: long value
	3922	+ * @v: pointer to atomic_long_t
	3923	+ *
	3924	+ * Atomically updates @v to (@v & @i) with release ordering.
	3925	+ *
	3926	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there.
	3927	+ *
	3928	+ * Return: The original value of @v.
	3929	+ */
1625	3930	static __always_inline long
1626	3931	atomic_long_fetch_and_release(long i, atomic_long_t *v)
1627	3932	{
1628	3933	kcsan_release();
1629	3934	instrument_atomic_read_write(v, sizeof(*v));
1630		− return arch_atomic_long_fetch_and_release(i, v);
	3935	+ return raw_atomic_long_fetch_and_release(i, v);
1631	3936	}
1632	3937
	3938	+/**
	3939	+ * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
	3940	+ * @i: long value
	3941	+ * @v: pointer to atomic_long_t
	3942	+ *
	3943	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	3944	+ *
	3945	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there.
	3946	+ *
	3947	+ * Return: The original value of @v.
	3948	+ */
1633	3949	static __always_inline long
1634	3950	atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1635	3951	{
1636	3952	instrument_atomic_read_write(v, sizeof(*v));
1637		− return arch_atomic_long_fetch_and_relaxed(i, v);
	3953	+ return raw_atomic_long_fetch_and_relaxed(i, v);
1638	3954	}
1639	3955
	3956	+/**
	3957	+ * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
	3958	+ * @i: long value
	3959	+ * @v: pointer to atomic_long_t
	3960	+ *
	3961	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	3962	+ *
	3963	+ * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there.
	3964	+ *
	3965	+ * Return: Nothing.
	3966	+ */
1640	3967	static __always_inline void
1641	3968	atomic_long_andnot(long i, atomic_long_t *v)
1642	3969	{
1643	3970	instrument_atomic_read_write(v, sizeof(*v));
1644		− arch_atomic_long_andnot(i, v);
	3971	+ raw_atomic_long_andnot(i, v);
1645	3972	}
1646	3973
	3974	+/**
	3975	+ * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
	3976	+ * @i: long value
	3977	+ * @v: pointer to atomic_long_t
	3978	+ *
	3979	+ * Atomically updates @v to (@v & ~@i) with full ordering.
	3980	+ *
	3981	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there.
	3982	+ *
	3983	+ * Return: The original value of @v.
	3984	+ */
1647	3985	static __always_inline long
1648	3986	atomic_long_fetch_andnot(long i, atomic_long_t *v)
1649	3987	{
1650	3988	kcsan_mb();
1651	3989	instrument_atomic_read_write(v, sizeof(*v));
1652		− return arch_atomic_long_fetch_andnot(i, v);
	3990	+ return raw_atomic_long_fetch_andnot(i, v);
1653	3991	}
1654	3992
	3993	+/**
	3994	+ * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
	3995	+ * @i: long value
	3996	+ * @v: pointer to atomic_long_t
	3997	+ *
	3998	+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
	3999	+ *
	4000	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there.
	4001	+ *
	4002	+ * Return: The original value of @v.
	4003	+ */
1655	4004	static __always_inline long
1656	4005	atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1657	4006	{
1658	4007	instrument_atomic_read_write(v, sizeof(*v));
1659		− return arch_atomic_long_fetch_andnot_acquire(i, v);
	4008	+ return raw_atomic_long_fetch_andnot_acquire(i, v);
1660	4009	}
1661	4010
	4011	+/**
	4012	+ * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
	4013	+ * @i: long value
	4014	+ * @v: pointer to atomic_long_t
	4015	+ *
	4016	+ * Atomically updates @v to (@v & ~@i) with release ordering.
	4017	+ *
	4018	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there.
	4019	+ *
	4020	+ * Return: The original value of @v.
	4021	+ */
1662	4022	static __always_inline long
1663	4023	atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1664	4024	{
1665	4025	kcsan_release();
1666	4026	instrument_atomic_read_write(v, sizeof(*v));
1667		− return arch_atomic_long_fetch_andnot_release(i, v);
	4027	+ return raw_atomic_long_fetch_andnot_release(i, v);
1668	4028	}
1669	4029
	4030	+/**
	4031	+ * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
	4032	+ * @i: long value
	4033	+ * @v: pointer to atomic_long_t
	4034	+ *
	4035	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	4036	+ *
	4037	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there.
	4038	+ *
	4039	+ * Return: The original value of @v.
	4040	+ */
1670	4041	static __always_inline long
1671	4042	atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1672	4043	{
1673	4044	instrument_atomic_read_write(v, sizeof(*v));
1674		− return arch_atomic_long_fetch_andnot_relaxed(i, v);
	4045	+ return raw_atomic_long_fetch_andnot_relaxed(i, v);
1675	4046	}
1676	4047
	4048	+/**
	4049	+ * atomic_long_or() - atomic bitwise OR with relaxed ordering
	4050	+ * @i: long value
	4051	+ * @v: pointer to atomic_long_t
	4052	+ *
	4053	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	4054	+ *
	4055	+ * Unsafe to use in noinstr code; use raw_atomic_long_or() there.
	4056	+ *
	4057	+ * Return: Nothing.
	4058	+ */
1677	4059	static __always_inline void
1678	4060	atomic_long_or(long i, atomic_long_t *v)
1679	4061	{
1680	4062	instrument_atomic_read_write(v, sizeof(*v));
1681		− arch_atomic_long_or(i, v);
	4063	+ raw_atomic_long_or(i, v);
1682	4064	}
1683	4065
	4066	+/**
	4067	+ * atomic_long_fetch_or() - atomic bitwise OR with full ordering
	4068	+ * @i: long value
	4069	+ * @v: pointer to atomic_long_t
	4070	+ *
	4071	+ * Atomically updates @v to (@v \| @i) with full ordering.
	4072	+ *
	4073	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there.
	4074	+ *
	4075	+ * Return: The original value of @v.
	4076	+ */
1684	4077	static __always_inline long
1685	4078	atomic_long_fetch_or(long i, atomic_long_t *v)
1686	4079	{
1687	4080	kcsan_mb();
1688	4081	instrument_atomic_read_write(v, sizeof(*v));
1689		− return arch_atomic_long_fetch_or(i, v);
	4082	+ return raw_atomic_long_fetch_or(i, v);
1690	4083	}
1691	4084
	4085	+/**
	4086	+ * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
	4087	+ * @i: long value
	4088	+ * @v: pointer to atomic_long_t
	4089	+ *
	4090	+ * Atomically updates @v to (@v \| @i) with acquire ordering.
	4091	+ *
	4092	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there.
	4093	+ *
	4094	+ * Return: The original value of @v.
	4095	+ */
1692	4096	static __always_inline long
1693	4097	atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1694	4098	{
1695	4099	instrument_atomic_read_write(v, sizeof(*v));
1696		− return arch_atomic_long_fetch_or_acquire(i, v);
	4100	+ return raw_atomic_long_fetch_or_acquire(i, v);
1697	4101	}
1698	4102
	4103	+/**
	4104	+ * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
	4105	+ * @i: long value
	4106	+ * @v: pointer to atomic_long_t
	4107	+ *
	4108	+ * Atomically updates @v to (@v \| @i) with release ordering.
	4109	+ *
	4110	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there.
	4111	+ *
	4112	+ * Return: The original value of @v.
	4113	+ */
1699	4114	static __always_inline long
1700	4115	atomic_long_fetch_or_release(long i, atomic_long_t *v)
1701	4116	{
1702	4117	kcsan_release();
1703	4118	instrument_atomic_read_write(v, sizeof(*v));
1704		− return arch_atomic_long_fetch_or_release(i, v);
	4119	+ return raw_atomic_long_fetch_or_release(i, v);
1705	4120	}
1706	4121
	4122	+/**
	4123	+ * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
	4124	+ * @i: long value
	4125	+ * @v: pointer to atomic_long_t
	4126	+ *
	4127	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	4128	+ *
	4129	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there.
	4130	+ *
	4131	+ * Return: The original value of @v.
	4132	+ */
1707	4133	static __always_inline long
1708	4134	atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1709	4135	{
1710	4136	instrument_atomic_read_write(v, sizeof(*v));
1711		− return arch_atomic_long_fetch_or_relaxed(i, v);
	4137	+ return raw_atomic_long_fetch_or_relaxed(i, v);
1712	4138	}
1713	4139
	4140	+/**
	4141	+ * atomic_long_xor() - atomic bitwise XOR with relaxed ordering
	4142	+ * @i: long value
	4143	+ * @v: pointer to atomic_long_t
	4144	+ *
	4145	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	4146	+ *
	4147	+ * Unsafe to use in noinstr code; use raw_atomic_long_xor() there.
	4148	+ *
	4149	+ * Return: Nothing.
	4150	+ */
1714	4151	static __always_inline void
1715	4152	atomic_long_xor(long i, atomic_long_t *v)
1716	4153	{
1717	4154	instrument_atomic_read_write(v, sizeof(*v));
1718		− arch_atomic_long_xor(i, v);
	4155	+ raw_atomic_long_xor(i, v);
1719	4156	}
1720	4157
	4158	+/**
	4159	+ * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
	4160	+ * @i: long value
	4161	+ * @v: pointer to atomic_long_t
	4162	+ *
	4163	+ * Atomically updates @v to (@v ^ @i) with full ordering.
	4164	+ *
	4165	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there.
	4166	+ *
	4167	+ * Return: The original value of @v.
	4168	+ */
1721	4169	static __always_inline long
1722	4170	atomic_long_fetch_xor(long i, atomic_long_t *v)
1723	4171	{
1724	4172	kcsan_mb();
1725	4173	instrument_atomic_read_write(v, sizeof(*v));
1726		− return arch_atomic_long_fetch_xor(i, v);
	4174	+ return raw_atomic_long_fetch_xor(i, v);
1727	4175	}
1728	4176
	4177	+/**
	4178	+ * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
	4179	+ * @i: long value
	4180	+ * @v: pointer to atomic_long_t
	4181	+ *
	4182	+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
	4183	+ *
	4184	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there.
	4185	+ *
	4186	+ * Return: The original value of @v.
	4187	+ */
1729	4188	static __always_inline long
1730	4189	atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1731	4190	{
1732	4191	instrument_atomic_read_write(v, sizeof(*v));
1733		− return arch_atomic_long_fetch_xor_acquire(i, v);
	4192	+ return raw_atomic_long_fetch_xor_acquire(i, v);
1734	4193	}
1735	4194
	4195	+/**
	4196	+ * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
	4197	+ * @i: long value
	4198	+ * @v: pointer to atomic_long_t
	4199	+ *
	4200	+ * Atomically updates @v to (@v ^ @i) with release ordering.
	4201	+ *
	4202	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there.
	4203	+ *
	4204	+ * Return: The original value of @v.
	4205	+ */
1736	4206	static __always_inline long
1737	4207	atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1738	4208	{
1739	4209	kcsan_release();
1740	4210	instrument_atomic_read_write(v, sizeof(*v));
1741		− return arch_atomic_long_fetch_xor_release(i, v);
	4211	+ return raw_atomic_long_fetch_xor_release(i, v);
1742	4212	}
1743	4213
	4214	+/**
	4215	+ * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
	4216	+ * @i: long value
	4217	+ * @v: pointer to atomic_long_t
	4218	+ *
	4219	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	4220	+ *
	4221	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there.
	4222	+ *
	4223	+ * Return: The original value of @v.
	4224	+ */
1744	4225	static __always_inline long
1745	4226	atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1746	4227	{
1747	4228	instrument_atomic_read_write(v, sizeof(*v));
1748		− return arch_atomic_long_fetch_xor_relaxed(i, v);
	4229	+ return raw_atomic_long_fetch_xor_relaxed(i, v);
1749	4230	}
1750	4231
	4232	+/**
	4233	+ * atomic_long_xchg() - atomic exchange with full ordering
	4234	+ * @v: pointer to atomic_long_t
	4235	+ * @new: long value to assign
	4236	+ *
	4237	+ * Atomically updates @v to @new with full ordering.
	4238	+ *
	4239	+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there.
	4240	+ *
	4241	+ * Return: The original value of @v.
	4242	+ */
1751	4243	static __always_inline long
1752		−atomic_long_xchg(atomic_long_t *v, long i)
	4244	+atomic_long_xchg(atomic_long_t *v, long new)
1753	4245	{
1754	4246	kcsan_mb();
1755	4247	instrument_atomic_read_write(v, sizeof(*v));
1756		− return arch_atomic_long_xchg(v, i);
	4248	+ return raw_atomic_long_xchg(v, new);
1757	4249	}
1758	4250
	4251	+/**
	4252	+ * atomic_long_xchg_acquire() - atomic exchange with acquire ordering
	4253	+ * @v: pointer to atomic_long_t
	4254	+ * @new: long value to assign
	4255	+ *
	4256	+ * Atomically updates @v to @new with acquire ordering.
	4257	+ *
	4258	+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there.
	4259	+ *
	4260	+ * Return: The original value of @v.
	4261	+ */
1759	4262	static __always_inline long
1760		−atomic_long_xchg_acquire(atomic_long_t *v, long i)
	4263	+atomic_long_xchg_acquire(atomic_long_t *v, long new)
1761	4264	{
1762	4265	instrument_atomic_read_write(v, sizeof(*v));
1763		− return arch_atomic_long_xchg_acquire(v, i);
	4266	+ return raw_atomic_long_xchg_acquire(v, new);
1764	4267	}
1765	4268
	4269	+/**
	4270	+ * atomic_long_xchg_release() - atomic exchange with release ordering
	4271	+ * @v: pointer to atomic_long_t
	4272	+ * @new: long value to assign
	4273	+ *
	4274	+ * Atomically updates @v to @new with release ordering.
	4275	+ *
	4276	+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there.
	4277	+ *
	4278	+ * Return: The original value of @v.
	4279	+ */
1766	4280	static __always_inline long
1767		−atomic_long_xchg_release(atomic_long_t *v, long i)
	4281	+atomic_long_xchg_release(atomic_long_t *v, long new)
1768	4282	{
1769	4283	kcsan_release();
1770	4284	instrument_atomic_read_write(v, sizeof(*v));
1771		− return arch_atomic_long_xchg_release(v, i);
	4285	+ return raw_atomic_long_xchg_release(v, new);
1772	4286	}
1773	4287
	4288	+/**
	4289	+ * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
	4290	+ * @v: pointer to atomic_long_t
	4291	+ * @new: long value to assign
	4292	+ *
	4293	+ * Atomically updates @v to @new with relaxed ordering.
	4294	+ *
	4295	+ * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there.
	4296	+ *
	4297	+ * Return: The original value of @v.
	4298	+ */
1774	4299	static __always_inline long
1775		−atomic_long_xchg_relaxed(atomic_long_t *v, long i)
	4300	+atomic_long_xchg_relaxed(atomic_long_t *v, long new)
1776	4301	{
1777	4302	instrument_atomic_read_write(v, sizeof(*v));
1778		− return arch_atomic_long_xchg_relaxed(v, i);
	4303	+ return raw_atomic_long_xchg_relaxed(v, new);
1779	4304	}
1780	4305
	4306	+/**
	4307	+ * atomic_long_cmpxchg() - atomic compare and exchange with full ordering
	4308	+ * @v: pointer to atomic_long_t
	4309	+ * @old: long value to compare with
	4310	+ * @new: long value to assign
	4311	+ *
	4312	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	4313	+ *
	4314	+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there.
	4315	+ *
	4316	+ * Return: The original value of @v.
	4317	+ */
1781	4318	static __always_inline long
1782	4319	atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1783	4320	{
1784	4321	kcsan_mb();
1785	4322	instrument_atomic_read_write(v, sizeof(*v));
1786		− return arch_atomic_long_cmpxchg(v, old, new);
	4323	+ return raw_atomic_long_cmpxchg(v, old, new);
1787	4324	}
1788	4325
	4326	+/**
	4327	+ * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	4328	+ * @v: pointer to atomic_long_t
	4329	+ * @old: long value to compare with
	4330	+ * @new: long value to assign
	4331	+ *
	4332	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	4333	+ *
	4334	+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there.
	4335	+ *
	4336	+ * Return: The original value of @v.
	4337	+ */
1789	4338	static __always_inline long
1790	4339	atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1791	4340	{
1792	4341	instrument_atomic_read_write(v, sizeof(*v));
1793		− return arch_atomic_long_cmpxchg_acquire(v, old, new);
	4342	+ return raw_atomic_long_cmpxchg_acquire(v, old, new);
1794	4343	}
1795	4344
	4345	+/**
	4346	+ * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
	4347	+ * @v: pointer to atomic_long_t
	4348	+ * @old: long value to compare with
	4349	+ * @new: long value to assign
	4350	+ *
	4351	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	4352	+ *
	4353	+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there.
	4354	+ *
	4355	+ * Return: The original value of @v.
	4356	+ */
1796	4357	static __always_inline long
1797	4358	atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1798	4359	{
1799	4360	kcsan_release();
1800	4361	instrument_atomic_read_write(v, sizeof(*v));
1801		− return arch_atomic_long_cmpxchg_release(v, old, new);
	4362	+ return raw_atomic_long_cmpxchg_release(v, old, new);
1802	4363	}
1803	4364
	4365	+/**
	4366	+ * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	4367	+ * @v: pointer to atomic_long_t
	4368	+ * @old: long value to compare with
	4369	+ * @new: long value to assign
	4370	+ *
	4371	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	4372	+ *
	4373	+ * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there.
	4374	+ *
	4375	+ * Return: The original value of @v.
	4376	+ */
1804	4377	static __always_inline long
1805	4378	atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1806	4379	{
1807	4380	instrument_atomic_read_write(v, sizeof(*v));
1808		− return arch_atomic_long_cmpxchg_relaxed(v, old, new);
	4381	+ return raw_atomic_long_cmpxchg_relaxed(v, old, new);
1809	4382	}
1810	4383
	4384	+/**
	4385	+ * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
	4386	+ * @v: pointer to atomic_long_t
	4387	+ * @old: pointer to long value to compare with
	4388	+ * @new: long value to assign
	4389	+ *
	4390	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	4391	+ * Otherwise, updates @old to the current value of @v.
	4392	+ *
	4393	+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there.
	4394	+ *
	4395	+ * Return: @true if the exchange occured, @false otherwise.
	4396	+ */
1811	4397	static __always_inline bool
1812	4398	atomic_long_try_cmpxchg(atomic_long_t v, long old, long new)
1813	4399	{
1814	4400	kcsan_mb();
1815	4401	instrument_atomic_read_write(v, sizeof(*v));
1816	4402	instrument_atomic_read_write(old, sizeof(*old));
1817		− return arch_atomic_long_try_cmpxchg(v, old, new);
1818		−}
1819		−
	4403	+ return raw_atomic_long_try_cmpxchg(v, old, new);
	4404	+}
	4405	+
	4406	+/**
	4407	+ * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	4408	+ * @v: pointer to atomic_long_t
	4409	+ * @old: pointer to long value to compare with
	4410	+ * @new: long value to assign
	4411	+ *
	4412	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	4413	+ * Otherwise, updates @old to the current value of @v.
	4414	+ *
	4415	+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there.
	4416	+ *
	4417	+ * Return: @true if the exchange occured, @false otherwise.
	4418	+ */
1820	4419	static __always_inline bool
1821	4420	atomic_long_try_cmpxchg_acquire(atomic_long_t v, long old, long new)
1822	4421	{
1823	4422	instrument_atomic_read_write(v, sizeof(*v));
1824	4423	instrument_atomic_read_write(old, sizeof(*old));
1825		− return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1826		−}
1827		−
	4424	+ return raw_atomic_long_try_cmpxchg_acquire(v, old, new);
	4425	+}
	4426	+
	4427	+/**
	4428	+ * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
	4429	+ * @v: pointer to atomic_long_t
	4430	+ * @old: pointer to long value to compare with
	4431	+ * @new: long value to assign
	4432	+ *
	4433	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	4434	+ * Otherwise, updates @old to the current value of @v.
	4435	+ *
	4436	+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there.
	4437	+ *
	4438	+ * Return: @true if the exchange occured, @false otherwise.
	4439	+ */
1828	4440	static __always_inline bool
1829	4441	atomic_long_try_cmpxchg_release(atomic_long_t v, long old, long new)
1830	4442	{
1831	4443	kcsan_release();
1832	4444	instrument_atomic_read_write(v, sizeof(*v));
1833	4445	instrument_atomic_read_write(old, sizeof(*old));
1834		− return arch_atomic_long_try_cmpxchg_release(v, old, new);
1835		−}
1836		−
	4446	+ return raw_atomic_long_try_cmpxchg_release(v, old, new);
	4447	+}
	4448	+
	4449	+/**
	4450	+ * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	4451	+ * @v: pointer to atomic_long_t
	4452	+ * @old: pointer to long value to compare with
	4453	+ * @new: long value to assign
	4454	+ *
	4455	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	4456	+ * Otherwise, updates @old to the current value of @v.
	4457	+ *
	4458	+ * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there.
	4459	+ *
	4460	+ * Return: @true if the exchange occured, @false otherwise.
	4461	+ */
1837	4462	static __always_inline bool
1838	4463	atomic_long_try_cmpxchg_relaxed(atomic_long_t v, long old, long new)
1839	4464	{
1840	4465	instrument_atomic_read_write(v, sizeof(*v));
1841	4466	instrument_atomic_read_write(old, sizeof(*old));
1842		− return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1843		−}
1844		−
	4467	+ return raw_atomic_long_try_cmpxchg_relaxed(v, old, new);
	4468	+}
	4469	+
	4470	+/**
	4471	+ * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
	4472	+ * @i: long value to add
	4473	+ * @v: pointer to atomic_long_t
	4474	+ *
	4475	+ * Atomically updates @v to (@v - @i) with full ordering.
	4476	+ *
	4477	+ * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there.
	4478	+ *
	4479	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	4480	+ */
1845	4481	static __always_inline bool
1846	4482	atomic_long_sub_and_test(long i, atomic_long_t *v)
1847	4483	{
1848	4484	kcsan_mb();
1849	4485	instrument_atomic_read_write(v, sizeof(*v));
1850		− return arch_atomic_long_sub_and_test(i, v);
	4486	+ return raw_atomic_long_sub_and_test(i, v);
1851	4487	}
1852	4488
	4489	+/**
	4490	+ * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
	4491	+ * @v: pointer to atomic_long_t
	4492	+ *
	4493	+ * Atomically updates @v to (@v - 1) with full ordering.
	4494	+ *
	4495	+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there.
	4496	+ *
	4497	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	4498	+ */
1853	4499	static __always_inline bool
1854	4500	atomic_long_dec_and_test(atomic_long_t *v)
1855	4501	{
1856	4502	kcsan_mb();
1857	4503	instrument_atomic_read_write(v, sizeof(*v));
1858		− return arch_atomic_long_dec_and_test(v);
	4504	+ return raw_atomic_long_dec_and_test(v);
1859	4505	}
1860	4506
	4507	+/**
	4508	+ * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
	4509	+ * @v: pointer to atomic_long_t
	4510	+ *
	4511	+ * Atomically updates @v to (@v + 1) with full ordering.
	4512	+ *
	4513	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there.
	4514	+ *
	4515	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	4516	+ */
1861	4517	static __always_inline bool
1862	4518	atomic_long_inc_and_test(atomic_long_t *v)
1863	4519	{
1864	4520	kcsan_mb();
1865	4521	instrument_atomic_read_write(v, sizeof(*v));
1866		− return arch_atomic_long_inc_and_test(v);
	4522	+ return raw_atomic_long_inc_and_test(v);
1867	4523	}
1868	4524
	4525	+/**
	4526	+ * atomic_long_add_negative() - atomic add and test if negative with full ordering
	4527	+ * @i: long value to add
	4528	+ * @v: pointer to atomic_long_t
	4529	+ *
	4530	+ * Atomically updates @v to (@v + @i) with full ordering.
	4531	+ *
	4532	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there.
	4533	+ *
	4534	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	4535	+ */
1869	4536	static __always_inline bool
1870	4537	atomic_long_add_negative(long i, atomic_long_t *v)
1871	4538	{
1872	4539	kcsan_mb();
1873	4540	instrument_atomic_read_write(v, sizeof(*v));
1874		− return arch_atomic_long_add_negative(i, v);
	4541	+ return raw_atomic_long_add_negative(i, v);
1875	4542	}
1876	4543
	4544	+/**
	4545	+ * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
	4546	+ * @i: long value to add
	4547	+ * @v: pointer to atomic_long_t
	4548	+ *
	4549	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	4550	+ *
	4551	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there.
	4552	+ *
	4553	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	4554	+ */
1877	4555	static __always_inline bool
1878	4556	atomic_long_add_negative_acquire(long i, atomic_long_t *v)
1879	4557	{
1880	4558	instrument_atomic_read_write(v, sizeof(*v));
1881		− return arch_atomic_long_add_negative_acquire(i, v);
	4559	+ return raw_atomic_long_add_negative_acquire(i, v);
1882	4560	}
1883	4561
	4562	+/**
	4563	+ * atomic_long_add_negative_release() - atomic add and test if negative with release ordering
	4564	+ * @i: long value to add
	4565	+ * @v: pointer to atomic_long_t
	4566	+ *
	4567	+ * Atomically updates @v to (@v + @i) with release ordering.
	4568	+ *
	4569	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there.
	4570	+ *
	4571	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	4572	+ */
1884	4573	static __always_inline bool
1885	4574	atomic_long_add_negative_release(long i, atomic_long_t *v)
1886	4575	{
1887	4576	kcsan_release();
1888	4577	instrument_atomic_read_write(v, sizeof(*v));
1889		− return arch_atomic_long_add_negative_release(i, v);
	4578	+ return raw_atomic_long_add_negative_release(i, v);
1890	4579	}
1891	4580
	4581	+/**
	4582	+ * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
	4583	+ * @i: long value to add
	4584	+ * @v: pointer to atomic_long_t
	4585	+ *
	4586	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	4587	+ *
	4588	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there.
	4589	+ *
	4590	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	4591	+ */
1892	4592	static __always_inline bool
1893	4593	atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1894	4594	{
1895	4595	instrument_atomic_read_write(v, sizeof(*v));
1896		− return arch_atomic_long_add_negative_relaxed(i, v);
	4596	+ return raw_atomic_long_add_negative_relaxed(i, v);
1897	4597	}
1898	4598
	4599	+/**
	4600	+ * atomic_long_fetch_add_unless() - atomic add unless value with full ordering
	4601	+ * @v: pointer to atomic_long_t
	4602	+ * @a: long value to add
	4603	+ * @u: long value to compare with
	4604	+ *
	4605	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	4606	+ *
	4607	+ * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there.
	4608	+ *
	4609	+ * Return: The original value of @v.
	4610	+ */
1899	4611	static __always_inline long
1900	4612	atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1901	4613	{
1902	4614	kcsan_mb();
1903	4615	instrument_atomic_read_write(v, sizeof(*v));
1904		− return arch_atomic_long_fetch_add_unless(v, a, u);
	4616	+ return raw_atomic_long_fetch_add_unless(v, a, u);
1905	4617	}
1906	4618
	4619	+/**
	4620	+ * atomic_long_add_unless() - atomic add unless value with full ordering
	4621	+ * @v: pointer to atomic_long_t
	4622	+ * @a: long value to add
	4623	+ * @u: long value to compare with
	4624	+ *
	4625	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	4626	+ *
	4627	+ * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there.
	4628	+ *
	4629	+ * Return: @true if @v was updated, @false otherwise.
	4630	+ */
1907	4631	static __always_inline bool
1908	4632	atomic_long_add_unless(atomic_long_t *v, long a, long u)
1909	4633	{
1910	4634	kcsan_mb();
1911	4635	instrument_atomic_read_write(v, sizeof(*v));
1912		− return arch_atomic_long_add_unless(v, a, u);
	4636	+ return raw_atomic_long_add_unless(v, a, u);
1913	4637	}
1914	4638
	4639	+/**
	4640	+ * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
	4641	+ * @v: pointer to atomic_long_t
	4642	+ *
	4643	+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	4644	+ *
	4645	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there.
	4646	+ *
	4647	+ * Return: @true if @v was updated, @false otherwise.
	4648	+ */
1915	4649	static __always_inline bool
1916	4650	atomic_long_inc_not_zero(atomic_long_t *v)
1917	4651	{
1918	4652	kcsan_mb();
1919	4653	instrument_atomic_read_write(v, sizeof(*v));
1920		− return arch_atomic_long_inc_not_zero(v);
	4654	+ return raw_atomic_long_inc_not_zero(v);
1921	4655	}
1922	4656
	4657	+/**
	4658	+ * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
	4659	+ * @v: pointer to atomic_long_t
	4660	+ *
	4661	+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	4662	+ *
	4663	+ * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there.
	4664	+ *
	4665	+ * Return: @true if @v was updated, @false otherwise.
	4666	+ */
1923	4667	static __always_inline bool
1924	4668	atomic_long_inc_unless_negative(atomic_long_t *v)
1925	4669	{
1926	4670	kcsan_mb();
1927	4671	instrument_atomic_read_write(v, sizeof(*v));
1928		− return arch_atomic_long_inc_unless_negative(v);
	4672	+ return raw_atomic_long_inc_unless_negative(v);
1929	4673	}
1930	4674
	4675	+/**
	4676	+ * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
	4677	+ * @v: pointer to atomic_long_t
	4678	+ *
	4679	+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	4680	+ *
	4681	+ * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there.

347 more lines aren’t shown here.

include/linux/atomic/atomic-long.h+1435−687

@@ -21,1030 +21,1778 @@ typedef atomic_t atomic_long_t; · 20 unmodified lines
21	21	#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
22	22	#endif
23	23
24		−#ifdef CONFIG_64BIT
25		−
26		−static __always_inline long
27		−arch_atomic_long_read(const atomic_long_t *v)
28		−{
29		− return arch_atomic64_read(v);
30		−}
31		−
32		−static __always_inline long
33		−arch_atomic_long_read_acquire(const atomic_long_t *v)
34		−{
35		− return arch_atomic64_read_acquire(v);
36		−}
37		−
38		−static __always_inline void
39		−arch_atomic_long_set(atomic_long_t *v, long i)
40		−{
41		− arch_atomic64_set(v, i);
42		−}
43		−
44		−static __always_inline void
45		−arch_atomic_long_set_release(atomic_long_t *v, long i)
46		−{
47		− arch_atomic64_set_release(v, i);
48		−}
49		−
50		−static __always_inline void
51		−arch_atomic_long_add(long i, atomic_long_t *v)
52		−{
53		− arch_atomic64_add(i, v);
54		−}
55		−
56		−static __always_inline long
57		−arch_atomic_long_add_return(long i, atomic_long_t *v)
58		−{
59		− return arch_atomic64_add_return(i, v);
60		−}
61		−
62		−static __always_inline long
63		−arch_atomic_long_add_return_acquire(long i, atomic_long_t *v)
64		−{
65		− return arch_atomic64_add_return_acquire(i, v);
66		−}
67		−
68		−static __always_inline long
69		−arch_atomic_long_add_return_release(long i, atomic_long_t *v)
70		−{
71		− return arch_atomic64_add_return_release(i, v);
72		−}
73		−
74		−static __always_inline long
75		−arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
76		−{
77		− return arch_atomic64_add_return_relaxed(i, v);
78		−}
79		−
80		−static __always_inline long
81		−arch_atomic_long_fetch_add(long i, atomic_long_t *v)
82		−{
83		− return arch_atomic64_fetch_add(i, v);
84		−}
85		−
86		−static __always_inline long
87		−arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
88		−{
89		− return arch_atomic64_fetch_add_acquire(i, v);
90		−}
91		−
92		−static __always_inline long
93		−arch_atomic_long_fetch_add_release(long i, atomic_long_t *v)
94		−{
95		− return arch_atomic64_fetch_add_release(i, v);
96		−}
97		−
98		−static __always_inline long
99		−arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
100		−{
101		− return arch_atomic64_fetch_add_relaxed(i, v);
102		−}
103		−
104		−static __always_inline void
105		−arch_atomic_long_sub(long i, atomic_long_t *v)
106		−{
107		− arch_atomic64_sub(i, v);
108		−}
109		−
110		−static __always_inline long
111		−arch_atomic_long_sub_return(long i, atomic_long_t *v)
112		−{
113		− return arch_atomic64_sub_return(i, v);
114		−}
115		−
116		−static __always_inline long
117		−arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
118		−{
119		− return arch_atomic64_sub_return_acquire(i, v);
120		−}
121		−
122		−static __always_inline long
123		−arch_atomic_long_sub_return_release(long i, atomic_long_t *v)
124		−{
125		− return arch_atomic64_sub_return_release(i, v);
126		−}
127		−
128		−static __always_inline long
129		−arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
130		−{
131		− return arch_atomic64_sub_return_relaxed(i, v);
132		−}
133		−
134		−static __always_inline long
135		−arch_atomic_long_fetch_sub(long i, atomic_long_t *v)
136		−{
137		− return arch_atomic64_fetch_sub(i, v);
138		−}
139		−
140		−static __always_inline long
141		−arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
142		−{
143		− return arch_atomic64_fetch_sub_acquire(i, v);
144		−}
145		−
146		−static __always_inline long
147		−arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
148		−{
149		− return arch_atomic64_fetch_sub_release(i, v);
150		−}
151		−
152		−static __always_inline long
153		−arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
154		−{
155		− return arch_atomic64_fetch_sub_relaxed(i, v);
156		−}
157		−
158		−static __always_inline void
159		−arch_atomic_long_inc(atomic_long_t *v)
160		−{
161		− arch_atomic64_inc(v);
162		−}
163		−
164		−static __always_inline long
165		−arch_atomic_long_inc_return(atomic_long_t *v)
166		−{
167		− return arch_atomic64_inc_return(v);
168		−}
169		−
170		−static __always_inline long
171		−arch_atomic_long_inc_return_acquire(atomic_long_t *v)
172		−{
173		− return arch_atomic64_inc_return_acquire(v);
174		−}
175		−
176		−static __always_inline long
177		−arch_atomic_long_inc_return_release(atomic_long_t *v)
178		−{
179		− return arch_atomic64_inc_return_release(v);
180		−}
181		−
182		−static __always_inline long
183		−arch_atomic_long_inc_return_relaxed(atomic_long_t *v)
184		−{
185		− return arch_atomic64_inc_return_relaxed(v);
186		−}
187		−
188		−static __always_inline long
189		−arch_atomic_long_fetch_inc(atomic_long_t *v)
190		−{
191		− return arch_atomic64_fetch_inc(v);
192		−}
193		−
194		−static __always_inline long
195		−arch_atomic_long_fetch_inc_acquire(atomic_long_t *v)
196		−{
197		− return arch_atomic64_fetch_inc_acquire(v);
198		−}
199		−
200		−static __always_inline long
201		−arch_atomic_long_fetch_inc_release(atomic_long_t *v)
202		−{
203		− return arch_atomic64_fetch_inc_release(v);
204		−}
205		−
206		−static __always_inline long
207		−arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
208		−{
209		− return arch_atomic64_fetch_inc_relaxed(v);
210		−}
211		−
212		−static __always_inline void
213		−arch_atomic_long_dec(atomic_long_t *v)
214		−{
215		− arch_atomic64_dec(v);
216		−}
217		−
218		−static __always_inline long
219		−arch_atomic_long_dec_return(atomic_long_t *v)
220		−{
221		− return arch_atomic64_dec_return(v);
222		−}
223		−
224		−static __always_inline long
225		−arch_atomic_long_dec_return_acquire(atomic_long_t *v)
226		−{
227		− return arch_atomic64_dec_return_acquire(v);
228		−}
229		−
230		−static __always_inline long
231		−arch_atomic_long_dec_return_release(atomic_long_t *v)
232		−{
233		− return arch_atomic64_dec_return_release(v);
234		−}
235		−
236		−static __always_inline long
237		−arch_atomic_long_dec_return_relaxed(atomic_long_t *v)
238		−{
239		− return arch_atomic64_dec_return_relaxed(v);
240		−}
241		−
242		−static __always_inline long
243		−arch_atomic_long_fetch_dec(atomic_long_t *v)
244		−{
245		− return arch_atomic64_fetch_dec(v);
246		−}
247		−
248		−static __always_inline long
249		−arch_atomic_long_fetch_dec_acquire(atomic_long_t *v)
250		−{
251		− return arch_atomic64_fetch_dec_acquire(v);
252		−}
253		−
254		−static __always_inline long
255		−arch_atomic_long_fetch_dec_release(atomic_long_t *v)
256		−{
257		− return arch_atomic64_fetch_dec_release(v);
258		−}
259		−
260		−static __always_inline long
261		−arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
262		−{
263		− return arch_atomic64_fetch_dec_relaxed(v);
264		−}
265		−
266		−static __always_inline void
267		−arch_atomic_long_and(long i, atomic_long_t *v)
268		−{
269		− arch_atomic64_and(i, v);
270		−}
271		−
272		−static __always_inline long
273		−arch_atomic_long_fetch_and(long i, atomic_long_t *v)
274		−{
275		− return arch_atomic64_fetch_and(i, v);
276		−}
277		−
278		−static __always_inline long
279		−arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
280		−{
281		− return arch_atomic64_fetch_and_acquire(i, v);
282		−}
283		−
284		−static __always_inline long
285		−arch_atomic_long_fetch_and_release(long i, atomic_long_t *v)
286		−{
287		− return arch_atomic64_fetch_and_release(i, v);
288		−}
289		−
290		−static __always_inline long
291		−arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
292		−{
293		− return arch_atomic64_fetch_and_relaxed(i, v);
294		−}
295		−
296		−static __always_inline void
297		−arch_atomic_long_andnot(long i, atomic_long_t *v)
298		−{
299		− arch_atomic64_andnot(i, v);
300		−}
301		−
302		−static __always_inline long
303		−arch_atomic_long_fetch_andnot(long i, atomic_long_t *v)
304		−{
305		− return arch_atomic64_fetch_andnot(i, v);
306		−}
307		−
308		−static __always_inline long
309		−arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
310		−{
311		− return arch_atomic64_fetch_andnot_acquire(i, v);
312		−}
313		−
314		−static __always_inline long
315		−arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
316		−{
317		− return arch_atomic64_fetch_andnot_release(i, v);
318		−}
319		−
320		−static __always_inline long
321		−arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
322		−{
323		− return arch_atomic64_fetch_andnot_relaxed(i, v);
324		−}
325		−
326		−static __always_inline void
327		−arch_atomic_long_or(long i, atomic_long_t *v)
328		−{
329		− arch_atomic64_or(i, v);
330		−}
331		−
332		−static __always_inline long
333		−arch_atomic_long_fetch_or(long i, atomic_long_t *v)
334		−{
335		− return arch_atomic64_fetch_or(i, v);
336		−}
337		−
338		−static __always_inline long
339		−arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
340		−{
341		− return arch_atomic64_fetch_or_acquire(i, v);
342		−}
343		−
344		−static __always_inline long
345		−arch_atomic_long_fetch_or_release(long i, atomic_long_t *v)
346		−{
347		− return arch_atomic64_fetch_or_release(i, v);
348		−}
349		−
350		−static __always_inline long
351		−arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
352		−{
353		− return arch_atomic64_fetch_or_relaxed(i, v);
354		−}
355		−
356		−static __always_inline void
357		−arch_atomic_long_xor(long i, atomic_long_t *v)
358		−{
359		− arch_atomic64_xor(i, v);
360		−}
361		−
362		−static __always_inline long
363		−arch_atomic_long_fetch_xor(long i, atomic_long_t *v)
364		−{
365		− return arch_atomic64_fetch_xor(i, v);
366		−}
367		−
368		−static __always_inline long
369		−arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
370		−{
371		− return arch_atomic64_fetch_xor_acquire(i, v);
372		−}
373		−
374		−static __always_inline long
375		−arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
376		−{
377		− return arch_atomic64_fetch_xor_release(i, v);
378		−}
379		−
380		−static __always_inline long
381		−arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
382		−{
383		− return arch_atomic64_fetch_xor_relaxed(i, v);
384		−}
385		−
386		−static __always_inline long
387		−arch_atomic_long_xchg(atomic_long_t *v, long i)
388		−{
389		− return arch_atomic64_xchg(v, i);
390		−}
391		−
392		−static __always_inline long
393		−arch_atomic_long_xchg_acquire(atomic_long_t *v, long i)
394		−{
395		− return arch_atomic64_xchg_acquire(v, i);
396		−}
397		−
398		−static __always_inline long
399		−arch_atomic_long_xchg_release(atomic_long_t *v, long i)
400		−{
401		− return arch_atomic64_xchg_release(v, i);
402		−}
403		−
404		−static __always_inline long
405		−arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i)
406		−{
407		− return arch_atomic64_xchg_relaxed(v, i);
408		−}
409		−
410		−static __always_inline long
411		−arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
412		−{
413		− return arch_atomic64_cmpxchg(v, old, new);
414		−}
415		−
416		−static __always_inline long
417		−arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
418		−{
419		− return arch_atomic64_cmpxchg_acquire(v, old, new);
420		−}
421		−
422		−static __always_inline long
423		−arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
424		−{
425		− return arch_atomic64_cmpxchg_release(v, old, new);
426		−}
427		−
428		−static __always_inline long
429		−arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
430		−{
431		− return arch_atomic64_cmpxchg_relaxed(v, old, new);
432		−}
433		−
434		−static __always_inline bool
435		−arch_atomic_long_try_cmpxchg(atomic_long_t v, long old, long new)
436		−{
437		− return arch_atomic64_try_cmpxchg(v, (s64 *)old, new);
438		−}
439		−
440		−static __always_inline bool
441		−arch_atomic_long_try_cmpxchg_acquire(atomic_long_t v, long old, long new)
442		−{
443		− return arch_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
444		−}
445		−
446		−static __always_inline bool
447		−arch_atomic_long_try_cmpxchg_release(atomic_long_t v, long old, long new)
448		−{
449		− return arch_atomic64_try_cmpxchg_release(v, (s64 *)old, new);
450		−}
451		−
452		−static __always_inline bool
453		−arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t v, long old, long new)
454		−{
455		− return arch_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
456		−}
457		−
458		−static __always_inline bool
459		−arch_atomic_long_sub_and_test(long i, atomic_long_t *v)
460		−{
461		− return arch_atomic64_sub_and_test(i, v);
462		−}
463		−
464		−static __always_inline bool
465		−arch_atomic_long_dec_and_test(atomic_long_t *v)
466		−{
467		− return arch_atomic64_dec_and_test(v);
468		−}
469		−
470		−static __always_inline bool
471		−arch_atomic_long_inc_and_test(atomic_long_t *v)
472		−{
473		− return arch_atomic64_inc_and_test(v);
474		−}
475		−
476		−static __always_inline bool
477		−arch_atomic_long_add_negative(long i, atomic_long_t *v)
478		−{
479		− return arch_atomic64_add_negative(i, v);
480		−}
481		−
482		−static __always_inline bool
483		−arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
484		−{
485		− return arch_atomic64_add_negative_acquire(i, v);
486		−}
487		−
488		−static __always_inline bool
489		−arch_atomic_long_add_negative_release(long i, atomic_long_t *v)
490		−{
491		− return arch_atomic64_add_negative_release(i, v);
492		−}
493		−
494		−static __always_inline bool
495		−arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
496		−{
497		− return arch_atomic64_add_negative_relaxed(i, v);
498		−}
499		−
500		−static __always_inline long
501		−arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
502		−{
503		− return arch_atomic64_fetch_add_unless(v, a, u);
504		−}
505		−
506		−static __always_inline bool
507		−arch_atomic_long_add_unless(atomic_long_t *v, long a, long u)
508		−{
509		− return arch_atomic64_add_unless(v, a, u);
510		−}
511		−
512		−static __always_inline bool
513		−arch_atomic_long_inc_not_zero(atomic_long_t *v)
514		−{
515		− return arch_atomic64_inc_not_zero(v);
516		−}
517		−
518		−static __always_inline bool
519		−arch_atomic_long_inc_unless_negative(atomic_long_t *v)
520		−{
521		− return arch_atomic64_inc_unless_negative(v);
522		−}
523		−
524		−static __always_inline bool
525		−arch_atomic_long_dec_unless_positive(atomic_long_t *v)
526		−{
527		− return arch_atomic64_dec_unless_positive(v);
528		−}
529		−
530		−static __always_inline long
531		−arch_atomic_long_dec_if_positive(atomic_long_t *v)
532		−{
533		− return arch_atomic64_dec_if_positive(v);
534		−}
535		−
536		−#else /* CONFIG_64BIT */
537		−
538		−static __always_inline long
539		−arch_atomic_long_read(const atomic_long_t *v)
	24	+/**
	25	+ * raw_atomic_long_read() - atomic load with relaxed ordering
	26	+ * @v: pointer to atomic_long_t
	27	+ *
	28	+ * Atomically loads the value of @v with relaxed ordering.
	29	+ *
	30	+ * Safe to use in noinstr code; prefer atomic_long_read() elsewhere.
	31	+ *
	32	+ * Return: The value loaded from @v.
	33	+ */
	34	+static __always_inline long
	35	+raw_atomic_long_read(const atomic_long_t *v)
540	36	{
541		− return arch_atomic_read(v);
	37	+#ifdef CONFIG_64BIT
	38	+ return raw_atomic64_read(v);
	39	+#else
	40	+ return raw_atomic_read(v);
	41	+#endif
542	42	}
543	43
	44	+/**
	45	+ * raw_atomic_long_read_acquire() - atomic load with acquire ordering
	46	+ * @v: pointer to atomic_long_t
	47	+ *
	48	+ * Atomically loads the value of @v with acquire ordering.
	49	+ *
	50	+ * Safe to use in noinstr code; prefer atomic_long_read_acquire() elsewhere.
	51	+ *
	52	+ * Return: The value loaded from @v.
	53	+ */
544	54	static __always_inline long
545		−arch_atomic_long_read_acquire(const atomic_long_t *v)
	55	+raw_atomic_long_read_acquire(const atomic_long_t *v)
546	56	{
547		− return arch_atomic_read_acquire(v);
	57	+#ifdef CONFIG_64BIT
	58	+ return raw_atomic64_read_acquire(v);
	59	+#else
	60	+ return raw_atomic_read_acquire(v);
	61	+#endif
548	62	}
549	63
	64	+/**
	65	+ * raw_atomic_long_set() - atomic set with relaxed ordering
	66	+ * @v: pointer to atomic_long_t
	67	+ * @i: long value to assign
	68	+ *
	69	+ * Atomically sets @v to @i with relaxed ordering.
	70	+ *
	71	+ * Safe to use in noinstr code; prefer atomic_long_set() elsewhere.
	72	+ *
	73	+ * Return: Nothing.
	74	+ */
550	75	static __always_inline void
551		−arch_atomic_long_set(atomic_long_t *v, long i)
	76	+raw_atomic_long_set(atomic_long_t *v, long i)
552	77	{
553		− arch_atomic_set(v, i);
	78	+#ifdef CONFIG_64BIT
	79	+ raw_atomic64_set(v, i);
	80	+#else
	81	+ raw_atomic_set(v, i);
	82	+#endif
554	83	}
555	84
	85	+/**
	86	+ * raw_atomic_long_set_release() - atomic set with release ordering
	87	+ * @v: pointer to atomic_long_t
	88	+ * @i: long value to assign
	89	+ *
	90	+ * Atomically sets @v to @i with release ordering.
	91	+ *
	92	+ * Safe to use in noinstr code; prefer atomic_long_set_release() elsewhere.
	93	+ *
	94	+ * Return: Nothing.
	95	+ */
556	96	static __always_inline void
557		−arch_atomic_long_set_release(atomic_long_t *v, long i)
	97	+raw_atomic_long_set_release(atomic_long_t *v, long i)
558	98	{
559		− arch_atomic_set_release(v, i);
	99	+#ifdef CONFIG_64BIT
	100	+ raw_atomic64_set_release(v, i);
	101	+#else
	102	+ raw_atomic_set_release(v, i);
	103	+#endif
560	104	}
561	105
	106	+/**
	107	+ * raw_atomic_long_add() - atomic add with relaxed ordering
	108	+ * @i: long value to add
	109	+ * @v: pointer to atomic_long_t
	110	+ *
	111	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	112	+ *
	113	+ * Safe to use in noinstr code; prefer atomic_long_add() elsewhere.
	114	+ *
	115	+ * Return: Nothing.
	116	+ */
562	117	static __always_inline void
563		−arch_atomic_long_add(long i, atomic_long_t *v)
	118	+raw_atomic_long_add(long i, atomic_long_t *v)
564	119	{
565		− arch_atomic_add(i, v);
	120	+#ifdef CONFIG_64BIT
	121	+ raw_atomic64_add(i, v);
	122	+#else
	123	+ raw_atomic_add(i, v);
	124	+#endif
566	125	}
567	126
	127	+/**
	128	+ * raw_atomic_long_add_return() - atomic add with full ordering
	129	+ * @i: long value to add
	130	+ * @v: pointer to atomic_long_t
	131	+ *
	132	+ * Atomically updates @v to (@v + @i) with full ordering.
	133	+ *
	134	+ * Safe to use in noinstr code; prefer atomic_long_add_return() elsewhere.
	135	+ *
	136	+ * Return: The updated value of @v.
	137	+ */
568	138	static __always_inline long
569		−arch_atomic_long_add_return(long i, atomic_long_t *v)
	139	+raw_atomic_long_add_return(long i, atomic_long_t *v)
570	140	{
571		− return arch_atomic_add_return(i, v);
	141	+#ifdef CONFIG_64BIT
	142	+ return raw_atomic64_add_return(i, v);
	143	+#else
	144	+ return raw_atomic_add_return(i, v);
	145	+#endif
572	146	}
573	147
	148	+/**
	149	+ * raw_atomic_long_add_return_acquire() - atomic add with acquire ordering
	150	+ * @i: long value to add
	151	+ * @v: pointer to atomic_long_t
	152	+ *
	153	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	154	+ *
	155	+ * Safe to use in noinstr code; prefer atomic_long_add_return_acquire() elsewhere.
	156	+ *
	157	+ * Return: The updated value of @v.
	158	+ */
574	159	static __always_inline long
575		−arch_atomic_long_add_return_acquire(long i, atomic_long_t *v)
	160	+raw_atomic_long_add_return_acquire(long i, atomic_long_t *v)
576	161	{
577		− return arch_atomic_add_return_acquire(i, v);
	162	+#ifdef CONFIG_64BIT
	163	+ return raw_atomic64_add_return_acquire(i, v);
	164	+#else
	165	+ return raw_atomic_add_return_acquire(i, v);
	166	+#endif
578	167	}
579	168
	169	+/**
	170	+ * raw_atomic_long_add_return_release() - atomic add with release ordering
	171	+ * @i: long value to add
	172	+ * @v: pointer to atomic_long_t
	173	+ *
	174	+ * Atomically updates @v to (@v + @i) with release ordering.
	175	+ *
	176	+ * Safe to use in noinstr code; prefer atomic_long_add_return_release() elsewhere.
	177	+ *
	178	+ * Return: The updated value of @v.
	179	+ */
580	180	static __always_inline long
581		−arch_atomic_long_add_return_release(long i, atomic_long_t *v)
	181	+raw_atomic_long_add_return_release(long i, atomic_long_t *v)
582	182	{
583		− return arch_atomic_add_return_release(i, v);
	183	+#ifdef CONFIG_64BIT
	184	+ return raw_atomic64_add_return_release(i, v);
	185	+#else
	186	+ return raw_atomic_add_return_release(i, v);
	187	+#endif
584	188	}
585	189
	190	+/**
	191	+ * raw_atomic_long_add_return_relaxed() - atomic add with relaxed ordering
	192	+ * @i: long value to add
	193	+ * @v: pointer to atomic_long_t
	194	+ *
	195	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	196	+ *
	197	+ * Safe to use in noinstr code; prefer atomic_long_add_return_relaxed() elsewhere.
	198	+ *
	199	+ * Return: The updated value of @v.
	200	+ */
586	201	static __always_inline long
587		−arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
	202	+raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v)
588	203	{
589		− return arch_atomic_add_return_relaxed(i, v);
	204	+#ifdef CONFIG_64BIT
	205	+ return raw_atomic64_add_return_relaxed(i, v);
	206	+#else
	207	+ return raw_atomic_add_return_relaxed(i, v);
	208	+#endif
590	209	}
591	210
	211	+/**
	212	+ * raw_atomic_long_fetch_add() - atomic add with full ordering
	213	+ * @i: long value to add
	214	+ * @v: pointer to atomic_long_t
	215	+ *
	216	+ * Atomically updates @v to (@v + @i) with full ordering.
	217	+ *
	218	+ * Safe to use in noinstr code; prefer atomic_long_fetch_add() elsewhere.
	219	+ *
	220	+ * Return: The original value of @v.
	221	+ */
592	222	static __always_inline long
593		−arch_atomic_long_fetch_add(long i, atomic_long_t *v)
	223	+raw_atomic_long_fetch_add(long i, atomic_long_t *v)
594	224	{
595		− return arch_atomic_fetch_add(i, v);
	225	+#ifdef CONFIG_64BIT
	226	+ return raw_atomic64_fetch_add(i, v);
	227	+#else
	228	+ return raw_atomic_fetch_add(i, v);
	229	+#endif
596	230	}
597	231
	232	+/**
	233	+ * raw_atomic_long_fetch_add_acquire() - atomic add with acquire ordering
	234	+ * @i: long value to add
	235	+ * @v: pointer to atomic_long_t
	236	+ *
	237	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	238	+ *
	239	+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_acquire() elsewhere.
	240	+ *
	241	+ * Return: The original value of @v.
	242	+ */
598	243	static __always_inline long
599		−arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
	244	+raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
600	245	{
601		− return arch_atomic_fetch_add_acquire(i, v);
	246	+#ifdef CONFIG_64BIT
	247	+ return raw_atomic64_fetch_add_acquire(i, v);
	248	+#else
	249	+ return raw_atomic_fetch_add_acquire(i, v);
	250	+#endif
602	251	}
603	252
	253	+/**
	254	+ * raw_atomic_long_fetch_add_release() - atomic add with release ordering
	255	+ * @i: long value to add
	256	+ * @v: pointer to atomic_long_t
	257	+ *
	258	+ * Atomically updates @v to (@v + @i) with release ordering.
	259	+ *
	260	+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_release() elsewhere.
	261	+ *
	262	+ * Return: The original value of @v.
	263	+ */
604	264	static __always_inline long
605		−arch_atomic_long_fetch_add_release(long i, atomic_long_t *v)
	265	+raw_atomic_long_fetch_add_release(long i, atomic_long_t *v)
606	266	{
607		− return arch_atomic_fetch_add_release(i, v);
	267	+#ifdef CONFIG_64BIT
	268	+ return raw_atomic64_fetch_add_release(i, v);
	269	+#else
	270	+ return raw_atomic_fetch_add_release(i, v);
	271	+#endif
608	272	}
609	273
	274	+/**
	275	+ * raw_atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering
	276	+ * @i: long value to add
	277	+ * @v: pointer to atomic_long_t
	278	+ *
	279	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	280	+ *
	281	+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_relaxed() elsewhere.
	282	+ *
	283	+ * Return: The original value of @v.
	284	+ */
610	285	static __always_inline long
611		−arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
	286	+raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
612	287	{
613		− return arch_atomic_fetch_add_relaxed(i, v);
	288	+#ifdef CONFIG_64BIT
	289	+ return raw_atomic64_fetch_add_relaxed(i, v);
	290	+#else
	291	+ return raw_atomic_fetch_add_relaxed(i, v);
	292	+#endif
614	293	}
615	294
	295	+/**
	296	+ * raw_atomic_long_sub() - atomic subtract with relaxed ordering
	297	+ * @i: long value to subtract
	298	+ * @v: pointer to atomic_long_t
	299	+ *
	300	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	301	+ *
	302	+ * Safe to use in noinstr code; prefer atomic_long_sub() elsewhere.
	303	+ *
	304	+ * Return: Nothing.
	305	+ */
616	306	static __always_inline void
617		−arch_atomic_long_sub(long i, atomic_long_t *v)
	307	+raw_atomic_long_sub(long i, atomic_long_t *v)
618	308	{
619		− arch_atomic_sub(i, v);
	309	+#ifdef CONFIG_64BIT
	310	+ raw_atomic64_sub(i, v);
	311	+#else
	312	+ raw_atomic_sub(i, v);
	313	+#endif
620	314	}
621	315
	316	+/**
	317	+ * raw_atomic_long_sub_return() - atomic subtract with full ordering
	318	+ * @i: long value to subtract
	319	+ * @v: pointer to atomic_long_t
	320	+ *
	321	+ * Atomically updates @v to (@v - @i) with full ordering.
	322	+ *
	323	+ * Safe to use in noinstr code; prefer atomic_long_sub_return() elsewhere.
	324	+ *
	325	+ * Return: The updated value of @v.
	326	+ */
622	327	static __always_inline long
623		−arch_atomic_long_sub_return(long i, atomic_long_t *v)
	328	+raw_atomic_long_sub_return(long i, atomic_long_t *v)
624	329	{
625		− return arch_atomic_sub_return(i, v);
	330	+#ifdef CONFIG_64BIT
	331	+ return raw_atomic64_sub_return(i, v);
	332	+#else
	333	+ return raw_atomic_sub_return(i, v);
	334	+#endif
626	335	}
627	336
	337	+/**
	338	+ * raw_atomic_long_sub_return_acquire() - atomic subtract with acquire ordering
	339	+ * @i: long value to subtract
	340	+ * @v: pointer to atomic_long_t
	341	+ *
	342	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	343	+ *
	344	+ * Safe to use in noinstr code; prefer atomic_long_sub_return_acquire() elsewhere.
	345	+ *
	346	+ * Return: The updated value of @v.
	347	+ */
628	348	static __always_inline long
629		−arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
	349	+raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v)
630	350	{
631		− return arch_atomic_sub_return_acquire(i, v);
	351	+#ifdef CONFIG_64BIT
	352	+ return raw_atomic64_sub_return_acquire(i, v);
	353	+#else
	354	+ return raw_atomic_sub_return_acquire(i, v);
	355	+#endif
632	356	}
633	357
	358	+/**
	359	+ * raw_atomic_long_sub_return_release() - atomic subtract with release ordering
	360	+ * @i: long value to subtract
	361	+ * @v: pointer to atomic_long_t
	362	+ *
	363	+ * Atomically updates @v to (@v - @i) with release ordering.
	364	+ *
	365	+ * Safe to use in noinstr code; prefer atomic_long_sub_return_release() elsewhere.
	366	+ *
	367	+ * Return: The updated value of @v.
	368	+ */
634	369	static __always_inline long
635		−arch_atomic_long_sub_return_release(long i, atomic_long_t *v)
	370	+raw_atomic_long_sub_return_release(long i, atomic_long_t *v)
636	371	{
637		− return arch_atomic_sub_return_release(i, v);
	372	+#ifdef CONFIG_64BIT
	373	+ return raw_atomic64_sub_return_release(i, v);
	374	+#else
	375	+ return raw_atomic_sub_return_release(i, v);
	376	+#endif
638	377	}
639	378
	379	+/**
	380	+ * raw_atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering
	381	+ * @i: long value to subtract
	382	+ * @v: pointer to atomic_long_t
	383	+ *
	384	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	385	+ *
	386	+ * Safe to use in noinstr code; prefer atomic_long_sub_return_relaxed() elsewhere.
	387	+ *
	388	+ * Return: The updated value of @v.
	389	+ */
640	390	static __always_inline long
641		−arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
	391	+raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
642	392	{
643		− return arch_atomic_sub_return_relaxed(i, v);
	393	+#ifdef CONFIG_64BIT
	394	+ return raw_atomic64_sub_return_relaxed(i, v);
	395	+#else
	396	+ return raw_atomic_sub_return_relaxed(i, v);
	397	+#endif
644	398	}
645	399
	400	+/**
	401	+ * raw_atomic_long_fetch_sub() - atomic subtract with full ordering
	402	+ * @i: long value to subtract
	403	+ * @v: pointer to atomic_long_t
	404	+ *
	405	+ * Atomically updates @v to (@v - @i) with full ordering.
	406	+ *
	407	+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub() elsewhere.
	408	+ *
	409	+ * Return: The original value of @v.
	410	+ */
646	411	static __always_inline long
647		−arch_atomic_long_fetch_sub(long i, atomic_long_t *v)
	412	+raw_atomic_long_fetch_sub(long i, atomic_long_t *v)
648	413	{
649		− return arch_atomic_fetch_sub(i, v);
	414	+#ifdef CONFIG_64BIT
	415	+ return raw_atomic64_fetch_sub(i, v);
	416	+#else
	417	+ return raw_atomic_fetch_sub(i, v);
	418	+#endif
650	419	}
651	420
	421	+/**
	422	+ * raw_atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering
	423	+ * @i: long value to subtract
	424	+ * @v: pointer to atomic_long_t
	425	+ *
	426	+ * Atomically updates @v to (@v - @i) with acquire ordering.
	427	+ *
	428	+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub_acquire() elsewhere.
	429	+ *
	430	+ * Return: The original value of @v.
	431	+ */
652	432	static __always_inline long
653		−arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
	433	+raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
654	434	{
655		− return arch_atomic_fetch_sub_acquire(i, v);
	435	+#ifdef CONFIG_64BIT
	436	+ return raw_atomic64_fetch_sub_acquire(i, v);
	437	+#else
	438	+ return raw_atomic_fetch_sub_acquire(i, v);
	439	+#endif
656	440	}
657	441
	442	+/**
	443	+ * raw_atomic_long_fetch_sub_release() - atomic subtract with release ordering
	444	+ * @i: long value to subtract
	445	+ * @v: pointer to atomic_long_t
	446	+ *
	447	+ * Atomically updates @v to (@v - @i) with release ordering.
	448	+ *
	449	+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub_release() elsewhere.
	450	+ *
	451	+ * Return: The original value of @v.
	452	+ */
658	453	static __always_inline long
659		−arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
	454	+raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v)
660	455	{
661		− return arch_atomic_fetch_sub_release(i, v);
	456	+#ifdef CONFIG_64BIT
	457	+ return raw_atomic64_fetch_sub_release(i, v);
	458	+#else
	459	+ return raw_atomic_fetch_sub_release(i, v);
	460	+#endif
662	461	}
663	462
	463	+/**
	464	+ * raw_atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering
	465	+ * @i: long value to subtract
	466	+ * @v: pointer to atomic_long_t
	467	+ *
	468	+ * Atomically updates @v to (@v - @i) with relaxed ordering.
	469	+ *
	470	+ * Safe to use in noinstr code; prefer atomic_long_fetch_sub_relaxed() elsewhere.
	471	+ *
	472	+ * Return: The original value of @v.
	473	+ */
664	474	static __always_inline long
665		−arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
	475	+raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
666	476	{
667		− return arch_atomic_fetch_sub_relaxed(i, v);
	477	+#ifdef CONFIG_64BIT
	478	+ return raw_atomic64_fetch_sub_relaxed(i, v);
	479	+#else
	480	+ return raw_atomic_fetch_sub_relaxed(i, v);
	481	+#endif
668	482	}
669	483
	484	+/**
	485	+ * raw_atomic_long_inc() - atomic increment with relaxed ordering
	486	+ * @v: pointer to atomic_long_t
	487	+ *
	488	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	489	+ *
	490	+ * Safe to use in noinstr code; prefer atomic_long_inc() elsewhere.
	491	+ *
	492	+ * Return: Nothing.
	493	+ */
670	494	static __always_inline void
671		−arch_atomic_long_inc(atomic_long_t *v)
	495	+raw_atomic_long_inc(atomic_long_t *v)
672	496	{
673		− arch_atomic_inc(v);
	497	+#ifdef CONFIG_64BIT
	498	+ raw_atomic64_inc(v);
	499	+#else
	500	+ raw_atomic_inc(v);
	501	+#endif
674	502	}
675	503
	504	+/**
	505	+ * raw_atomic_long_inc_return() - atomic increment with full ordering
	506	+ * @v: pointer to atomic_long_t
	507	+ *
	508	+ * Atomically updates @v to (@v + 1) with full ordering.
	509	+ *
	510	+ * Safe to use in noinstr code; prefer atomic_long_inc_return() elsewhere.
	511	+ *
	512	+ * Return: The updated value of @v.
	513	+ */
676	514	static __always_inline long
677		−arch_atomic_long_inc_return(atomic_long_t *v)
	515	+raw_atomic_long_inc_return(atomic_long_t *v)
678	516	{
679		− return arch_atomic_inc_return(v);
	517	+#ifdef CONFIG_64BIT
	518	+ return raw_atomic64_inc_return(v);
	519	+#else
	520	+ return raw_atomic_inc_return(v);
	521	+#endif
680	522	}
681	523
	524	+/**
	525	+ * raw_atomic_long_inc_return_acquire() - atomic increment with acquire ordering
	526	+ * @v: pointer to atomic_long_t
	527	+ *
	528	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	529	+ *
	530	+ * Safe to use in noinstr code; prefer atomic_long_inc_return_acquire() elsewhere.
	531	+ *
	532	+ * Return: The updated value of @v.
	533	+ */
682	534	static __always_inline long
683		−arch_atomic_long_inc_return_acquire(atomic_long_t *v)
	535	+raw_atomic_long_inc_return_acquire(atomic_long_t *v)
684	536	{
685		− return arch_atomic_inc_return_acquire(v);
	537	+#ifdef CONFIG_64BIT
	538	+ return raw_atomic64_inc_return_acquire(v);
	539	+#else
	540	+ return raw_atomic_inc_return_acquire(v);
	541	+#endif
686	542	}
687	543
	544	+/**
	545	+ * raw_atomic_long_inc_return_release() - atomic increment with release ordering
	546	+ * @v: pointer to atomic_long_t
	547	+ *
	548	+ * Atomically updates @v to (@v + 1) with release ordering.
	549	+ *
	550	+ * Safe to use in noinstr code; prefer atomic_long_inc_return_release() elsewhere.
	551	+ *
	552	+ * Return: The updated value of @v.
	553	+ */
688	554	static __always_inline long
689		−arch_atomic_long_inc_return_release(atomic_long_t *v)
	555	+raw_atomic_long_inc_return_release(atomic_long_t *v)
690	556	{
691		− return arch_atomic_inc_return_release(v);
	557	+#ifdef CONFIG_64BIT
	558	+ return raw_atomic64_inc_return_release(v);
	559	+#else
	560	+ return raw_atomic_inc_return_release(v);
	561	+#endif
692	562	}
693	563
	564	+/**
	565	+ * raw_atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering
	566	+ * @v: pointer to atomic_long_t
	567	+ *
	568	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	569	+ *
	570	+ * Safe to use in noinstr code; prefer atomic_long_inc_return_relaxed() elsewhere.
	571	+ *
	572	+ * Return: The updated value of @v.
	573	+ */
694	574	static __always_inline long
695		−arch_atomic_long_inc_return_relaxed(atomic_long_t *v)
	575	+raw_atomic_long_inc_return_relaxed(atomic_long_t *v)
696	576	{
697		− return arch_atomic_inc_return_relaxed(v);
	577	+#ifdef CONFIG_64BIT
	578	+ return raw_atomic64_inc_return_relaxed(v);
	579	+#else
	580	+ return raw_atomic_inc_return_relaxed(v);
	581	+#endif
698	582	}
699	583
	584	+/**
	585	+ * raw_atomic_long_fetch_inc() - atomic increment with full ordering
	586	+ * @v: pointer to atomic_long_t
	587	+ *
	588	+ * Atomically updates @v to (@v + 1) with full ordering.
	589	+ *
	590	+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc() elsewhere.
	591	+ *
	592	+ * Return: The original value of @v.
	593	+ */
700	594	static __always_inline long
701		−arch_atomic_long_fetch_inc(atomic_long_t *v)
	595	+raw_atomic_long_fetch_inc(atomic_long_t *v)
702	596	{
703		− return arch_atomic_fetch_inc(v);
	597	+#ifdef CONFIG_64BIT
	598	+ return raw_atomic64_fetch_inc(v);
	599	+#else
	600	+ return raw_atomic_fetch_inc(v);
	601	+#endif
704	602	}
705	603
	604	+/**
	605	+ * raw_atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering
	606	+ * @v: pointer to atomic_long_t
	607	+ *
	608	+ * Atomically updates @v to (@v + 1) with acquire ordering.
	609	+ *
	610	+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc_acquire() elsewhere.
	611	+ *
	612	+ * Return: The original value of @v.
	613	+ */
706	614	static __always_inline long
707		−arch_atomic_long_fetch_inc_acquire(atomic_long_t *v)
	615	+raw_atomic_long_fetch_inc_acquire(atomic_long_t *v)
708	616	{
709		− return arch_atomic_fetch_inc_acquire(v);
	617	+#ifdef CONFIG_64BIT
	618	+ return raw_atomic64_fetch_inc_acquire(v);
	619	+#else
	620	+ return raw_atomic_fetch_inc_acquire(v);
	621	+#endif
710	622	}
711	623
	624	+/**
	625	+ * raw_atomic_long_fetch_inc_release() - atomic increment with release ordering
	626	+ * @v: pointer to atomic_long_t
	627	+ *
	628	+ * Atomically updates @v to (@v + 1) with release ordering.
	629	+ *
	630	+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc_release() elsewhere.
	631	+ *
	632	+ * Return: The original value of @v.
	633	+ */
712	634	static __always_inline long
713		−arch_atomic_long_fetch_inc_release(atomic_long_t *v)
	635	+raw_atomic_long_fetch_inc_release(atomic_long_t *v)
714	636	{
715		− return arch_atomic_fetch_inc_release(v);
	637	+#ifdef CONFIG_64BIT
	638	+ return raw_atomic64_fetch_inc_release(v);
	639	+#else
	640	+ return raw_atomic_fetch_inc_release(v);
	641	+#endif
716	642	}
717	643
	644	+/**
	645	+ * raw_atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering
	646	+ * @v: pointer to atomic_long_t
	647	+ *
	648	+ * Atomically updates @v to (@v + 1) with relaxed ordering.
	649	+ *
	650	+ * Safe to use in noinstr code; prefer atomic_long_fetch_inc_relaxed() elsewhere.
	651	+ *
	652	+ * Return: The original value of @v.
	653	+ */
718	654	static __always_inline long
719		−arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
	655	+raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v)
720	656	{
721		− return arch_atomic_fetch_inc_relaxed(v);
	657	+#ifdef CONFIG_64BIT
	658	+ return raw_atomic64_fetch_inc_relaxed(v);
	659	+#else
	660	+ return raw_atomic_fetch_inc_relaxed(v);
	661	+#endif
722	662	}
723	663
	664	+/**
	665	+ * raw_atomic_long_dec() - atomic decrement with relaxed ordering
	666	+ * @v: pointer to atomic_long_t
	667	+ *
	668	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	669	+ *
	670	+ * Safe to use in noinstr code; prefer atomic_long_dec() elsewhere.
	671	+ *
	672	+ * Return: Nothing.
	673	+ */
724	674	static __always_inline void
725		−arch_atomic_long_dec(atomic_long_t *v)
	675	+raw_atomic_long_dec(atomic_long_t *v)
726	676	{
727		− arch_atomic_dec(v);
	677	+#ifdef CONFIG_64BIT
	678	+ raw_atomic64_dec(v);
	679	+#else
	680	+ raw_atomic_dec(v);
	681	+#endif
728	682	}
729	683
	684	+/**
	685	+ * raw_atomic_long_dec_return() - atomic decrement with full ordering
	686	+ * @v: pointer to atomic_long_t
	687	+ *
	688	+ * Atomically updates @v to (@v - 1) with full ordering.
	689	+ *
	690	+ * Safe to use in noinstr code; prefer atomic_long_dec_return() elsewhere.
	691	+ *
	692	+ * Return: The updated value of @v.
	693	+ */
730	694	static __always_inline long
731		−arch_atomic_long_dec_return(atomic_long_t *v)
	695	+raw_atomic_long_dec_return(atomic_long_t *v)
732	696	{
733		− return arch_atomic_dec_return(v);
	697	+#ifdef CONFIG_64BIT
	698	+ return raw_atomic64_dec_return(v);
	699	+#else
	700	+ return raw_atomic_dec_return(v);
	701	+#endif
734	702	}
735	703
	704	+/**
	705	+ * raw_atomic_long_dec_return_acquire() - atomic decrement with acquire ordering
	706	+ * @v: pointer to atomic_long_t
	707	+ *
	708	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	709	+ *
	710	+ * Safe to use in noinstr code; prefer atomic_long_dec_return_acquire() elsewhere.
	711	+ *
	712	+ * Return: The updated value of @v.
	713	+ */
736	714	static __always_inline long
737		−arch_atomic_long_dec_return_acquire(atomic_long_t *v)
	715	+raw_atomic_long_dec_return_acquire(atomic_long_t *v)
738	716	{
739		− return arch_atomic_dec_return_acquire(v);
	717	+#ifdef CONFIG_64BIT
	718	+ return raw_atomic64_dec_return_acquire(v);
	719	+#else
	720	+ return raw_atomic_dec_return_acquire(v);
	721	+#endif
740	722	}
741	723
	724	+/**
	725	+ * raw_atomic_long_dec_return_release() - atomic decrement with release ordering
	726	+ * @v: pointer to atomic_long_t
	727	+ *
	728	+ * Atomically updates @v to (@v - 1) with release ordering.
	729	+ *
	730	+ * Safe to use in noinstr code; prefer atomic_long_dec_return_release() elsewhere.
	731	+ *
	732	+ * Return: The updated value of @v.
	733	+ */
742	734	static __always_inline long
743		−arch_atomic_long_dec_return_release(atomic_long_t *v)
	735	+raw_atomic_long_dec_return_release(atomic_long_t *v)
744	736	{
745		− return arch_atomic_dec_return_release(v);
	737	+#ifdef CONFIG_64BIT
	738	+ return raw_atomic64_dec_return_release(v);
	739	+#else
	740	+ return raw_atomic_dec_return_release(v);
	741	+#endif
746	742	}
747	743
	744	+/**
	745	+ * raw_atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering
	746	+ * @v: pointer to atomic_long_t
	747	+ *
	748	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	749	+ *
	750	+ * Safe to use in noinstr code; prefer atomic_long_dec_return_relaxed() elsewhere.
	751	+ *
	752	+ * Return: The updated value of @v.
	753	+ */
748	754	static __always_inline long
749		−arch_atomic_long_dec_return_relaxed(atomic_long_t *v)
	755	+raw_atomic_long_dec_return_relaxed(atomic_long_t *v)
750	756	{
751		− return arch_atomic_dec_return_relaxed(v);
	757	+#ifdef CONFIG_64BIT
	758	+ return raw_atomic64_dec_return_relaxed(v);
	759	+#else
	760	+ return raw_atomic_dec_return_relaxed(v);
	761	+#endif
752	762	}
753	763
	764	+/**
	765	+ * raw_atomic_long_fetch_dec() - atomic decrement with full ordering
	766	+ * @v: pointer to atomic_long_t
	767	+ *
	768	+ * Atomically updates @v to (@v - 1) with full ordering.
	769	+ *
	770	+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec() elsewhere.
	771	+ *
	772	+ * Return: The original value of @v.
	773	+ */
754	774	static __always_inline long
755		−arch_atomic_long_fetch_dec(atomic_long_t *v)
	775	+raw_atomic_long_fetch_dec(atomic_long_t *v)
756	776	{
757		− return arch_atomic_fetch_dec(v);
	777	+#ifdef CONFIG_64BIT
	778	+ return raw_atomic64_fetch_dec(v);
	779	+#else
	780	+ return raw_atomic_fetch_dec(v);
	781	+#endif
758	782	}
759	783
	784	+/**
	785	+ * raw_atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering
	786	+ * @v: pointer to atomic_long_t
	787	+ *
	788	+ * Atomically updates @v to (@v - 1) with acquire ordering.
	789	+ *
	790	+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec_acquire() elsewhere.
	791	+ *
	792	+ * Return: The original value of @v.
	793	+ */
760	794	static __always_inline long
761		−arch_atomic_long_fetch_dec_acquire(atomic_long_t *v)
	795	+raw_atomic_long_fetch_dec_acquire(atomic_long_t *v)
762	796	{
763		− return arch_atomic_fetch_dec_acquire(v);
	797	+#ifdef CONFIG_64BIT
	798	+ return raw_atomic64_fetch_dec_acquire(v);
	799	+#else
	800	+ return raw_atomic_fetch_dec_acquire(v);
	801	+#endif
764	802	}
765	803
	804	+/**
	805	+ * raw_atomic_long_fetch_dec_release() - atomic decrement with release ordering
	806	+ * @v: pointer to atomic_long_t
	807	+ *
	808	+ * Atomically updates @v to (@v - 1) with release ordering.
	809	+ *
	810	+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec_release() elsewhere.
	811	+ *
	812	+ * Return: The original value of @v.
	813	+ */
766	814	static __always_inline long
767		−arch_atomic_long_fetch_dec_release(atomic_long_t *v)
	815	+raw_atomic_long_fetch_dec_release(atomic_long_t *v)
768	816	{
769		− return arch_atomic_fetch_dec_release(v);
	817	+#ifdef CONFIG_64BIT
	818	+ return raw_atomic64_fetch_dec_release(v);
	819	+#else
	820	+ return raw_atomic_fetch_dec_release(v);
	821	+#endif
770	822	}
771	823
	824	+/**
	825	+ * raw_atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering
	826	+ * @v: pointer to atomic_long_t
	827	+ *
	828	+ * Atomically updates @v to (@v - 1) with relaxed ordering.
	829	+ *
	830	+ * Safe to use in noinstr code; prefer atomic_long_fetch_dec_relaxed() elsewhere.
	831	+ *
	832	+ * Return: The original value of @v.
	833	+ */
772	834	static __always_inline long
773		−arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
	835	+raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v)
774	836	{
775		− return arch_atomic_fetch_dec_relaxed(v);
	837	+#ifdef CONFIG_64BIT
	838	+ return raw_atomic64_fetch_dec_relaxed(v);
	839	+#else
	840	+ return raw_atomic_fetch_dec_relaxed(v);
	841	+#endif
776	842	}
777	843
	844	+/**
	845	+ * raw_atomic_long_and() - atomic bitwise AND with relaxed ordering
	846	+ * @i: long value
	847	+ * @v: pointer to atomic_long_t
	848	+ *
	849	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	850	+ *
	851	+ * Safe to use in noinstr code; prefer atomic_long_and() elsewhere.
	852	+ *
	853	+ * Return: Nothing.
	854	+ */
778	855	static __always_inline void
779		−arch_atomic_long_and(long i, atomic_long_t *v)
	856	+raw_atomic_long_and(long i, atomic_long_t *v)
780	857	{
781		− arch_atomic_and(i, v);
	858	+#ifdef CONFIG_64BIT
	859	+ raw_atomic64_and(i, v);
	860	+#else
	861	+ raw_atomic_and(i, v);
	862	+#endif
782	863	}
783	864
	865	+/**
	866	+ * raw_atomic_long_fetch_and() - atomic bitwise AND with full ordering
	867	+ * @i: long value
	868	+ * @v: pointer to atomic_long_t
	869	+ *
	870	+ * Atomically updates @v to (@v & @i) with full ordering.
	871	+ *
	872	+ * Safe to use in noinstr code; prefer atomic_long_fetch_and() elsewhere.
	873	+ *
	874	+ * Return: The original value of @v.
	875	+ */
784	876	static __always_inline long
785		−arch_atomic_long_fetch_and(long i, atomic_long_t *v)
	877	+raw_atomic_long_fetch_and(long i, atomic_long_t *v)
786	878	{
787		− return arch_atomic_fetch_and(i, v);
	879	+#ifdef CONFIG_64BIT
	880	+ return raw_atomic64_fetch_and(i, v);
	881	+#else
	882	+ return raw_atomic_fetch_and(i, v);
	883	+#endif
788	884	}
789	885
	886	+/**
	887	+ * raw_atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering
	888	+ * @i: long value
	889	+ * @v: pointer to atomic_long_t
	890	+ *
	891	+ * Atomically updates @v to (@v & @i) with acquire ordering.
	892	+ *
	893	+ * Safe to use in noinstr code; prefer atomic_long_fetch_and_acquire() elsewhere.
	894	+ *
	895	+ * Return: The original value of @v.
	896	+ */
790	897	static __always_inline long
791		−arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
	898	+raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
792	899	{
793		− return arch_atomic_fetch_and_acquire(i, v);
	900	+#ifdef CONFIG_64BIT
	901	+ return raw_atomic64_fetch_and_acquire(i, v);
	902	+#else
	903	+ return raw_atomic_fetch_and_acquire(i, v);
	904	+#endif
794	905	}
795	906
	907	+/**
	908	+ * raw_atomic_long_fetch_and_release() - atomic bitwise AND with release ordering
	909	+ * @i: long value
	910	+ * @v: pointer to atomic_long_t
	911	+ *
	912	+ * Atomically updates @v to (@v & @i) with release ordering.
	913	+ *
	914	+ * Safe to use in noinstr code; prefer atomic_long_fetch_and_release() elsewhere.
	915	+ *
	916	+ * Return: The original value of @v.
	917	+ */
796	918	static __always_inline long
797		−arch_atomic_long_fetch_and_release(long i, atomic_long_t *v)
	919	+raw_atomic_long_fetch_and_release(long i, atomic_long_t *v)
798	920	{
799		− return arch_atomic_fetch_and_release(i, v);
	921	+#ifdef CONFIG_64BIT
	922	+ return raw_atomic64_fetch_and_release(i, v);
	923	+#else
	924	+ return raw_atomic_fetch_and_release(i, v);
	925	+#endif
800	926	}
801	927
	928	+/**
	929	+ * raw_atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
	930	+ * @i: long value
	931	+ * @v: pointer to atomic_long_t
	932	+ *
	933	+ * Atomically updates @v to (@v & @i) with relaxed ordering.
	934	+ *
	935	+ * Safe to use in noinstr code; prefer atomic_long_fetch_and_relaxed() elsewhere.
	936	+ *
	937	+ * Return: The original value of @v.
	938	+ */
802	939	static __always_inline long
803		−arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
	940	+raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
804	941	{
805		− return arch_atomic_fetch_and_relaxed(i, v);
	942	+#ifdef CONFIG_64BIT
	943	+ return raw_atomic64_fetch_and_relaxed(i, v);
	944	+#else
	945	+ return raw_atomic_fetch_and_relaxed(i, v);
	946	+#endif
806	947	}
807	948
	949	+/**
	950	+ * raw_atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering
	951	+ * @i: long value
	952	+ * @v: pointer to atomic_long_t
	953	+ *
	954	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	955	+ *
	956	+ * Safe to use in noinstr code; prefer atomic_long_andnot() elsewhere.
	957	+ *
	958	+ * Return: Nothing.
	959	+ */
808	960	static __always_inline void
809		−arch_atomic_long_andnot(long i, atomic_long_t *v)
	961	+raw_atomic_long_andnot(long i, atomic_long_t *v)
810	962	{
811		− arch_atomic_andnot(i, v);
	963	+#ifdef CONFIG_64BIT
	964	+ raw_atomic64_andnot(i, v);
	965	+#else
	966	+ raw_atomic_andnot(i, v);
	967	+#endif
812	968	}
813	969
	970	+/**
	971	+ * raw_atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering
	972	+ * @i: long value
	973	+ * @v: pointer to atomic_long_t
	974	+ *
	975	+ * Atomically updates @v to (@v & ~@i) with full ordering.
	976	+ *
	977	+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot() elsewhere.
	978	+ *
	979	+ * Return: The original value of @v.
	980	+ */
814	981	static __always_inline long
815		−arch_atomic_long_fetch_andnot(long i, atomic_long_t *v)
	982	+raw_atomic_long_fetch_andnot(long i, atomic_long_t *v)
816	983	{
817		− return arch_atomic_fetch_andnot(i, v);
	984	+#ifdef CONFIG_64BIT
	985	+ return raw_atomic64_fetch_andnot(i, v);
	986	+#else
	987	+ return raw_atomic_fetch_andnot(i, v);
	988	+#endif
818	989	}
819	990
	991	+/**
	992	+ * raw_atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
	993	+ * @i: long value
	994	+ * @v: pointer to atomic_long_t
	995	+ *
	996	+ * Atomically updates @v to (@v & ~@i) with acquire ordering.
	997	+ *
	998	+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_acquire() elsewhere.
	999	+ *
	1000	+ * Return: The original value of @v.
	1001	+ */
820	1002	static __always_inline long
821		−arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
	1003	+raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
822	1004	{
823		− return arch_atomic_fetch_andnot_acquire(i, v);
	1005	+#ifdef CONFIG_64BIT
	1006	+ return raw_atomic64_fetch_andnot_acquire(i, v);
	1007	+#else
	1008	+ return raw_atomic_fetch_andnot_acquire(i, v);
	1009	+#endif
824	1010	}
825	1011
	1012	+/**
	1013	+ * raw_atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
	1014	+ * @i: long value
	1015	+ * @v: pointer to atomic_long_t
	1016	+ *
	1017	+ * Atomically updates @v to (@v & ~@i) with release ordering.
	1018	+ *
	1019	+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_release() elsewhere.
	1020	+ *
	1021	+ * Return: The original value of @v.
	1022	+ */
826	1023	static __always_inline long
827		−arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
	1024	+raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
828	1025	{
829		− return arch_atomic_fetch_andnot_release(i, v);
	1026	+#ifdef CONFIG_64BIT
	1027	+ return raw_atomic64_fetch_andnot_release(i, v);
	1028	+#else
	1029	+ return raw_atomic_fetch_andnot_release(i, v);
	1030	+#endif
830	1031	}
831	1032
	1033	+/**
	1034	+ * raw_atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
	1035	+ * @i: long value
	1036	+ * @v: pointer to atomic_long_t
	1037	+ *
	1038	+ * Atomically updates @v to (@v & ~@i) with relaxed ordering.
	1039	+ *
	1040	+ * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_relaxed() elsewhere.
	1041	+ *
	1042	+ * Return: The original value of @v.
	1043	+ */
832	1044	static __always_inline long
833		−arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
	1045	+raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
834	1046	{
835		− return arch_atomic_fetch_andnot_relaxed(i, v);
	1047	+#ifdef CONFIG_64BIT
	1048	+ return raw_atomic64_fetch_andnot_relaxed(i, v);
	1049	+#else
	1050	+ return raw_atomic_fetch_andnot_relaxed(i, v);
	1051	+#endif
836	1052	}
837	1053
	1054	+/**
	1055	+ * raw_atomic_long_or() - atomic bitwise OR with relaxed ordering
	1056	+ * @i: long value
	1057	+ * @v: pointer to atomic_long_t
	1058	+ *
	1059	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	1060	+ *
	1061	+ * Safe to use in noinstr code; prefer atomic_long_or() elsewhere.
	1062	+ *
	1063	+ * Return: Nothing.
	1064	+ */
838	1065	static __always_inline void
839		−arch_atomic_long_or(long i, atomic_long_t *v)
	1066	+raw_atomic_long_or(long i, atomic_long_t *v)
840	1067	{
841		− arch_atomic_or(i, v);
	1068	+#ifdef CONFIG_64BIT
	1069	+ raw_atomic64_or(i, v);
	1070	+#else
	1071	+ raw_atomic_or(i, v);
	1072	+#endif
842	1073	}
843	1074
	1075	+/**
	1076	+ * raw_atomic_long_fetch_or() - atomic bitwise OR with full ordering
	1077	+ * @i: long value
	1078	+ * @v: pointer to atomic_long_t
	1079	+ *
	1080	+ * Atomically updates @v to (@v \| @i) with full ordering.
	1081	+ *
	1082	+ * Safe to use in noinstr code; prefer atomic_long_fetch_or() elsewhere.
	1083	+ *
	1084	+ * Return: The original value of @v.
	1085	+ */
844	1086	static __always_inline long
845		−arch_atomic_long_fetch_or(long i, atomic_long_t *v)
	1087	+raw_atomic_long_fetch_or(long i, atomic_long_t *v)
846	1088	{
847		− return arch_atomic_fetch_or(i, v);
	1089	+#ifdef CONFIG_64BIT
	1090	+ return raw_atomic64_fetch_or(i, v);
	1091	+#else
	1092	+ return raw_atomic_fetch_or(i, v);
	1093	+#endif
848	1094	}
849	1095
	1096	+/**
	1097	+ * raw_atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering
	1098	+ * @i: long value
	1099	+ * @v: pointer to atomic_long_t
	1100	+ *
	1101	+ * Atomically updates @v to (@v \| @i) with acquire ordering.
	1102	+ *
	1103	+ * Safe to use in noinstr code; prefer atomic_long_fetch_or_acquire() elsewhere.
	1104	+ *
	1105	+ * Return: The original value of @v.
	1106	+ */
850	1107	static __always_inline long
851		−arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
	1108	+raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
852	1109	{
853		− return arch_atomic_fetch_or_acquire(i, v);
	1110	+#ifdef CONFIG_64BIT
	1111	+ return raw_atomic64_fetch_or_acquire(i, v);
	1112	+#else
	1113	+ return raw_atomic_fetch_or_acquire(i, v);
	1114	+#endif
854	1115	}
855	1116
	1117	+/**
	1118	+ * raw_atomic_long_fetch_or_release() - atomic bitwise OR with release ordering
	1119	+ * @i: long value
	1120	+ * @v: pointer to atomic_long_t
	1121	+ *
	1122	+ * Atomically updates @v to (@v \| @i) with release ordering.
	1123	+ *
	1124	+ * Safe to use in noinstr code; prefer atomic_long_fetch_or_release() elsewhere.
	1125	+ *
	1126	+ * Return: The original value of @v.
	1127	+ */
856	1128	static __always_inline long
857		−arch_atomic_long_fetch_or_release(long i, atomic_long_t *v)
	1129	+raw_atomic_long_fetch_or_release(long i, atomic_long_t *v)
858	1130	{
859		− return arch_atomic_fetch_or_release(i, v);
	1131	+#ifdef CONFIG_64BIT
	1132	+ return raw_atomic64_fetch_or_release(i, v);
	1133	+#else
	1134	+ return raw_atomic_fetch_or_release(i, v);
	1135	+#endif
860	1136	}
861	1137
	1138	+/**
	1139	+ * raw_atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
	1140	+ * @i: long value
	1141	+ * @v: pointer to atomic_long_t
	1142	+ *
	1143	+ * Atomically updates @v to (@v \| @i) with relaxed ordering.
	1144	+ *
	1145	+ * Safe to use in noinstr code; prefer atomic_long_fetch_or_relaxed() elsewhere.
	1146	+ *
	1147	+ * Return: The original value of @v.
	1148	+ */
862	1149	static __always_inline long
863		−arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
	1150	+raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
864	1151	{
865		− return arch_atomic_fetch_or_relaxed(i, v);
	1152	+#ifdef CONFIG_64BIT
	1153	+ return raw_atomic64_fetch_or_relaxed(i, v);
	1154	+#else
	1155	+ return raw_atomic_fetch_or_relaxed(i, v);
	1156	+#endif
866	1157	}
867	1158
	1159	+/**
	1160	+ * raw_atomic_long_xor() - atomic bitwise XOR with relaxed ordering
	1161	+ * @i: long value
	1162	+ * @v: pointer to atomic_long_t
	1163	+ *
	1164	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	1165	+ *
	1166	+ * Safe to use in noinstr code; prefer atomic_long_xor() elsewhere.
	1167	+ *
	1168	+ * Return: Nothing.
	1169	+ */
868	1170	static __always_inline void
869		−arch_atomic_long_xor(long i, atomic_long_t *v)
	1171	+raw_atomic_long_xor(long i, atomic_long_t *v)
870	1172	{
871		− arch_atomic_xor(i, v);
	1173	+#ifdef CONFIG_64BIT
	1174	+ raw_atomic64_xor(i, v);
	1175	+#else
	1176	+ raw_atomic_xor(i, v);
	1177	+#endif
872	1178	}
873	1179
	1180	+/**
	1181	+ * raw_atomic_long_fetch_xor() - atomic bitwise XOR with full ordering
	1182	+ * @i: long value
	1183	+ * @v: pointer to atomic_long_t
	1184	+ *
	1185	+ * Atomically updates @v to (@v ^ @i) with full ordering.
	1186	+ *
	1187	+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor() elsewhere.
	1188	+ *
	1189	+ * Return: The original value of @v.
	1190	+ */
874	1191	static __always_inline long
875		−arch_atomic_long_fetch_xor(long i, atomic_long_t *v)
	1192	+raw_atomic_long_fetch_xor(long i, atomic_long_t *v)
876	1193	{
877		− return arch_atomic_fetch_xor(i, v);
	1194	+#ifdef CONFIG_64BIT
	1195	+ return raw_atomic64_fetch_xor(i, v);
	1196	+#else
	1197	+ return raw_atomic_fetch_xor(i, v);
	1198	+#endif
878	1199	}
879	1200
	1201	+/**
	1202	+ * raw_atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
	1203	+ * @i: long value
	1204	+ * @v: pointer to atomic_long_t
	1205	+ *
	1206	+ * Atomically updates @v to (@v ^ @i) with acquire ordering.
	1207	+ *
	1208	+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor_acquire() elsewhere.
	1209	+ *
	1210	+ * Return: The original value of @v.
	1211	+ */
880	1212	static __always_inline long
881		−arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
	1213	+raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
882	1214	{
883		− return arch_atomic_fetch_xor_acquire(i, v);
	1215	+#ifdef CONFIG_64BIT
	1216	+ return raw_atomic64_fetch_xor_acquire(i, v);
	1217	+#else
	1218	+ return raw_atomic_fetch_xor_acquire(i, v);
	1219	+#endif
884	1220	}
885	1221
	1222	+/**
	1223	+ * raw_atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering
	1224	+ * @i: long value
	1225	+ * @v: pointer to atomic_long_t
	1226	+ *
	1227	+ * Atomically updates @v to (@v ^ @i) with release ordering.
	1228	+ *
	1229	+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor_release() elsewhere.
	1230	+ *
	1231	+ * Return: The original value of @v.
	1232	+ */
886	1233	static __always_inline long
887		−arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
	1234	+raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v)
888	1235	{
889		− return arch_atomic_fetch_xor_release(i, v);
	1236	+#ifdef CONFIG_64BIT
	1237	+ return raw_atomic64_fetch_xor_release(i, v);
	1238	+#else
	1239	+ return raw_atomic_fetch_xor_release(i, v);
	1240	+#endif
890	1241	}
891	1242
	1243	+/**
	1244	+ * raw_atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
	1245	+ * @i: long value
	1246	+ * @v: pointer to atomic_long_t
	1247	+ *
	1248	+ * Atomically updates @v to (@v ^ @i) with relaxed ordering.
	1249	+ *
	1250	+ * Safe to use in noinstr code; prefer atomic_long_fetch_xor_relaxed() elsewhere.
	1251	+ *
	1252	+ * Return: The original value of @v.
	1253	+ */
892	1254	static __always_inline long
893		−arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
	1255	+raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
894	1256	{
895		− return arch_atomic_fetch_xor_relaxed(i, v);
	1257	+#ifdef CONFIG_64BIT
	1258	+ return raw_atomic64_fetch_xor_relaxed(i, v);
	1259	+#else
	1260	+ return raw_atomic_fetch_xor_relaxed(i, v);
	1261	+#endif
896	1262	}
897	1263
	1264	+/**
	1265	+ * raw_atomic_long_xchg() - atomic exchange with full ordering
	1266	+ * @v: pointer to atomic_long_t
	1267	+ * @new: long value to assign
	1268	+ *
	1269	+ * Atomically updates @v to @new with full ordering.
	1270	+ *
	1271	+ * Safe to use in noinstr code; prefer atomic_long_xchg() elsewhere.
	1272	+ *
	1273	+ * Return: The original value of @v.
	1274	+ */
898	1275	static __always_inline long
899		−arch_atomic_long_xchg(atomic_long_t *v, long i)
	1276	+raw_atomic_long_xchg(atomic_long_t *v, long new)
900	1277	{
901		− return arch_atomic_xchg(v, i);
	1278	+#ifdef CONFIG_64BIT
	1279	+ return raw_atomic64_xchg(v, new);
	1280	+#else
	1281	+ return raw_atomic_xchg(v, new);
	1282	+#endif
902	1283	}
903	1284
	1285	+/**
	1286	+ * raw_atomic_long_xchg_acquire() - atomic exchange with acquire ordering
	1287	+ * @v: pointer to atomic_long_t
	1288	+ * @new: long value to assign
	1289	+ *
	1290	+ * Atomically updates @v to @new with acquire ordering.
	1291	+ *
	1292	+ * Safe to use in noinstr code; prefer atomic_long_xchg_acquire() elsewhere.
	1293	+ *
	1294	+ * Return: The original value of @v.
	1295	+ */
904	1296	static __always_inline long
905		−arch_atomic_long_xchg_acquire(atomic_long_t *v, long i)
	1297	+raw_atomic_long_xchg_acquire(atomic_long_t *v, long new)
906	1298	{
907		− return arch_atomic_xchg_acquire(v, i);
	1299	+#ifdef CONFIG_64BIT
	1300	+ return raw_atomic64_xchg_acquire(v, new);
	1301	+#else
	1302	+ return raw_atomic_xchg_acquire(v, new);
	1303	+#endif
908	1304	}
909	1305
	1306	+/**
	1307	+ * raw_atomic_long_xchg_release() - atomic exchange with release ordering
	1308	+ * @v: pointer to atomic_long_t
	1309	+ * @new: long value to assign
	1310	+ *
	1311	+ * Atomically updates @v to @new with release ordering.
	1312	+ *
	1313	+ * Safe to use in noinstr code; prefer atomic_long_xchg_release() elsewhere.
	1314	+ *
	1315	+ * Return: The original value of @v.
	1316	+ */
910	1317	static __always_inline long
911		−arch_atomic_long_xchg_release(atomic_long_t *v, long i)
	1318	+raw_atomic_long_xchg_release(atomic_long_t *v, long new)
912	1319	{
913		− return arch_atomic_xchg_release(v, i);
	1320	+#ifdef CONFIG_64BIT
	1321	+ return raw_atomic64_xchg_release(v, new);
	1322	+#else
	1323	+ return raw_atomic_xchg_release(v, new);
	1324	+#endif
914	1325	}
915	1326
	1327	+/**
	1328	+ * raw_atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering
	1329	+ * @v: pointer to atomic_long_t
	1330	+ * @new: long value to assign
	1331	+ *
	1332	+ * Atomically updates @v to @new with relaxed ordering.
	1333	+ *
	1334	+ * Safe to use in noinstr code; prefer atomic_long_xchg_relaxed() elsewhere.
	1335	+ *
	1336	+ * Return: The original value of @v.
	1337	+ */
916	1338	static __always_inline long
917		−arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i)
	1339	+raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new)
918	1340	{
919		− return arch_atomic_xchg_relaxed(v, i);
	1341	+#ifdef CONFIG_64BIT
	1342	+ return raw_atomic64_xchg_relaxed(v, new);
	1343	+#else
	1344	+ return raw_atomic_xchg_relaxed(v, new);
	1345	+#endif
920	1346	}
921	1347
	1348	+/**
	1349	+ * raw_atomic_long_cmpxchg() - atomic compare and exchange with full ordering
	1350	+ * @v: pointer to atomic_long_t
	1351	+ * @old: long value to compare with
	1352	+ * @new: long value to assign
	1353	+ *
	1354	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	1355	+ *
	1356	+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg() elsewhere.
	1357	+ *
	1358	+ * Return: The original value of @v.
	1359	+ */
922	1360	static __always_inline long
923		−arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
	1361	+raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
924	1362	{
925		− return arch_atomic_cmpxchg(v, old, new);
	1363	+#ifdef CONFIG_64BIT
	1364	+ return raw_atomic64_cmpxchg(v, old, new);
	1365	+#else
	1366	+ return raw_atomic_cmpxchg(v, old, new);
	1367	+#endif
926	1368	}
927	1369
	1370	+/**
	1371	+ * raw_atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	1372	+ * @v: pointer to atomic_long_t
	1373	+ * @old: long value to compare with
	1374	+ * @new: long value to assign
	1375	+ *
	1376	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	1377	+ *
	1378	+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg_acquire() elsewhere.
	1379	+ *
	1380	+ * Return: The original value of @v.
	1381	+ */
928	1382	static __always_inline long
929		−arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
	1383	+raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
930	1384	{
931		− return arch_atomic_cmpxchg_acquire(v, old, new);
	1385	+#ifdef CONFIG_64BIT
	1386	+ return raw_atomic64_cmpxchg_acquire(v, old, new);
	1387	+#else
	1388	+ return raw_atomic_cmpxchg_acquire(v, old, new);
	1389	+#endif
932	1390	}
933	1391
	1392	+/**
	1393	+ * raw_atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering
	1394	+ * @v: pointer to atomic_long_t
	1395	+ * @old: long value to compare with
	1396	+ * @new: long value to assign
	1397	+ *
	1398	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	1399	+ *
	1400	+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg_release() elsewhere.
	1401	+ *
	1402	+ * Return: The original value of @v.
	1403	+ */
934	1404	static __always_inline long
935		−arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
	1405	+raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
936	1406	{
937		− return arch_atomic_cmpxchg_release(v, old, new);
	1407	+#ifdef CONFIG_64BIT
	1408	+ return raw_atomic64_cmpxchg_release(v, old, new);
	1409	+#else
	1410	+ return raw_atomic_cmpxchg_release(v, old, new);
	1411	+#endif
938	1412	}
939	1413
	1414	+/**
	1415	+ * raw_atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	1416	+ * @v: pointer to atomic_long_t
	1417	+ * @old: long value to compare with
	1418	+ * @new: long value to assign
	1419	+ *
	1420	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	1421	+ *
	1422	+ * Safe to use in noinstr code; prefer atomic_long_cmpxchg_relaxed() elsewhere.
	1423	+ *
	1424	+ * Return: The original value of @v.
	1425	+ */
940	1426	static __always_inline long
941		−arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
	1427	+raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
942	1428	{
943		− return arch_atomic_cmpxchg_relaxed(v, old, new);
	1429	+#ifdef CONFIG_64BIT
	1430	+ return raw_atomic64_cmpxchg_relaxed(v, old, new);
	1431	+#else
	1432	+ return raw_atomic_cmpxchg_relaxed(v, old, new);
	1433	+#endif
944	1434	}
945	1435
	1436	+/**
	1437	+ * raw_atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering
	1438	+ * @v: pointer to atomic_long_t
	1439	+ * @old: pointer to long value to compare with
	1440	+ * @new: long value to assign
	1441	+ *
	1442	+ * If (@v == @old), atomically updates @v to @new with full ordering.
	1443	+ * Otherwise, updates @old to the current value of @v.
	1444	+ *
	1445	+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg() elsewhere.
	1446	+ *
	1447	+ * Return: @true if the exchange occured, @false otherwise.
	1448	+ */
946	1449	static __always_inline bool
947		−arch_atomic_long_try_cmpxchg(atomic_long_t v, long old, long new)
	1450	+raw_atomic_long_try_cmpxchg(atomic_long_t v, long old, long new)
948	1451	{
949		− return arch_atomic_try_cmpxchg(v, (int *)old, new);
	1452	+#ifdef CONFIG_64BIT
	1453	+ return raw_atomic64_try_cmpxchg(v, (s64 *)old, new);
	1454	+#else
	1455	+ return raw_atomic_try_cmpxchg(v, (int *)old, new);
	1456	+#endif
950	1457	}
951	1458
	1459	+/**
	1460	+ * raw_atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
	1461	+ * @v: pointer to atomic_long_t
	1462	+ * @old: pointer to long value to compare with
	1463	+ * @new: long value to assign
	1464	+ *
	1465	+ * If (@v == @old), atomically updates @v to @new with acquire ordering.
	1466	+ * Otherwise, updates @old to the current value of @v.
	1467	+ *
	1468	+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_acquire() elsewhere.
	1469	+ *
	1470	+ * Return: @true if the exchange occured, @false otherwise.
	1471	+ */
952	1472	static __always_inline bool
953		−arch_atomic_long_try_cmpxchg_acquire(atomic_long_t v, long old, long new)
	1473	+raw_atomic_long_try_cmpxchg_acquire(atomic_long_t v, long old, long new)
954	1474	{
955		− return arch_atomic_try_cmpxchg_acquire(v, (int *)old, new);
	1475	+#ifdef CONFIG_64BIT
	1476	+ return raw_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
	1477	+#else
	1478	+ return raw_atomic_try_cmpxchg_acquire(v, (int *)old, new);
	1479	+#endif
956	1480	}
957	1481
	1482	+/**
	1483	+ * raw_atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering
	1484	+ * @v: pointer to atomic_long_t
	1485	+ * @old: pointer to long value to compare with
	1486	+ * @new: long value to assign
	1487	+ *
	1488	+ * If (@v == @old), atomically updates @v to @new with release ordering.
	1489	+ * Otherwise, updates @old to the current value of @v.
	1490	+ *
	1491	+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_release() elsewhere.
	1492	+ *
	1493	+ * Return: @true if the exchange occured, @false otherwise.
	1494	+ */
958	1495	static __always_inline bool
959		−arch_atomic_long_try_cmpxchg_release(atomic_long_t v, long old, long new)
	1496	+raw_atomic_long_try_cmpxchg_release(atomic_long_t v, long old, long new)
960	1497	{
961		− return arch_atomic_try_cmpxchg_release(v, (int *)old, new);
	1498	+#ifdef CONFIG_64BIT
	1499	+ return raw_atomic64_try_cmpxchg_release(v, (s64 *)old, new);
	1500	+#else
	1501	+ return raw_atomic_try_cmpxchg_release(v, (int *)old, new);
	1502	+#endif
962	1503	}
963	1504
	1505	+/**
	1506	+ * raw_atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
	1507	+ * @v: pointer to atomic_long_t
	1508	+ * @old: pointer to long value to compare with
	1509	+ * @new: long value to assign
	1510	+ *
	1511	+ * If (@v == @old), atomically updates @v to @new with relaxed ordering.
	1512	+ * Otherwise, updates @old to the current value of @v.
	1513	+ *
	1514	+ * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_relaxed() elsewhere.
	1515	+ *
	1516	+ * Return: @true if the exchange occured, @false otherwise.
	1517	+ */
964	1518	static __always_inline bool
965		−arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t v, long old, long new)
	1519	+raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t v, long old, long new)
966	1520	{
967		− return arch_atomic_try_cmpxchg_relaxed(v, (int *)old, new);
	1521	+#ifdef CONFIG_64BIT
	1522	+ return raw_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
	1523	+#else
	1524	+ return raw_atomic_try_cmpxchg_relaxed(v, (int *)old, new);
	1525	+#endif
968	1526	}
969	1527
	1528	+/**
	1529	+ * raw_atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering
	1530	+ * @i: long value to add
	1531	+ * @v: pointer to atomic_long_t
	1532	+ *
	1533	+ * Atomically updates @v to (@v - @i) with full ordering.
	1534	+ *
	1535	+ * Safe to use in noinstr code; prefer atomic_long_sub_and_test() elsewhere.
	1536	+ *
	1537	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	1538	+ */
970	1539	static __always_inline bool
971		−arch_atomic_long_sub_and_test(long i, atomic_long_t *v)
	1540	+raw_atomic_long_sub_and_test(long i, atomic_long_t *v)
972	1541	{
973		− return arch_atomic_sub_and_test(i, v);
	1542	+#ifdef CONFIG_64BIT
	1543	+ return raw_atomic64_sub_and_test(i, v);
	1544	+#else
	1545	+ return raw_atomic_sub_and_test(i, v);
	1546	+#endif
974	1547	}
975	1548
	1549	+/**
	1550	+ * raw_atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering
	1551	+ * @v: pointer to atomic_long_t
	1552	+ *
	1553	+ * Atomically updates @v to (@v - 1) with full ordering.
	1554	+ *
	1555	+ * Safe to use in noinstr code; prefer atomic_long_dec_and_test() elsewhere.
	1556	+ *
	1557	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	1558	+ */
976	1559	static __always_inline bool
977		−arch_atomic_long_dec_and_test(atomic_long_t *v)
	1560	+raw_atomic_long_dec_and_test(atomic_long_t *v)
978	1561	{
979		− return arch_atomic_dec_and_test(v);
	1562	+#ifdef CONFIG_64BIT
	1563	+ return raw_atomic64_dec_and_test(v);
	1564	+#else
	1565	+ return raw_atomic_dec_and_test(v);
	1566	+#endif
980	1567	}
981	1568
	1569	+/**
	1570	+ * raw_atomic_long_inc_and_test() - atomic increment and test if zero with full ordering
	1571	+ * @v: pointer to atomic_long_t
	1572	+ *
	1573	+ * Atomically updates @v to (@v + 1) with full ordering.
	1574	+ *
	1575	+ * Safe to use in noinstr code; prefer atomic_long_inc_and_test() elsewhere.
	1576	+ *
	1577	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	1578	+ */
982	1579	static __always_inline bool
983		−arch_atomic_long_inc_and_test(atomic_long_t *v)
	1580	+raw_atomic_long_inc_and_test(atomic_long_t *v)
984	1581	{
985		− return arch_atomic_inc_and_test(v);
	1582	+#ifdef CONFIG_64BIT
	1583	+ return raw_atomic64_inc_and_test(v);
	1584	+#else
	1585	+ return raw_atomic_inc_and_test(v);
	1586	+#endif
986	1587	}
987	1588
	1589	+/**
	1590	+ * raw_atomic_long_add_negative() - atomic add and test if negative with full ordering
	1591	+ * @i: long value to add
	1592	+ * @v: pointer to atomic_long_t
	1593	+ *
	1594	+ * Atomically updates @v to (@v + @i) with full ordering.
	1595	+ *
	1596	+ * Safe to use in noinstr code; prefer atomic_long_add_negative() elsewhere.
	1597	+ *
	1598	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1599	+ */
988	1600	static __always_inline bool
989		−arch_atomic_long_add_negative(long i, atomic_long_t *v)
	1601	+raw_atomic_long_add_negative(long i, atomic_long_t *v)
990	1602	{
991		− return arch_atomic_add_negative(i, v);
	1603	+#ifdef CONFIG_64BIT
	1604	+ return raw_atomic64_add_negative(i, v);
	1605	+#else
	1606	+ return raw_atomic_add_negative(i, v);
	1607	+#endif
992	1608	}
993	1609
	1610	+/**
	1611	+ * raw_atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering
	1612	+ * @i: long value to add
	1613	+ * @v: pointer to atomic_long_t
	1614	+ *
	1615	+ * Atomically updates @v to (@v + @i) with acquire ordering.
	1616	+ *
	1617	+ * Safe to use in noinstr code; prefer atomic_long_add_negative_acquire() elsewhere.
	1618	+ *
	1619	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1620	+ */
994	1621	static __always_inline bool
995		−arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
	1622	+raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v)
996	1623	{
997		− return arch_atomic_add_negative_acquire(i, v);
	1624	+#ifdef CONFIG_64BIT
	1625	+ return raw_atomic64_add_negative_acquire(i, v);
	1626	+#else
	1627	+ return raw_atomic_add_negative_acquire(i, v);
	1628	+#endif
998	1629	}
999	1630
	1631	+/**
	1632	+ * raw_atomic_long_add_negative_release() - atomic add and test if negative with release ordering
	1633	+ * @i: long value to add
	1634	+ * @v: pointer to atomic_long_t
	1635	+ *
	1636	+ * Atomically updates @v to (@v + @i) with release ordering.
	1637	+ *
	1638	+ * Safe to use in noinstr code; prefer atomic_long_add_negative_release() elsewhere.
	1639	+ *
	1640	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1641	+ */
1000	1642	static __always_inline bool
1001		−arch_atomic_long_add_negative_release(long i, atomic_long_t *v)
	1643	+raw_atomic_long_add_negative_release(long i, atomic_long_t *v)
1002	1644	{
1003		− return arch_atomic_add_negative_release(i, v);
	1645	+#ifdef CONFIG_64BIT
	1646	+ return raw_atomic64_add_negative_release(i, v);
	1647	+#else
	1648	+ return raw_atomic_add_negative_release(i, v);
	1649	+#endif
1004	1650	}
1005	1651
	1652	+/**
	1653	+ * raw_atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
	1654	+ * @i: long value to add
	1655	+ * @v: pointer to atomic_long_t
	1656	+ *
	1657	+ * Atomically updates @v to (@v + @i) with relaxed ordering.
	1658	+ *
	1659	+ * Safe to use in noinstr code; prefer atomic_long_add_negative_relaxed() elsewhere.
	1660	+ *
	1661	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	1662	+ */
1006	1663	static __always_inline bool
1007		−arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
	1664	+raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1008	1665	{
1009		− return arch_atomic_add_negative_relaxed(i, v);
	1666	+#ifdef CONFIG_64BIT
	1667	+ return raw_atomic64_add_negative_relaxed(i, v);
	1668	+#else
	1669	+ return raw_atomic_add_negative_relaxed(i, v);
	1670	+#endif
1010	1671	}
1011	1672
	1673	+/**
	1674	+ * raw_atomic_long_fetch_add_unless() - atomic add unless value with full ordering
	1675	+ * @v: pointer to atomic_long_t
	1676	+ * @a: long value to add
	1677	+ * @u: long value to compare with
	1678	+ *
	1679	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1680	+ *
	1681	+ * Safe to use in noinstr code; prefer atomic_long_fetch_add_unless() elsewhere.
	1682	+ *
	1683	+ * Return: The original value of @v.
	1684	+ */
1012	1685	static __always_inline long
1013		−arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
	1686	+raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1014	1687	{
1015		− return arch_atomic_fetch_add_unless(v, a, u);
	1688	+#ifdef CONFIG_64BIT
	1689	+ return raw_atomic64_fetch_add_unless(v, a, u);
	1690	+#else
	1691	+ return raw_atomic_fetch_add_unless(v, a, u);
	1692	+#endif
1016	1693	}
1017	1694
	1695	+/**
	1696	+ * raw_atomic_long_add_unless() - atomic add unless value with full ordering
	1697	+ * @v: pointer to atomic_long_t
	1698	+ * @a: long value to add
	1699	+ * @u: long value to compare with
	1700	+ *
	1701	+ * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
	1702	+ *
	1703	+ * Safe to use in noinstr code; prefer atomic_long_add_unless() elsewhere.
	1704	+ *
	1705	+ * Return: @true if @v was updated, @false otherwise.
	1706	+ */
1018	1707	static __always_inline bool
1019		−arch_atomic_long_add_unless(atomic_long_t *v, long a, long u)
	1708	+raw_atomic_long_add_unless(atomic_long_t *v, long a, long u)
1020	1709	{
1021		− return arch_atomic_add_unless(v, a, u);
	1710	+#ifdef CONFIG_64BIT
	1711	+ return raw_atomic64_add_unless(v, a, u);
	1712	+#else
	1713	+ return raw_atomic_add_unless(v, a, u);
	1714	+#endif
1022	1715	}
1023	1716
	1717	+/**
	1718	+ * raw_atomic_long_inc_not_zero() - atomic increment unless zero with full ordering
	1719	+ * @v: pointer to atomic_long_t
	1720	+ *
	1721	+ * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
	1722	+ *
	1723	+ * Safe to use in noinstr code; prefer atomic_long_inc_not_zero() elsewhere.
	1724	+ *
	1725	+ * Return: @true if @v was updated, @false otherwise.
	1726	+ */
1024	1727	static __always_inline bool
1025		−arch_atomic_long_inc_not_zero(atomic_long_t *v)
	1728	+raw_atomic_long_inc_not_zero(atomic_long_t *v)
1026	1729	{
1027		− return arch_atomic_inc_not_zero(v);
	1730	+#ifdef CONFIG_64BIT
	1731	+ return raw_atomic64_inc_not_zero(v);
	1732	+#else
	1733	+ return raw_atomic_inc_not_zero(v);
	1734	+#endif
1028	1735	}
1029	1736
	1737	+/**
	1738	+ * raw_atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering
	1739	+ * @v: pointer to atomic_long_t
	1740	+ *
	1741	+ * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
	1742	+ *
	1743	+ * Safe to use in noinstr code; prefer atomic_long_inc_unless_negative() elsewhere.
	1744	+ *
	1745	+ * Return: @true if @v was updated, @false otherwise.
	1746	+ */
1030	1747	static __always_inline bool
1031		−arch_atomic_long_inc_unless_negative(atomic_long_t *v)
	1748	+raw_atomic_long_inc_unless_negative(atomic_long_t *v)
1032	1749	{
1033		− return arch_atomic_inc_unless_negative(v);
	1750	+#ifdef CONFIG_64BIT
	1751	+ return raw_atomic64_inc_unless_negative(v);
	1752	+#else
	1753	+ return raw_atomic_inc_unless_negative(v);
	1754	+#endif
1034	1755	}
1035	1756
	1757	+/**
	1758	+ * raw_atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering
	1759	+ * @v: pointer to atomic_long_t
	1760	+ *
	1761	+ * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
	1762	+ *
	1763	+ * Safe to use in noinstr code; prefer atomic_long_dec_unless_positive() elsewhere.
	1764	+ *
	1765	+ * Return: @true if @v was updated, @false otherwise.
	1766	+ */
1036	1767	static __always_inline bool
1037		−arch_atomic_long_dec_unless_positive(atomic_long_t *v)
	1768	+raw_atomic_long_dec_unless_positive(atomic_long_t *v)
1038	1769	{
1039		− return arch_atomic_dec_unless_positive(v);
	1770	+#ifdef CONFIG_64BIT
	1771	+ return raw_atomic64_dec_unless_positive(v);
	1772	+#else
	1773	+ return raw_atomic_dec_unless_positive(v);
	1774	+#endif
1040	1775	}
1041	1776
	1777	+/**
	1778	+ * raw_atomic_long_dec_if_positive() - atomic decrement if positive with full ordering
	1779	+ * @v: pointer to atomic_long_t
	1780	+ *
	1781	+ * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
	1782	+ *
	1783	+ * Safe to use in noinstr code; prefer atomic_long_dec_if_positive() elsewhere.
	1784	+ *
	1785	+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
	1786	+ */
1042	1787	static __always_inline long
1043		−arch_atomic_long_dec_if_positive(atomic_long_t *v)
	1788	+raw_atomic_long_dec_if_positive(atomic_long_t *v)
1044	1789	{
1045		− return arch_atomic_dec_if_positive(v);
	1790	+#ifdef CONFIG_64BIT
	1791	+ return raw_atomic64_dec_if_positive(v);
	1792	+#else
	1793	+ return raw_atomic_dec_if_positive(v);
	1794	+#endif
1046	1795	}
1047	1796
1048		−#endif /* CONFIG_64BIT */
1049	1797	#endif /* _LINUX_ATOMIC_LONG_H */
1050		−// a194c07d7d2f4b0e178d3c118c919775d5d65f50
	1798	+// 4ef23f98c73cff96d239896175fd26b10b88899e

include/linux/context_tracking.h+2−2

@@ -119,7 +119,7 @@ extern void ct_idle_exit(void); · 118 unmodified lines
119	119	*/
120	120	static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
121	121	{
122		− return !(arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX);
	122	+ return !(raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX);
123	123	}
124	124
125	125	/*
@@ -128,7 +128,7 @@ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) · 2 unmodified lines
128	128	*/
129	129	static __always_inline unsigned long ct_state_inc(int incby)
130	130	{
131		− return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));
	131	+ return raw_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state));
132	132	}
133	133
134	134	static __always_inline bool warn_rcu_enter(void)

include/linux/context_tracking_state.h+1−1

@@ -51,7 +51,7 @@ DECLARE_PER_CPU(struct context_tracking, context_tracking); · 50 unmodified lines
51	51	#ifdef CONFIG_CONTEXT_TRACKING_USER
52	52	static __always_inline int __ct_state(void)
53	53	{
54		− return arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;
	54	+ return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK;
55	55	}
56	56	#endif
57	57

include/linux/cpumask.h+1−1

@@ -1071,7 +1071,7 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu) · 1070 unmodified lines
1071	1071	*/
1072	1072	static __always_inline unsigned int num_online_cpus(void)
1073	1073	{
1074		− return arch_atomic_read(&__num_online_cpus);
	1074	+ return raw_atomic_read(&__num_online_cpus);
1075	1075	}
1076	1076	#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
1077	1077	#define num_present_cpus() cpumask_weight(cpu_present_mask)

include/linux/dmar.h+66−59

@@ -202,67 +202,74 @@ static inline void detect_intel_iommu(void) · 201 unmodified lines
202	202
203	203	struct irte {
204	204	union {
205		− /* Shared between remapped and posted mode*/
206	205	struct {
207		− __u64 present : 1, /* 0 */
208		− fpd : 1, /* 1 */
209		− __res0 : 6, /* 2 - 6 */
210		− avail : 4, /* 8 - 11 */
211		− __res1 : 3, /* 12 - 14 */
212		− pst : 1, /* 15 */
213		− vector : 8, /* 16 - 23 */
214		− __res2 : 40; /* 24 - 63 */
	206	+ union {
	207	+ /* Shared between remapped and posted mode*/
	208	+ struct {
	209	+ __u64 present : 1, /* 0 */
	210	+ fpd : 1, /* 1 */
	211	+ __res0 : 6, /* 2 - 6 */
	212	+ avail : 4, /* 8 - 11 */
	213	+ __res1 : 3, /* 12 - 14 */
	214	+ pst : 1, /* 15 */
	215	+ vector : 8, /* 16 - 23 */
	216	+ __res2 : 40; /* 24 - 63 */
	217	+ };
	218	+
	219	+ /* Remapped mode */
	220	+ struct {
	221	+ __u64 r_present : 1, /* 0 */
	222	+ r_fpd : 1, /* 1 */
	223	+ dst_mode : 1, /* 2 */
	224	+ redir_hint : 1, /* 3 */
	225	+ trigger_mode : 1, /* 4 */
	226	+ dlvry_mode : 3, /* 5 - 7 */
	227	+ r_avail : 4, /* 8 - 11 */
	228	+ r_res0 : 4, /* 12 - 15 */
	229	+ r_vector : 8, /* 16 - 23 */
	230	+ r_res1 : 8, /* 24 - 31 */
	231	+ dest_id : 32; /* 32 - 63 */
	232	+ };
	233	+
	234	+ /* Posted mode */
	235	+ struct {
	236	+ __u64 p_present : 1, /* 0 */
	237	+ p_fpd : 1, /* 1 */
	238	+ p_res0 : 6, /* 2 - 7 */
	239	+ p_avail : 4, /* 8 - 11 */
	240	+ p_res1 : 2, /* 12 - 13 */
	241	+ p_urgent : 1, /* 14 */
	242	+ p_pst : 1, /* 15 */
	243	+ p_vector : 8, /* 16 - 23 */
	244	+ p_res2 : 14, /* 24 - 37 */
	245	+ pda_l : 26; /* 38 - 63 */
	246	+ };
	247	+ __u64 low;
	248	+ };
	249	+
	250	+ union {
	251	+ /* Shared between remapped and posted mode*/
	252	+ struct {
	253	+ __u64 sid : 16, /* 64 - 79 */
	254	+ sq : 2, /* 80 - 81 */
	255	+ svt : 2, /* 82 - 83 */
	256	+ __res3 : 44; /* 84 - 127 */
	257	+ };
	258	+
	259	+ /* Posted mode*/
	260	+ struct {
	261	+ __u64 p_sid : 16, /* 64 - 79 */
	262	+ p_sq : 2, /* 80 - 81 */
	263	+ p_svt : 2, /* 82 - 83 */
	264	+ p_res3 : 12, /* 84 - 95 */
	265	+ pda_h : 32; /* 96 - 127 */
	266	+ };
	267	+ __u64 high;
	268	+ };
215	269	};
216		−
217		− /* Remapped mode */
218		− struct {
219		− __u64 r_present : 1, /* 0 */
220		− r_fpd : 1, /* 1 */
221		− dst_mode : 1, /* 2 */
222		− redir_hint : 1, /* 3 */
223		− trigger_mode : 1, /* 4 */
224		− dlvry_mode : 3, /* 5 - 7 */
225		− r_avail : 4, /* 8 - 11 */
226		− r_res0 : 4, /* 12 - 15 */
227		− r_vector : 8, /* 16 - 23 */
228		− r_res1 : 8, /* 24 - 31 */
229		− dest_id : 32; /* 32 - 63 */
230		− };
231		−
232		− /* Posted mode */
233		− struct {
234		− __u64 p_present : 1, /* 0 */
235		− p_fpd : 1, /* 1 */
236		− p_res0 : 6, /* 2 - 7 */
237		− p_avail : 4, /* 8 - 11 */
238		− p_res1 : 2, /* 12 - 13 */
239		− p_urgent : 1, /* 14 */
240		− p_pst : 1, /* 15 */
241		− p_vector : 8, /* 16 - 23 */
242		− p_res2 : 14, /* 24 - 37 */
243		− pda_l : 26; /* 38 - 63 */
244		− };
245		− __u64 low;
246		− };
247		−
248		− union {
249		− /* Shared between remapped and posted mode*/
250		− struct {
251		− __u64 sid : 16, /* 64 - 79 */
252		− sq : 2, /* 80 - 81 */
253		− svt : 2, /* 82 - 83 */
254		− __res3 : 44; /* 84 - 127 */
255		− };
256		−
257		− /* Posted mode*/
258		− struct {
259		− __u64 p_sid : 16, /* 64 - 79 */
260		− p_sq : 2, /* 80 - 81 */
261		− p_svt : 2, /* 82 - 83 */
262		− p_res3 : 12, /* 84 - 95 */
263		− pda_h : 32; /* 96 - 127 */
264		− };
265		− __u64 high;
	270	+#ifdef CONFIG_IRQ_REMAP
	271	+ __u128 irte;
	272	+#endif
266	273	};
267	274	};
268	275

include/linux/jump_label.h+1−1

@@ -257,7 +257,7 @@ extern enum jump_label_type jump_label_init_type(struct jump_entry *entry); · 256 unmodified lines
257	257
258	258	static __always_inline int static_key_count(struct static_key *key)
259	259	{
260		− return arch_atomic_read(&key->enabled);
	260	+ return raw_atomic_read(&key->enabled);
261	261	}
262	262
263	263	static __always_inline void jump_label_init(void)

include/linux/lockdep.h+8−0

@@ -447,6 +447,14 @@ extern int lockdep_is_held(const void *); · 446 unmodified lines
447	447
448	448	#endif /* !LOCKDEP */
449	449
	450	+#ifdef CONFIG_PROVE_LOCKING
	451	+void lockdep_set_lock_cmp_fn(struct lockdep_map *, lock_cmp_fn, lock_print_fn);
	452	+
	453	+#define lock_set_cmp_fn(lock, ...) lockdep_set_lock_cmp_fn(&(lock)->dep_map, __VA_ARGS__)
	454	+#else
	455	+#define lock_set_cmp_fn(lock, ...) do { } while (0)
	456	+#endif
	457	+
450	458	enum xhlock_context_t {
451	459	XHLOCK_HARD,
452	460	XHLOCK_SOFT,

include/linux/lockdep_types.h+8−0

@@ -85,6 +85,11 @@ struct lock_trace; · 84 unmodified lines
85	85
86	86	#define LOCKSTAT_POINTS 4
87	87
	88	+struct lockdep_map;
	89	+typedef int (lock_cmp_fn)(const struct lockdep_map a,
	90	+ const struct lockdep_map *b);
	91	+typedef void (lock_print_fn)(const struct lockdep_map map);
	92	+
88	93	/*
89	94	* The lock-class itself. The order of the structure members matters.
90	95	* reinit_class() zeroes the key member and all subsequent members.
@@ -110,6 +115,9 @@ struct lock_class { · 19 unmodified lines
110	115	struct list_head locks_after, locks_before;
111	116
112	117	const struct lockdep_subclass_key *key;
	118	+ lock_cmp_fn cmp_fn;
	119	+ lock_print_fn print_fn;
	120	+
113	121	unsigned int subclass;
114	122	unsigned int dep_gen_id;
115	123

include/linux/percpu-defs.h+13−32

@@ -343,31 +343,19 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } · 342 unmodified lines
343	343	pscr2_ret__; \
344	344	})
345	345
346		−/*
347		− * Special handling for cmpxchg_double. cmpxchg_double is passed two
348		− * percpu variables. The first has to be aligned to a double word
349		− * boundary and the second has to follow directly thereafter.
350		− * We enforce this on all architectures even if they don't support
351		− * a double cmpxchg instruction, since it's a cheap requirement, and it
352		− * avoids breaking the requirement for architectures with the instruction.
353		− */
354		−#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
	346	+#define __pcpu_size_call_return2bool(stem, variable, ...) \
355	347	({ \
356		− bool pdcrb_ret__; \
357		− __verify_pcpu_ptr(&(pcp1)); \
358		− BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
359		− VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
360		− VM_BUG_ON((unsigned long)(&(pcp2)) != \
361		− (unsigned long)(&(pcp1)) + sizeof(pcp1)); \
362		− switch(sizeof(pcp1)) { \
363		− case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
364		− case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
365		− case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
366		− case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
	348	+ bool pscr2_ret__; \
	349	+ __verify_pcpu_ptr(&(variable)); \
	350	+ switch(sizeof(variable)) { \
	351	+ case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
	352	+ case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
	353	+ case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
	354	+ case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
367	355	default: \
368	356	__bad_size_call_parameter(); break; \
369	357	} \
370		− pdcrb_ret__; \
	358	+ pscr2_ret__; \
371	359	})
372	360
373	361	#define __pcpu_size_call(stem, variable, ...) \
@@ -426,9 +414,8 @@ do { \ · 52 unmodified lines
426	414	#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
427	415	#define raw_cpu_cmpxchg(pcp, oval, nval) \
428	416	__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
429		−#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
430		− __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
431		−
	417	+#define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \
	418	+ __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)
432	419	#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
433	420	#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
434	421	#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
@@ -488,11 +475,6 @@ do { \ · 53 unmodified lines
488	475	raw_cpu_cmpxchg(pcp, oval, nval); \
489	476	})
490	477
491		−#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
492		−({ __this_cpu_preempt_check("cmpxchg_double"); \
493		− raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
494		−})
495		−
496	478	#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
497	479	#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
498	480	#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
@@ -513,9 +495,8 @@ do { \ · 14 unmodified lines
513	495	#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
514	496	#define this_cpu_cmpxchg(pcp, oval, nval) \
515	497	__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
516		−#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
517		− __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
518		−
	498	+#define this_cpu_try_cmpxchg(pcp, ovalp, nval) \
	499	+ __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)
519	500	#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
520	501	#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
521	502	#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)

include/linux/slub_def.h+9−3

@@ -39,7 +39,8 @@ enum stat_item { · 38 unmodified lines
39	39	CPU_PARTIAL_FREE, /* Refill cpu partial on free */
40	40	CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */
41	41	CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */
42		− NR_SLUB_STAT_ITEMS };
	42	+ NR_SLUB_STAT_ITEMS
	43	+};
43	44
44	45	#ifndef CONFIG_SLUB_TINY
45	46	/*
@@ -47,8 +48,13 @@ enum stat_item { · 1 unmodified line
47	48	* with this_cpu_cmpxchg_double() alignment requirements.
48	49	*/
49	50	struct kmem_cache_cpu {
50		− void *freelist; / Pointer to next available object */
51		− unsigned long tid; /* Globally unique transaction id */
	51	+ union {
	52	+ struct {
	53	+ void *freelist; / Pointer to next available object */
	54	+ unsigned long tid; /* Globally unique transaction id */
	55	+ };
	56	+ freelist_aba_t freelist_tid;
	57	+ };
52	58	struct slab slab; / The slab from which we are allocating */
53	59	#ifdef CONFIG_SLUB_CPU_PARTIAL
54	60	struct slab partial; / Partially allocated frozen slabs */

include/linux/types.h+5−0

@@ -10,6 +10,11 @@ · 9 unmodified lines
10	10	#define DECLARE_BITMAP(name,bits) \
11	11	unsigned long name[BITS_TO_LONGS(bits)]
12	12
	13	+#ifdef __SIZEOF_INT128__
	14	+typedef __s128 s128;
	15	+typedef __u128 u128;
	16	+#endif
	17	+
13	18	typedef u32 __kernel_dev_t;
14	19
15	20	typedef __kernel_fd_set fd_set;

include/uapi/linux/types.h+4−0

@@ -13,6 +13,10 @@ · 12 unmodified lines
13	13
14	14	#include <linux/posix_types.h>
15	15
	16	+#ifdef __SIZEOF_INT128__
	17	+typedef __signed__ __int128 __s128 __attribute__((aligned(16)));
	18	+typedef unsigned __int128 __u128 __attribute__((aligned(16)));
	19	+#endif
16	20
17	21	/*
18	22	* Below are truly Linux-specific types that should never collide with

kernel/context_tracking.c+6−6

@@ -510,7 +510,7 @@ void noinstr __ct_user_enter(enum ctx_state state) · 509 unmodified lines
510	510	* In this we case we don't care about any concurrency/ordering.
511	511	*/
512	512	if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE))
513		− arch_atomic_set(&ct->state, state);
	513	+ raw_atomic_set(&ct->state, state);
514	514	} else {
515	515	/*
516	516	* Even if context tracking is disabled on this CPU, because it's outside
@@ -527,7 +527,7 @@ void noinstr __ct_user_enter(enum ctx_state state) · 10 unmodified lines
527	527	*/
528	528	if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) {
529	529	/* Tracking for vtime only, no concurrent RCU EQS accounting */
530		− arch_atomic_set(&ct->state, state);
	530	+ raw_atomic_set(&ct->state, state);
531	531	} else {
532	532	/*
533	533	* Tracking for vtime and RCU EQS. Make sure we don't race
@@ -535,7 +535,7 @@ void noinstr __ct_user_enter(enum ctx_state state) · 1 unmodified line
535	535	* RCU only requires RCU_DYNTICKS_IDX increments to be fully
536	536	* ordered.
537	537	*/
538		− arch_atomic_add(state, &ct->state);
	538	+ raw_atomic_add(state, &ct->state);
539	539	}
540	540	}
541	541	}
@@ -630,12 +630,12 @@ void noinstr __ct_user_exit(enum ctx_state state) · 88 unmodified lines
630	630	* In this we case we don't care about any concurrency/ordering.
631	631	*/
632	632	if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE))
633		− arch_atomic_set(&ct->state, CONTEXT_KERNEL);
	633	+ raw_atomic_set(&ct->state, CONTEXT_KERNEL);
634	634
635	635	} else {
636	636	if (!IS_ENABLED(CONFIG_CONTEXT_TRACKING_IDLE)) {
637	637	/* Tracking for vtime only, no concurrent RCU EQS accounting */
638		− arch_atomic_set(&ct->state, CONTEXT_KERNEL);
	638	+ raw_atomic_set(&ct->state, CONTEXT_KERNEL);
639	639	} else {
640	640	/*
641	641	* Tracking for vtime and RCU EQS. Make sure we don't race
@@ -643,7 +643,7 @@ void noinstr __ct_user_exit(enum ctx_state state) · 1 unmodified line
643	643	* RCU only requires RCU_DYNTICKS_IDX increments to be fully
644	644	* ordered.
645	645	*/
646		− arch_atomic_sub(state, &ct->state);
	646	+ raw_atomic_sub(state, &ct->state);
647	647	}
648	648	}
649	649	}

kernel/locking/lockdep.c+87−31

@@ -709,7 +709,7 @@ void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS]) · 708 unmodified lines
709	709	usage[i] = '\0';
710	710	}
711	711
712		−static void __print_lock_name(struct lock_class *class)
	712	+static void __print_lock_name(struct held_lock hlock, struct lock_class class)
713	713	{
714	714	char str[KSYM_NAME_LEN];
715	715	const char *name;
@@ -724,17 +724,19 @@ static void __print_lock_name(struct lock_class *class) · 8 unmodified lines
724	724	printk(KERN_CONT "#%d", class->name_version);
725	725	if (class->subclass)
726	726	printk(KERN_CONT "/%d", class->subclass);
	727	+ if (hlock && class->print_fn)
	728	+ class->print_fn(hlock->instance);
727	729	}
728	730	}
729	731
730		−static void print_lock_name(struct lock_class *class)
	732	+static void print_lock_name(struct held_lock hlock, struct lock_class class)
731	733	{
732	734	char usage[LOCK_USAGE_CHARS];
733	735
734	736	get_usage_chars(class, usage);
735	737
736	738	printk(KERN_CONT " (");
737		− __print_lock_name(class);
	739	+ __print_lock_name(hlock, class);
738	740	printk(KERN_CONT "){%s}-{%d:%d}", usage,
739	741	class->wait_type_outer ?: class->wait_type_inner,
740	742	class->wait_type_inner);
@@ -772,7 +774,7 @@ static void print_lock(struct held_lock *hlock) · 31 unmodified lines
772	774	}
773	775
774	776	printk(KERN_CONT "%px", hlock->instance);
775		− print_lock_name(lock);
	777	+ print_lock_name(hlock, lock);
776	778	printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);
777	779	}
778	780
@@ -1868,7 +1870,7 @@ print_circular_bug_entry(struct lock_list *target, int depth) · 1089 unmodified lines
1868	1870	if (debug_locks_silent)
1869	1871	return;
1870	1872	printk("\n-> #%u", depth);
1871		− print_lock_name(target->class);
	1873	+ print_lock_name(NULL, target->class);
1872	1874	printk(KERN_CONT ":\n");
1873	1875	print_lock_trace(target->trace, 6);
1874	1876	}
@@ -1899,11 +1901,11 @@ print_circular_lock_scenario(struct held_lock *src, · 24 unmodified lines
1899	1901	*/
1900	1902	if (parent != source) {
1901	1903	printk("Chain exists of:\n ");
1902		− __print_lock_name(source);
	1904	+ __print_lock_name(src, source);
1903	1905	printk(KERN_CONT " --> ");
1904		− __print_lock_name(parent);
	1906	+ __print_lock_name(NULL, parent);
1905	1907	printk(KERN_CONT " --> ");
1906		− __print_lock_name(target);
	1908	+ __print_lock_name(tgt, target);
1907	1909	printk(KERN_CONT "\n\n");
1908	1910	}
1909	1911
@@ -1914,13 +1916,13 @@ print_circular_lock_scenario(struct held_lock *src, · 4 unmodified lines
1914	1916	printk(" rlock(");
1915	1917	else
1916	1918	printk(" lock(");
1917		− __print_lock_name(target);
	1919	+ __print_lock_name(tgt, target);
1918	1920	printk(KERN_CONT ");\n");
1919	1921	printk(" lock(");
1920		− __print_lock_name(parent);
	1922	+ __print_lock_name(NULL, parent);
1921	1923	printk(KERN_CONT ");\n");
1922	1924	printk(" lock(");
1923		− __print_lock_name(target);
	1925	+ __print_lock_name(tgt, target);
1924	1926	printk(KERN_CONT ");\n");
1925	1927	if (src_read != 0)
1926	1928	printk(" rlock(");
@@ -1928,7 +1930,7 @@ print_circular_lock_scenario(struct held_lock *src, · 1 unmodified line
1928	1930	printk(" sync(");
1929	1931	else
1930	1932	printk(" lock(");
1931		− __print_lock_name(source);
	1933	+ __print_lock_name(src, source);
1932	1934	printk(KERN_CONT ");\n");
1933	1935	printk("\n * DEADLOCK *\n\n");
1934	1936	}
@@ -2154,6 +2156,8 @@ check_path(struct held_lock target, struct lock_list src_entry, · 219 unmodified lines
2154	2156	return ret;
2155	2157	}
2156	2158
	2159	+static void print_deadlock_bug(struct task_struct , struct held_lock , struct held_lock *);
	2160	+
2157	2161	/*
2158	2162	* Prove that the dependency graph starting at <src> can not
2159	2163	* lead to <target>. If it can, there is a circle when adding
@@ -2185,7 +2189,10 @@ check_noncircular(struct held_lock src, struct held_lock target, · 25 unmodified lines
2185	2189	*trace = save_trace();
2186	2190	}
2187	2191
2188		− print_circular_bug(&src_entry, target_entry, src, target);
	2192	+ if (src->class_idx == target->class_idx)
	2193	+ print_deadlock_bug(current, src, target);
	2194	+ else
	2195	+ print_circular_bug(&src_entry, target_entry, src, target);
2189	2196	}
2190	2197
2191	2198	return ret;
@@ -2346,7 +2353,7 @@ static void print_lock_class_header(struct lock_class *class, int depth) · 154 unmodified lines
2346	2353	int bit;
2347	2354
2348	2355	printk("%*s->", depth, "");
2349		− print_lock_name(class);
	2356	+ print_lock_name(NULL, class);
2350	2357	#ifdef CONFIG_DEBUG_LOCKDEP
2351	2358	printk(KERN_CONT " ops: %lu", debug_class_ops_read(class));
2352	2359	#endif
@@ -2528,11 +2535,11 @@ print_irq_lock_scenario(struct lock_list *safe_entry, · 175 unmodified lines
2528	2535	*/
2529	2536	if (middle_class != unsafe_class) {
2530	2537	printk("Chain exists of:\n ");
2531		− __print_lock_name(safe_class);
	2538	+ __print_lock_name(NULL, safe_class);
2532	2539	printk(KERN_CONT " --> ");
2533		− __print_lock_name(middle_class);
	2540	+ __print_lock_name(NULL, middle_class);
2534	2541	printk(KERN_CONT " --> ");
2535		− __print_lock_name(unsafe_class);
	2542	+ __print_lock_name(NULL, unsafe_class);
2536	2543	printk(KERN_CONT "\n\n");
2537	2544	}
2538	2545
@@ -2540,18 +2547,18 @@ print_irq_lock_scenario(struct lock_list *safe_entry, · 1 unmodified line
2540	2547	printk(" CPU0 CPU1\n");
2541	2548	printk(" ---- ----\n");
2542	2549	printk(" lock(");
2543		− __print_lock_name(unsafe_class);
	2550	+ __print_lock_name(NULL, unsafe_class);
2544	2551	printk(KERN_CONT ");\n");
2545	2552	printk(" local_irq_disable();\n");
2546	2553	printk(" lock(");
2547		− __print_lock_name(safe_class);
	2554	+ __print_lock_name(NULL, safe_class);
2548	2555	printk(KERN_CONT ");\n");
2549	2556	printk(" lock(");
2550		− __print_lock_name(middle_class);
	2557	+ __print_lock_name(NULL, middle_class);
2551	2558	printk(KERN_CONT ");\n");
2552	2559	printk(" <Interrupt>\n");
2553	2560	printk(" lock(");
2554		− __print_lock_name(safe_class);
	2561	+ __print_lock_name(NULL, safe_class);
2555	2562	printk(KERN_CONT ");\n");
2556	2563	printk("\n * DEADLOCK *\n\n");
2557	2564	}
@@ -2588,20 +2595,20 @@ print_bad_irq_dependency(struct task_struct *curr, · 30 unmodified lines
2588	2595	pr_warn("\nand this task is already holding:\n");
2589	2596	print_lock(prev);
2590	2597	pr_warn("which would create a new lock dependency:\n");
2591		− print_lock_name(hlock_class(prev));
	2598	+ print_lock_name(prev, hlock_class(prev));
2592	2599	pr_cont(" ->");
2593		− print_lock_name(hlock_class(next));
	2600	+ print_lock_name(next, hlock_class(next));
2594	2601	pr_cont("\n");
2595	2602
2596	2603	pr_warn("\nbut this new dependency connects a %s-irq-safe lock:\n",
2597	2604	irqclass);
2598		− print_lock_name(backwards_entry->class);
	2605	+ print_lock_name(NULL, backwards_entry->class);
2599	2606	pr_warn("\n... which became %s-irq-safe at:\n", irqclass);
2600	2607
2601	2608	print_lock_trace(backwards_entry->class->usage_traces[bit1], 1);
2602	2609
2603	2610	pr_warn("\nto a %s-irq-unsafe lock:\n", irqclass);
2604		− print_lock_name(forwards_entry->class);
	2611	+ print_lock_name(NULL, forwards_entry->class);
2605	2612	pr_warn("\n... which became %s-irq-unsafe at:\n", irqclass);
2606	2613	pr_warn("...");
2607	2614
@@ -2971,10 +2978,10 @@ print_deadlock_scenario(struct held_lock nxt, struct held_lock prv) · 363 unmodified lines
2971	2978	printk(" CPU0\n");
2972	2979	printk(" ----\n");
2973	2980	printk(" lock(");
2974		− __print_lock_name(prev);
	2981	+ __print_lock_name(prv, prev);
2975	2982	printk(KERN_CONT ");\n");
2976	2983	printk(" lock(");
2977		− __print_lock_name(next);
	2984	+ __print_lock_name(nxt, next);
2978	2985	printk(KERN_CONT ");\n");
2979	2986	printk("\n * DEADLOCK *\n\n");
2980	2987	printk(" May be due to missing lock nesting notation\n\n");
@@ -2984,6 +2991,8 @@ static void · 3 unmodified lines
2984	2991	print_deadlock_bug(struct task_struct curr, struct held_lock prev,
2985	2992	struct held_lock *next)
2986	2993	{
	2994	+ struct lock_class *class = hlock_class(prev);
	2995	+
2987	2996	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)
2988	2997	return;
2989	2998
@@ -2998,6 +3007,11 @@ print_deadlock_bug(struct task_struct curr, struct held_lock prev, · 8 unmodified lines
2998	3007	pr_warn("\nbut task is already holding lock:\n");
2999	3008	print_lock(prev);
3000	3009
	3010	+ if (class->cmp_fn) {
	3011	+ pr_warn("and the lock comparison function returns %i:\n",
	3012	+ class->cmp_fn(prev->instance, next->instance));
	3013	+ }
	3014	+
3001	3015	pr_warn("\nother info that might help us debug this:\n");
3002	3016	print_deadlock_scenario(next, prev);
3003	3017	lockdep_print_held_locks(curr);
@@ -3019,6 +3033,7 @@ print_deadlock_bug(struct task_struct curr, struct held_lock prev, · 15 unmodified lines
3019	3033	static int
3020	3034	check_deadlock(struct task_struct curr, struct held_lock next)
3021	3035	{
	3036	+ struct lock_class *class;
3022	3037	struct held_lock *prev;
3023	3038	struct held_lock *nest = NULL;
3024	3039	int i;
@@ -3039,6 +3054,12 @@ check_deadlock(struct task_struct curr, struct held_lock next) · 14 unmodified lines
3039	3054	if ((next->read == 2) && prev->read)
3040	3055	continue;
3041	3056
	3057	+ class = hlock_class(prev);
	3058	+
	3059	+ if (class->cmp_fn &&
	3060	+ class->cmp_fn(prev->instance, next->instance) < 0)
	3061	+ continue;
	3062	+
3042	3063	/*
3043	3064	* We're holding the nest_lock, which serializes this lock's
3044	3065	* nesting behaviour.
@@ -3100,6 +3121,14 @@ check_prev_add(struct task_struct curr, struct held_lock prev, · 55 unmodified lines
3100	3121	return 2;
3101	3122	}
3102	3123
	3124	+ if (prev->class_idx == next->class_idx) {
	3125	+ struct lock_class *class = hlock_class(prev);
	3126	+
	3127	+ if (class->cmp_fn &&
	3128	+ class->cmp_fn(prev->instance, next->instance) < 0)
	3129	+ return 2;
	3130	+ }
	3131	+
3103	3132	/*
3104	3133	* Prove that the new <prev> -> <next> dependency would not
3105	3134	* create a circular dependency in the graph. (We do this by
@@ -3576,7 +3605,7 @@ static void print_chain_keys_chain(struct lock_chain *chain) · 470 unmodified lines
3576	3605	hlock_id = chain_hlocks[chain->base + i];
3577	3606	chain_key = print_chain_key_iteration(hlock_id, chain_key);
3578	3607
3579		− print_lock_name(lock_classes + chain_hlock_class_idx(hlock_id));
	3608	+ print_lock_name(NULL, lock_classes + chain_hlock_class_idx(hlock_id));
3580	3609	printk("\n");
3581	3610	}
3582	3611	}
@@ -3933,11 +3962,11 @@ static void print_usage_bug_scenario(struct held_lock *lock) · 350 unmodified lines
3933	3962	printk(" CPU0\n");
3934	3963	printk(" ----\n");
3935	3964	printk(" lock(");
3936		− __print_lock_name(class);
	3965	+ __print_lock_name(lock, class);
3937	3966	printk(KERN_CONT ");\n");
3938	3967	printk(" <Interrupt>\n");
3939	3968	printk(" lock(");
3940		− __print_lock_name(class);
	3969	+ __print_lock_name(lock, class);
3941	3970	printk(KERN_CONT ");\n");
3942	3971	printk("\n * DEADLOCK *\n\n");
3943	3972	}
@@ -4023,7 +4052,7 @@ print_irq_inversion_bug(struct task_struct *curr, · 79 unmodified lines
4023	4052	pr_warn("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
4024	4053	else
4025	4054	pr_warn("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
4026		− print_lock_name(other->class);
	4055	+ print_lock_name(NULL, other->class);
4027	4056	pr_warn("\n\nand interrupts could create inverse lock ordering between them.\n\n");
4028	4057
4029	4058	pr_warn("\nother info that might help us debug this:\n");
@@ -4896,6 +4925,33 @@ EXPORT_SYMBOL_GPL(lockdep_init_map_type); · 866 unmodified lines
4896	4925	struct lock_class_key __lockdep_no_validate__;
4897	4926	EXPORT_SYMBOL_GPL(__lockdep_no_validate__);
4898	4927
	4928	+#ifdef CONFIG_PROVE_LOCKING
	4929	+void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn,
	4930	+ lock_print_fn print_fn)
	4931	+{
	4932	+ struct lock_class *class = lock->class_cache[0];
	4933	+ unsigned long flags;
	4934	+
	4935	+ raw_local_irq_save(flags);
	4936	+ lockdep_recursion_inc();
	4937	+
	4938	+ if (!class)
	4939	+ class = register_lock_class(lock, 0, 0);
	4940	+
	4941	+ if (class) {
	4942	+ WARN_ON(class->cmp_fn && class->cmp_fn != cmp_fn);
	4943	+ WARN_ON(class->print_fn && class->print_fn != print_fn);
	4944	+
	4945	+ class->cmp_fn = cmp_fn;
	4946	+ class->print_fn = print_fn;
	4947	+ }
	4948	+
	4949	+ lockdep_recursion_finish();
	4950	+ raw_local_irq_restore(flags);
	4951	+}
	4952	+EXPORT_SYMBOL_GPL(lockdep_set_lock_cmp_fn);
	4953	+#endif
	4954	+
4899	4955	static void
4900	4956	print_lock_nested_lock_not_held(struct task_struct *curr,
4901	4957	struct held_lock *hlock)

kernel/sched/clock.c+1−1

@@ -287,7 +287,7 @@ again: · 286 unmodified lines
287	287	clock = wrap_max(clock, min_clock);
288	288	clock = wrap_min(clock, max_clock);
289	289
290		− if (!arch_try_cmpxchg64(&scd->clock, &old_clock, clock))
	290	+ if (!raw_try_cmpxchg64(&scd->clock, &old_clock, clock))
291	291	goto again;
292	292
293	293	return clock;

lib/crypto/curve25519-hacl64.c+0−2

@@ -14,8 +14,6 @@ · 13 unmodified lines
14	14	#include <crypto/curve25519.h>
15	15	#include <linux/string.h>
16	16
17		−typedef __uint128_t u128;
18		−
19	17	static __always_inline u64 u64_eq_mask(u64 a, u64 b)
20	18	{
21	19	u64 x = a ^ b;

lib/crypto/poly1305-donna64.c+0−2

@@ -10,8 +10,6 @@ · 9 unmodified lines
10	10	#include <asm/unaligned.h>
11	11	#include <crypto/internal/poly1305.h>
12	12
13		−typedef __uint128_t u128;
14		−
15	13	void poly1305_core_setkey(struct poly1305_core_key *key,
16	14	const u8 raw_key[POLY1305_BLOCK_SIZE])
17	15	{

mm/slab.h+46−7

@@ -6,6 +6,38 @@ · 5 unmodified lines
6	6	*/
7	7	void __init kmem_cache_init(void);
8	8
	9	+#ifdef CONFIG_64BIT
	10	+# ifdef system_has_cmpxchg128
	11	+# define system_has_freelist_aba() system_has_cmpxchg128()
	12	+# define try_cmpxchg_freelist try_cmpxchg128
	13	+# endif
	14	+#define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg128
	15	+typedef u128 freelist_full_t;
	16	+#else /* CONFIG_64BIT */
	17	+# ifdef system_has_cmpxchg64
	18	+# define system_has_freelist_aba() system_has_cmpxchg64()
	19	+# define try_cmpxchg_freelist try_cmpxchg64
	20	+# endif
	21	+#define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg64
	22	+typedef u64 freelist_full_t;
	23	+#endif /* CONFIG_64BIT */
	24	+
	25	+#if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
	26	+#undef system_has_freelist_aba
	27	+#endif
	28	+
	29	+/*
	30	+ * Freelist pointer and counter to cmpxchg together, avoids the typical ABA
	31	+ * problems with cmpxchg of just a pointer.
	32	+ */
	33	+typedef union {
	34	+ struct {
	35	+ void *freelist;
	36	+ unsigned long counter;
	37	+ };
	38	+ freelist_full_t full;
	39	+} freelist_aba_t;
	40	+
9	41	/* Reuses the bits in struct page */
10	42	struct slab {
11	43	unsigned long __page_flags;
@@ -38,14 +70,21 @@ struct slab { · 26 unmodified lines
38	70	#endif
39	71	};
40	72	/* Double-word boundary */
41		− void freelist; / first free object */
42	73	union {
43		− unsigned long counters;
44	74	struct {
45		− unsigned inuse:16;
46		− unsigned objects:15;
47		− unsigned frozen:1;
	75	+ void freelist; / first free object */
	76	+ union {
	77	+ unsigned long counters;
	78	+ struct {
	79	+ unsigned inuse:16;
	80	+ unsigned objects:15;
	81	+ unsigned frozen:1;
	82	+ };
	83	+ };
48	84	};
	85	+#ifdef system_has_freelist_aba
	86	+ freelist_aba_t freelist_counter;
	87	+#endif
49	88	};
50	89	};
51	90	struct rcu_head rcu_head;
@@ -72,8 +111,8 @@ SLAB_MATCH(memcg_data, memcg_data); · 20 unmodified lines
72	111	#endif
73	112	#undef SLAB_MATCH
74	113	static_assert(sizeof(struct slab) <= sizeof(struct page));
75		−#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && defined(CONFIG_SLUB)
76		−static_assert(IS_ALIGNED(offsetof(struct slab, freelist), 2sizeof(void )));
	114	+#if defined(system_has_freelist_aba) && defined(CONFIG_SLUB)
	115	+static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t)));
77	116	#endif
78	117
79	118	/**

mm/slub.c+82−57

@@ -292,7 +292,12 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) · 291 unmodified lines
292	292	/* Poison object */
293	293	#define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)
294	294	/* Use cmpxchg_double */
	295	+
	296	+#ifdef system_has_freelist_aba
295	297	#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
	298	+#else
	299	+#define __CMPXCHG_DOUBLE ((slab_flags_t __force)0U)
	300	+#endif
296	301
297	302	/*
298	303	* Tracking user of a slab.
@@ -512,6 +517,40 @@ static __always_inline void slab_unlock(struct slab *slab) · 213 unmodified lines
512	517	__bit_spin_unlock(PG_locked, &page->flags);
513	518	}
514	519
	520	+static inline bool
	521	+__update_freelist_fast(struct slab *slab,
	522	+ void *freelist_old, unsigned long counters_old,
	523	+ void *freelist_new, unsigned long counters_new)
	524	+{
	525	+#ifdef system_has_freelist_aba
	526	+ freelist_aba_t old = { .freelist = freelist_old, .counter = counters_old };
	527	+ freelist_aba_t new = { .freelist = freelist_new, .counter = counters_new };
	528	+
	529	+ return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full);
	530	+#else
	531	+ return false;
	532	+#endif
	533	+}
	534	+
	535	+static inline bool
	536	+__update_freelist_slow(struct slab *slab,
	537	+ void *freelist_old, unsigned long counters_old,
	538	+ void *freelist_new, unsigned long counters_new)
	539	+{
	540	+ bool ret = false;
	541	+
	542	+ slab_lock(slab);
	543	+ if (slab->freelist == freelist_old &&
	544	+ slab->counters == counters_old) {
	545	+ slab->freelist = freelist_new;
	546	+ slab->counters = counters_new;
	547	+ ret = true;
	548	+ }
	549	+ slab_unlock(slab);
	550	+
	551	+ return ret;
	552	+}
	553	+
515	554	/*
516	555	* Interrupts must be disabled (for the fallback code to work right), typically
517	556	* by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is
@@ -519,33 +558,25 @@ static __always_inline void slab_unlock(struct slab *slab) · 1 unmodified line
519	558	* allocation/ free operation in hardirq context. Therefore nothing can
520	559	* interrupt the operation.
521	560	*/
522		−static inline bool __cmpxchg_double_slab(struct kmem_cache s, struct slab slab,
	561	+static inline bool __slab_update_freelist(struct kmem_cache s, struct slab slab,
523	562	void *freelist_old, unsigned long counters_old,
524	563	void *freelist_new, unsigned long counters_new,
525	564	const char *n)
526	565	{
	566	+ bool ret;
	567	+
527	568	if (USE_LOCKLESS_FAST_PATH())
528	569	lockdep_assert_irqs_disabled();
529		−#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
530		− defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
	570	+
531	571	if (s->flags & __CMPXCHG_DOUBLE) {
532		− if (cmpxchg_double(&slab->freelist, &slab->counters,
533		− freelist_old, counters_old,
534		− freelist_new, counters_new))
535		− return true;
536		− } else
537		−#endif
538		− {
539		− slab_lock(slab);
540		− if (slab->freelist == freelist_old &&
541		− slab->counters == counters_old) {
542		− slab->freelist = freelist_new;
543		− slab->counters = counters_new;
544		− slab_unlock(slab);
545		− return true;
546		− }
547		− slab_unlock(slab);
	572	+ ret = __update_freelist_fast(slab, freelist_old, counters_old,
	573	+ freelist_new, counters_new);
	574	+ } else {
	575	+ ret = __update_freelist_slow(slab, freelist_old, counters_old,
	576	+ freelist_new, counters_new);
548	577	}
	578	+ if (likely(ret))
	579	+ return true;
549	580
550	581	cpu_relax();
551	582	stat(s, CMPXCHG_DOUBLE_FAIL);
@@ -557,36 +588,26 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache s, struct slab slab · 5 unmodified lines
557	588	return false;
558	589	}
559	590
560		−static inline bool cmpxchg_double_slab(struct kmem_cache s, struct slab slab,
	591	+static inline bool slab_update_freelist(struct kmem_cache s, struct slab slab,
561	592	void *freelist_old, unsigned long counters_old,
562	593	void *freelist_new, unsigned long counters_new,
563	594	const char *n)
564	595	{
565		−#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
566		− defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
	596	+ bool ret;
	597	+
567	598	if (s->flags & __CMPXCHG_DOUBLE) {
568		− if (cmpxchg_double(&slab->freelist, &slab->counters,
569		− freelist_old, counters_old,
570		− freelist_new, counters_new))
571		− return true;
572		− } else
573		−#endif
574		− {
	599	+ ret = __update_freelist_fast(slab, freelist_old, counters_old,
	600	+ freelist_new, counters_new);
	601	+ } else {
575	602	unsigned long flags;
576	603
577	604	local_irq_save(flags);
578		− slab_lock(slab);
579		− if (slab->freelist == freelist_old &&
580		− slab->counters == counters_old) {
581		− slab->freelist = freelist_new;
582		− slab->counters = counters_new;
583		− slab_unlock(slab);
584		− local_irq_restore(flags);
585		− return true;
586		− }
587		− slab_unlock(slab);
	605	+ ret = __update_freelist_slow(slab, freelist_old, counters_old,
	606	+ freelist_new, counters_new);
588	607	local_irq_restore(flags);
589	608	}
	609	+ if (likely(ret))
	610	+ return true;
590	611
591	612	cpu_relax();
592	613	stat(s, CMPXCHG_DOUBLE_FAIL);
@@ -2228,7 +2249,7 @@ static inline void acquire_slab(struct kmem_cache s, · 1635 unmodified lines
2228	2249	VM_BUG_ON(new.frozen);
2229	2250	new.frozen = 1;
2230	2251
2231		− if (!__cmpxchg_double_slab(s, slab,
	2252	+ if (!__slab_update_freelist(s, slab,
2232	2253	freelist, counters,
2233	2254	new.freelist, new.counters,
2234	2255	"acquire_slab"))
@@ -2554,7 +2575,7 @@ redo: · 319 unmodified lines
2554	2575	}
2555	2576
2556	2577
2557		− if (!cmpxchg_double_slab(s, slab,
	2578	+ if (!slab_update_freelist(s, slab,
2558	2579	old.freelist, old.counters,
2559	2580	new.freelist, new.counters,
2560	2581	"unfreezing slab")) {
@@ -2611,7 +2632,7 @@ static void __unfreeze_partials(struct kmem_cache s, struct slab partial_slab) · 50 unmodified lines
2611	2632
2612	2633	new.frozen = 0;
2613	2634
2614		− } while (!__cmpxchg_double_slab(s, slab,
	2635	+ } while (!__slab_update_freelist(s, slab,
2615	2636	old.freelist, old.counters,
2616	2637	new.freelist, new.counters,
2617	2638	"unfreezing slab"));
@@ -3008,6 +3029,18 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) · 390 unmodified lines
3008	3029	}
3009	3030
3010	3031	#ifndef CONFIG_SLUB_TINY
	3032	+static inline bool
	3033	+__update_cpu_freelist_fast(struct kmem_cache *s,
	3034	+ void freelist_old, void freelist_new,
	3035	+ unsigned long tid)
	3036	+{
	3037	+ freelist_aba_t old = { .freelist = freelist_old, .counter = tid };
	3038	+ freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };
	3039	+
	3040	+ return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
	3041	+ &old.full, new.full);
	3042	+}
	3043	+
3011	3044	/*
3012	3045	* Check the slab->freelist and either transfer the freelist to the
3013	3046	* per cpu freelist or deactivate the slab.
@@ -3034,7 +3067,7 @@ static inline void get_freelist(struct kmem_cache s, struct slab *slab) · 20 unmodified lines
3034	3067	new.inuse = slab->objects;
3035	3068	new.frozen = freelist != NULL;
3036	3069
3037		− } while (!__cmpxchg_double_slab(s, slab,
	3070	+ } while (!__slab_update_freelist(s, slab,
3038	3071	freelist, counters,
3039	3072	NULL, new.counters,
3040	3073	"get_freelist"));
@@ -3359,11 +3392,7 @@ redo: · 318 unmodified lines
3359	3392	* against code executing on this cpu not from access by
3360	3393	* other cpus.
3361	3394	*/
3362		− if (unlikely(!this_cpu_cmpxchg_double(
3363		− s->cpu_slab->freelist, s->cpu_slab->tid,
3364		− object, tid,
3365		− next_object, next_tid(tid)))) {
3366		−
	3395	+ if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) {
3367	3396	note_cmpxchg_failure("slab_alloc", s, tid);
3368	3397	goto redo;
3369	3398	}
@@ -3631,7 +3660,7 @@ static void __slab_free(struct kmem_cache s, struct slab slab, · 261 unmodified lines
3631	3660	}
3632	3661	}
3633	3662
3634		− } while (!cmpxchg_double_slab(s, slab,
	3663	+ } while (!slab_update_freelist(s, slab,
3635	3664	prior, counters,
3636	3665	head, new.counters,
3637	3666	"__slab_free"));
@@ -3736,11 +3765,7 @@ redo: · 98 unmodified lines
3736	3765
3737	3766	set_freepointer(s, tail_obj, freelist);
3738	3767
3739		− if (unlikely(!this_cpu_cmpxchg_double(
3740		− s->cpu_slab->freelist, s->cpu_slab->tid,
3741		− freelist, tid,
3742		− head, next_tid(tid)))) {
3743		−
	3768	+ if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {
3744	3769	note_cmpxchg_failure("slab_free", s, tid);
3745	3770	goto redo;
3746	3771	}
@@ -4505,11 +4530,11 @@ static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags) · 758 unmodified lines
4505	4530	}
4506	4531	}
4507	4532
4508		−#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
4509		− defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
4510		− if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
	4533	+#ifdef system_has_freelist_aba
	4534	+ if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) {
4511	4535	/* Enable fast mode */
4512	4536	s->flags \|= __CMPXCHG_DOUBLE;
	4537	+ }
4513	4538	#endif
4514	4539
4515	4540	/*

scripts/atomic/atomic-tbl.sh+108−4

@@ -36,9 +36,16 @@ meta_has_relaxed() · 35 unmodified lines
36	36	meta_in "$1" "BFIR"
37	37	}
38	38
39		−#find_fallback_template(pfx, name, sfx, order)
40		−find_fallback_template()
	39	+#meta_is_implicitly_relaxed(meta)
	40	+meta_is_implicitly_relaxed()
41	41	{
	42	+ meta_in "$1" "vls"
	43	+}
	44	+
	45	+#find_template(tmpltype, pfx, name, sfx, order)
	46	+find_template()
	47	+{
	48	+ local tmpltype="$1"; shift
42	49	local pfx="$1"; shift
43	50	local name="$1"; shift
44	51	local sfx="$1"; shift
@@ -52,8 +59,8 @@ find_fallback_template() · 7 unmodified lines
52	59	#
53	60	# Start at the most specific, and fall back to the most general. Once
54	61	# we find a specific fallback, don't bother looking for more.
55		− for base in "${pfx}${name}${sfx}${order}" "${name}"; do
56		− file="${ATOMICDIR}/fallbacks/${base}"
	62	+ for base in "${pfx}${name}${sfx}${order}" "${pfx}${name}${sfx}" "${name}"; do
	63	+ file="${ATOMICDIR}/${tmpltype}/${base}"
57	64
58	65	if [ -f "${file}" ]; then
59	66	printf "${file}"
@@ -62,6 +69,18 @@ find_fallback_template() · 2 unmodified lines
62	69	done
63	70	}
64	71
	72	+#find_fallback_template(pfx, name, sfx, order)
	73	+find_fallback_template()
	74	+{
	75	+ find_template "fallbacks" "$@"
	76	+}
	77	+
	78	+#find_kerneldoc_template(pfx, name, sfx, order)
	79	+find_kerneldoc_template()
	80	+{
	81	+ find_template "kerneldoc" "$@"
	82	+}
	83	+
65	84	#gen_ret_type(meta, int)
66	85	gen_ret_type() {
67	86	local meta="$1"; shift
@@ -142,6 +161,91 @@ gen_args() · 74 unmodified lines
142	161	done
143	162	}
144	163
	164	+#gen_desc_return(meta)
	165	+gen_desc_return()
	166	+{
	167	+ local meta="$1"; shift
	168	+
	169	+ case "${meta}" in
	170	+ [v])
	171	+ printf "Return: Nothing."
	172	+ ;;
	173	+ [Ff])
	174	+ printf "Return: The original value of @v."
	175	+ ;;
	176	+ [R])
	177	+ printf "Return: The updated value of @v."
	178	+ ;;
	179	+ [l])
	180	+ printf "Return: The value of @v."
	181	+ ;;
	182	+ esac
	183	+}
	184	+
	185	+#gen_template_kerneldoc(template, class, meta, pfx, name, sfx, order, atomic, int, args...)
	186	+gen_template_kerneldoc()
	187	+{
	188	+ local template="$1"; shift
	189	+ local class="$1"; shift
	190	+ local meta="$1"; shift
	191	+ local pfx="$1"; shift
	192	+ local name="$1"; shift
	193	+ local sfx="$1"; shift
	194	+ local order="$1"; shift
	195	+ local atomic="$1"; shift
	196	+ local int="$1"; shift
	197	+
	198	+ local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
	199	+
	200	+ local ret="$(gen_ret_type "${meta}" "${int}")"
	201	+ local retstmt="$(gen_ret_stmt "${meta}")"
	202	+ local params="$(gen_params "${int}" "${atomic}" "$@")"
	203	+ local args="$(gen_args "$@")"
	204	+ local desc_order=""
	205	+ local desc_instrumentation=""
	206	+ local desc_return=""
	207	+
	208	+ if [ ! -z "${order}" ]; then
	209	+ desc_order="${order##_}"
	210	+ elif meta_is_implicitly_relaxed "${meta}"; then
	211	+ desc_order="relaxed"
	212	+ else
	213	+ desc_order="full"
	214	+ fi
	215	+
	216	+ if [ -z "${class}" ]; then
	217	+ desc_noinstr="Unsafe to use in noinstr code; use raw_${atomicname}() there."
	218	+ else
	219	+ desc_noinstr="Safe to use in noinstr code; prefer ${atomicname}() elsewhere."
	220	+ fi
	221	+
	222	+ desc_return="$(gen_desc_return "${meta}")"
	223	+
	224	+ . ${template}
	225	+}
	226	+
	227	+#gen_kerneldoc(class, meta, pfx, name, sfx, order, atomic, int, args...)
	228	+gen_kerneldoc()
	229	+{
	230	+ local class="$1"; shift
	231	+ local meta="$1"; shift
	232	+ local pfx="$1"; shift
	233	+ local name="$1"; shift
	234	+ local sfx="$1"; shift
	235	+ local order="$1"; shift
	236	+
	237	+ local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
	238	+
	239	+ local tmpl="$(find_kerneldoc_template "${pfx}" "${name}" "${sfx}" "${order}")"
	240	+ if [ -z "${tmpl}" ]; then
	241	+ printf "/*\n"
	242	+ printf " * No kerneldoc available for ${class}${atomicname}\n"
	243	+ printf " */\n"
	244	+ else
	245	+ gen_template_kerneldoc "${tmpl}" "${class}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
	246	+ fi
	247	+}
	248	+
145	249	#gen_proto_order_variants(meta, pfx, name, sfx, ...)
146	250	gen_proto_order_variants()
147	251	{

scripts/atomic/atomics.tbl+1−1

@@ -27,7 +27,7 @@ and vF i v · 26 unmodified lines
27	27	andnot vF i v
28	28	or vF i v
29	29	xor vF i v
30		−xchg I v i
	30	+xchg I v i:new
31	31	cmpxchg I v i:old i:new
32	32	try_cmpxchg B v p:old i:new
33	33	sub_and_test b i v

scripts/atomic/fallbacks/acquire+0−4

@@ -1,9 +1,5 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_${pfx}${name}${sfx}_acquire(${params})
4		−{
5	2	${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
6	3	__atomic_acquire_fence();
7	4	return ret;
8		−}
9	5	EOF

scripts/atomic/fallbacks/add_negative+1−13

@@ -1,15 +1,3 @@
1	1	cat <<EOF
2		−/**
3		− * arch_${atomic}_add_negative${order} - Add and test if negative
4		− * @i: integer value to add
5		− * @v: pointer of type ${atomic}_t
6		− *
7		− * Atomically adds @i to @v and returns true if the result is negative,
8		− * or false when the result is greater than or equal to zero.
9		− */
10		−static __always_inline bool
11		−arch_${atomic}_add_negative${order}(${int} i, ${atomic}_t *v)
12		−{
13		− return arch_${atomic}_add_return${order}(i, v) < 0;
14		−}
	2	+ return raw_${atomic}_add_return${order}(i, v) < 0;
15	3	EOF

scripts/atomic/fallbacks/add_unless+1−14

@@ -1,16 +1,3 @@
1	1	cat << EOF
2		−/**
3		− * arch_${atomic}_add_unless - add unless the number is already a given value
4		− * @v: pointer of type ${atomic}_t
5		− * @a: the amount to add to v...
6		− * @u: ...unless v is equal to u.
7		− *
8		− * Atomically adds @a to @v, if @v was not already @u.
9		− * Returns true if the addition was done.
10		− */
11		−static __always_inline bool
12		−arch_${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13		−{
14		− return arch_${atomic}_fetch_add_unless(v, a, u) != u;
15		−}
	2	+ return raw_${atomic}_fetch_add_unless(v, a, u) != u;
16	3	EOF

scripts/atomic/fallbacks/andnot+1−5

@@ -1,7 +1,3 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
4		−{
5		− ${retstmt}arch_${atomic}_${pfx}and${sfx}${order}(~i, v);
6		−}
	2	+ ${retstmt}raw_${atomic}_${pfx}and${sfx}${order}(~i, v);
7	3	EOF

scripts/atomic/fallbacks/cmpxchg+3−0

@@ -0,0 +1,3 @@
	1	+cat <<EOF
	2	+ return raw_cmpxchg${order}(&v->counter, old, new);
	3	+EOF

scripts/atomic/fallbacks/dec+1−5

@@ -1,7 +1,3 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
4		−{
5		− ${retstmt}arch_${atomic}_${pfx}sub${sfx}${order}(1, v);
6		−}
	2	+ ${retstmt}raw_${atomic}_${pfx}sub${sfx}${order}(1, v);
7	3	EOF

scripts/atomic/fallbacks/dec_and_test+1−13

@@ -1,15 +1,3 @@
1	1	cat <<EOF
2		−/**
3		− * arch_${atomic}_dec_and_test - decrement and test
4		− * @v: pointer of type ${atomic}_t
5		− *
6		− * Atomically decrements @v by 1 and
7		− * returns true if the result is 0, or false for all other
8		− * cases.
9		− */
10		−static __always_inline bool
11		−arch_${atomic}_dec_and_test(${atomic}_t *v)
12		−{
13		− return arch_${atomic}_dec_return(v) == 0;
14		−}
	2	+ return raw_${atomic}_dec_return(v) == 0;
15	3	EOF

scripts/atomic/fallbacks/dec_if_positive+2−6

@@ -1,15 +1,11 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_dec_if_positive(${atomic}_t *v)
4		−{
5		− ${int} dec, c = arch_${atomic}_read(v);
	2	+ ${int} dec, c = raw_${atomic}_read(v);
6	3
7	4	do {
8	5	dec = c - 1;
9	6	if (unlikely(dec < 0))
10	7	break;
11		− } while (!arch_${atomic}_try_cmpxchg(v, &c, dec));
	8	+ } while (!raw_${atomic}_try_cmpxchg(v, &c, dec));
12	9
13	10	return dec;
14		−}
15	11	EOF

scripts/atomic/fallbacks/dec_unless_positive+2−6

@@ -1,14 +1,10 @@
1	1	cat <<EOF
2		−static __always_inline bool
3		−arch_${atomic}_dec_unless_positive(${atomic}_t *v)
4		−{
5		− ${int} c = arch_${atomic}_read(v);
	2	+ ${int} c = raw_${atomic}_read(v);
6	3
7	4	do {
8	5	if (unlikely(c > 0))
9	6	return false;
10		− } while (!arch_${atomic}_try_cmpxchg(v, &c, c - 1));
	7	+ } while (!raw_${atomic}_try_cmpxchg(v, &c, c - 1));
11	8
12	9	return true;
13		−}
14	10	EOF

scripts/atomic/fallbacks/fence+0−4

@@ -1,11 +1,7 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_${pfx}${name}${sfx}(${params})
4		−{
5	2	${ret} ret;
6	3	__atomic_pre_full_fence();
7	4	ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
8	5	__atomic_post_full_fence();
9	6	return ret;
10		−}
11	7	EOF

scripts/atomic/fallbacks/fetch_add_unless+2−15

@@ -1,23 +1,10 @@
1	1	cat << EOF
2		−/**
3		− * arch_${atomic}_fetch_add_unless - add unless the number is already a given value
4		− * @v: pointer of type ${atomic}_t
5		− * @a: the amount to add to v...
6		− * @u: ...unless v is equal to u.
7		− *
8		− * Atomically adds @a to @v, so long as @v was not already @u.
9		− * Returns original value of @v
10		− */
11		−static __always_inline ${int}
12		−arch_${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13		−{
14		− ${int} c = arch_${atomic}_read(v);
	2	+ ${int} c = raw_${atomic}_read(v);
15	3
16	4	do {
17	5	if (unlikely(c == u))
18	6	break;
19		− } while (!arch_${atomic}_try_cmpxchg(v, &c, c + a));
	7	+ } while (!raw_${atomic}_try_cmpxchg(v, &c, c + a));
20	8
21	9	return c;
22		−}
23	10	EOF

scripts/atomic/fallbacks/inc+1−5

@@ -1,7 +1,3 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
4		−{
5		− ${retstmt}arch_${atomic}_${pfx}add${sfx}${order}(1, v);
6		−}
	2	+ ${retstmt}raw_${atomic}_${pfx}add${sfx}${order}(1, v);
7	3	EOF

scripts/atomic/fallbacks/inc_and_test+1−13

@@ -1,15 +1,3 @@
1	1	cat <<EOF
2		−/**
3		− * arch_${atomic}_inc_and_test - increment and test
4		− * @v: pointer of type ${atomic}_t
5		− *
6		− * Atomically increments @v by 1
7		− * and returns true if the result is zero, or false for all
8		− * other cases.
9		− */
10		−static __always_inline bool
11		−arch_${atomic}_inc_and_test(${atomic}_t *v)
12		−{
13		− return arch_${atomic}_inc_return(v) == 0;
14		−}
	2	+ return raw_${atomic}_inc_return(v) == 0;
15	3	EOF

scripts/atomic/fallbacks/inc_not_zero+1−12

@@ -1,14 +1,3 @@
1	1	cat <<EOF
2		−/**
3		− * arch_${atomic}_inc_not_zero - increment unless the number is zero
4		− * @v: pointer of type ${atomic}_t
5		− *
6		− * Atomically increments @v by 1, if @v is non-zero.
7		− * Returns true if the increment was done.
8		− */
9		−static __always_inline bool
10		−arch_${atomic}_inc_not_zero(${atomic}_t *v)
11		−{
12		− return arch_${atomic}_add_unless(v, 1, 0);
13		−}
	2	+ return raw_${atomic}_add_unless(v, 1, 0);
14	3	EOF

scripts/atomic/fallbacks/inc_unless_negative+2−6

@@ -1,14 +1,10 @@
1	1	cat <<EOF
2		−static __always_inline bool
3		−arch_${atomic}_inc_unless_negative(${atomic}_t *v)
4		−{
5		− ${int} c = arch_${atomic}_read(v);
	2	+ ${int} c = raw_${atomic}_read(v);
6	3
7	4	do {
8	5	if (unlikely(c < 0))
9	6	return false;
10		− } while (!arch_${atomic}_try_cmpxchg(v, &c, c + 1));
	7	+ } while (!raw_${atomic}_try_cmpxchg(v, &c, c + 1));
11	8
12	9	return true;
13		−}
14	10	EOF

scripts/atomic/fallbacks/read_acquire+1−5

@@ -1,16 +1,12 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_read_acquire(const ${atomic}_t *v)
4		−{
5	2	${int} ret;
6	3
7	4	if (__native_word(${atomic}_t)) {
8	5	ret = smp_load_acquire(&(v)->counter);
9	6	} else {
10		− ret = arch_${atomic}_read(v);
	7	+ ret = raw_${atomic}_read(v);
11	8	__atomic_acquire_fence();
12	9	}
13	10
14	11	return ret;
15		−}
16	12	EOF

scripts/atomic/fallbacks/release+0−4

@@ -1,8 +1,4 @@
1	1	cat <<EOF
2		−static __always_inline ${ret}
3		−arch_${atomic}_${pfx}${name}${sfx}_release(${params})
4		−{
5	2	__atomic_release_fence();
6	3	${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
7		−}
8	4	EOF

scripts/atomic/fallbacks/set_release+1−5

@@ -1,12 +1,8 @@
1	1	cat <<EOF
2		−static __always_inline void
3		−arch_${atomic}_set_release(${atomic}_t *v, ${int} i)
4		−{
5	2	if (__native_word(${atomic}_t)) {
6	3	smp_store_release(&(v)->counter, i);
7	4	} else {
8	5	__atomic_release_fence();
9		− arch_${atomic}_set(v, i);
	6	+ raw_${atomic}_set(v, i);
10	7	}
11		−}
12	8	EOF

scripts/atomic/fallbacks/sub_and_test+1−14

@@ -1,16 +1,3 @@
1	1	cat <<EOF
2		−/**
3		− * arch_${atomic}_sub_and_test - subtract value from variable and test result
4		− * @i: integer value to subtract
5		− * @v: pointer of type ${atomic}_t
6		− *
7		− * Atomically subtracts @i from @v and returns
8		− * true if the result is zero, or false for all
9		− * other cases.
10		− */
11		−static __always_inline bool
12		−arch_${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
13		−{
14		− return arch_${atomic}_sub_return(i, v) == 0;
15		−}
	2	+ return raw_${atomic}_sub_return(i, v) == 0;
16	3	EOF

scripts/atomic/fallbacks/try_cmpxchg+1−5

@@ -1,11 +1,7 @@
1	1	cat <<EOF
2		−static __always_inline bool
3		−arch_${atomic}_try_cmpxchg${order}(${atomic}_t v, ${int} old, ${int} new)
4		−{
5	2	${int} r, o = *old;
6		− r = arch_${atomic}_cmpxchg${order}(v, o, new);
	3	+ r = raw_${atomic}_cmpxchg${order}(v, o, new);
7	4	if (unlikely(r != o))
8	5	*old = r;
9	6	return likely(r == o);
10		−}
11	7	EOF

scripts/atomic/fallbacks/xchg+3−0

@@ -0,0 +1,3 @@
	1	+cat <<EOF
	2	+ return raw_xchg${order}(&v->counter, new);
	3	+EOF

scripts/atomic/gen-atomic-fallback.sh+175−91

@@ -17,23 +17,16 @@ gen_template_fallback() · 16 unmodified lines
17	17	local atomic="$1"; shift
18	18	local int="$1"; shift
19	19
20		− local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}"
21		−
22	20	local ret="$(gen_ret_type "${meta}" "${int}")"
23	21	local retstmt="$(gen_ret_stmt "${meta}")"
24	22	local params="$(gen_params "${int}" "${atomic}" "$@")"
25	23	local args="$(gen_args "$@")"
26	24
27		− if [ ! -z "${template}" ]; then
28		− printf "#ifndef ${atomicname}\n"
29		− . ${template}
30		− printf "#define ${atomicname} ${atomicname}\n"
31		− printf "#endif\n\n"
32		− fi
	25	+ . ${template}
33	26	}
34	27
35		−#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
36		−gen_proto_fallback()
	28	+#gen_order_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
	29	+gen_order_fallback()
37	30	{
38	31	local meta="$1"; shift
39	32	local pfx="$1"; shift
@@ -41,87 +34,124 @@ gen_proto_fallback() · 1 unmodified line
41	34	local sfx="$1"; shift
42	35	local order="$1"; shift
43	36
44		− local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
	37	+ local tmpl_order=${order#_}
	38	+ local tmpl="${ATOMICDIR}/fallbacks/${tmpl_order:-fence}"
45	39	gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
46	40	}
47	41
48		−#gen_basic_fallbacks(basename)
49		−gen_basic_fallbacks()
50		−{
51		− local basename="$1"; shift
52		−cat << EOF
53		−#define ${basename}_acquire ${basename}
54		−#define ${basename}_release ${basename}
55		−#define ${basename}_relaxed ${basename}
56		−EOF
57		−}
58		−
59		−gen_proto_order_variant()
	42	+#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
	43	+gen_proto_fallback()
60	44	{
61	45	local meta="$1"; shift
62	46	local pfx="$1"; shift
63	47	local name="$1"; shift
64	48	local sfx="$1"; shift
65	49	local order="$1"; shift
66		− local atomic="$1"
67	50
68		− local basename="arch_${atomic}_${pfx}${name}${sfx}"
69		−
70		− printf "#define ${basename}${order} ${basename}${order}\n"
	51	+ local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
	52	+ gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
71	53	}
72	54
73		−#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
74		−gen_proto_order_variants()
	55	+#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, args...)
	56	+gen_proto_order_variant()
75	57	{
76	58	local meta="$1"; shift
77	59	local pfx="$1"; shift
78	60	local name="$1"; shift
79	61	local sfx="$1"; shift
80		− local atomic="$1"
	62	+ local order="$1"; shift
	63	+ local atomic="$1"; shift
	64	+ local int="$1"; shift
81	65
82		− local basename="arch_${atomic}_${pfx}${name}${sfx}"
	66	+ local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
	67	+ local basename="${atomic}_${pfx}${name}${sfx}"
83	68
84	69	local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
85	70
86		− # If we don't have relaxed atomics, then we don't bother with ordering fallbacks
87		− # read_acquire and set_release need to be templated, though
88		− if ! meta_has_relaxed "${meta}"; then
89		− gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
	71	+ local ret="$(gen_ret_type "${meta}" "${int}")"
	72	+ local retstmt="$(gen_ret_stmt "${meta}")"
	73	+ local params="$(gen_params "${int}" "${atomic}" "$@")"
	74	+ local args="$(gen_args "$@")"
90	75
91		− if meta_has_acquire "${meta}"; then
92		− gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
93		− fi
	76	+ gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
	77	+
	78	+ printf "static __always_inline ${ret}\n"
	79	+ printf "raw_${atomicname}(${params})\n"
	80	+ printf "{\n"
	81	+
	82	+ # Where there is no possible fallback, this order variant is mandatory
	83	+ # and must be provided by arch code. Add a comment to the header to
	84	+ # make this obvious.
	85	+ #
	86	+ # Ideally we'd error on a missing definition, but arch code might
	87	+ # define this order variant as a C function without a preprocessor
	88	+ # symbol.
	89	+ if [ -z ${template} ] && [ -z "${order}" ] && ! meta_has_relaxed "${meta}"; then
	90	+ printf "\t${retstmt}arch_${atomicname}(${args});\n"
	91	+ printf "}\n\n"
	92	+ return
	93	+ fi
94	94
95		− if meta_has_release "${meta}"; then
96		− gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
97		− fi
	95	+ printf "#if defined(arch_${atomicname})\n"
	96	+ printf "\t${retstmt}arch_${atomicname}(${args});\n"
98	97
99		− return
	98	+ # Allow FULL/ACQUIRE/RELEASE ops to be defined in terms of RELAXED ops
	99	+ if [ "${order}" != "_relaxed" ] && meta_has_relaxed "${meta}"; then
	100	+ printf "#elif defined(arch_${basename}_relaxed)\n"
	101	+ gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
100	102	fi
101	103
102		− printf "#ifndef ${basename}_relaxed\n"
	104	+ # Allow ACQUIRE/RELEASE/RELAXED ops to be defined in terms of FULL ops
	105	+ if [ ! -z "${order}" ]; then
	106	+ printf "#elif defined(arch_${basename})\n"
	107	+ printf "\t${retstmt}arch_${basename}(${args});\n"
	108	+ fi
103	109
	110	+ printf "#else\n"
104	111	if [ ! -z "${template}" ]; then
105		− printf "#ifdef ${basename}\n"
	112	+ gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
	113	+ else
	114	+ printf "#error \"Unable to define raw_${atomicname}\"\n"
106	115	fi
107	116
108		− gen_basic_fallbacks "${basename}"
	117	+ printf "#endif\n"
	118	+ printf "}\n\n"
	119	+}
109	120
110		− if [ ! -z "${template}" ]; then
111		− printf "#endif /* ${basename} */\n\n"
112		− gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
113		− gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
114		− gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
115		− gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
	121	+
	122	+#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
	123	+gen_proto_order_variants()
	124	+{
	125	+ local meta="$1"; shift
	126	+ local pfx="$1"; shift
	127	+ local name="$1"; shift
	128	+ local sfx="$1"; shift
	129	+ local atomic="$1"
	130	+
	131	+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
	132	+
	133	+ if meta_has_acquire "${meta}"; then
	134	+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
116	135	fi
117	136
118		− printf "#else /* ${basename}_relaxed */\n\n"
	137	+ if meta_has_release "${meta}"; then
	138	+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
	139	+ fi
119	140
120		− gen_template_fallback "${ATOMICDIR}/fallbacks/acquire" "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
121		− gen_template_fallback "${ATOMICDIR}/fallbacks/release" "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
122		− gen_template_fallback "${ATOMICDIR}/fallbacks/fence" "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
	141	+ if meta_has_relaxed "${meta}"; then
	142	+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
	143	+ fi
	144	+}
123	145
124		− printf "#endif /* ${basename}_relaxed */\n\n"
	146	+#gen_basic_fallbacks(basename)
	147	+gen_basic_fallbacks()
	148	+{
	149	+ local basename="$1"; shift
	150	+cat << EOF
	151	+#define raw_${basename}_acquire arch_${basename}
	152	+#define raw_${basename}_release arch_${basename}
	153	+#define raw_${basename}_relaxed arch_${basename}
	154	+EOF
125	155	}
126	156
127	157	gen_order_fallbacks()
@@ -130,36 +160,65 @@ gen_order_fallbacks() · 2 unmodified lines
130	160
131	161	cat <<EOF
132	162
133		−#ifndef ${xchg}_acquire
134		−#define ${xchg}_acquire(...) \\
135		− __atomic_op_acquire(${xchg}, __VA_ARGS__)
	163	+#define raw_${xchg}_relaxed arch_${xchg}_relaxed
	164	+
	165	+#ifdef arch_${xchg}_acquire
	166	+#define raw_${xchg}_acquire arch_${xchg}_acquire
	167	+#else
	168	+#define raw_${xchg}_acquire(...) \\
	169	+ __atomic_op_acquire(arch_${xchg}, __VA_ARGS__)
136	170	#endif
137	171
138		−#ifndef ${xchg}_release
139		−#define ${xchg}_release(...) \\
140		− __atomic_op_release(${xchg}, __VA_ARGS__)
	172	+#ifdef arch_${xchg}_release
	173	+#define raw_${xchg}_release arch_${xchg}_release
	174	+#else
	175	+#define raw_${xchg}_release(...) \\
	176	+ __atomic_op_release(arch_${xchg}, __VA_ARGS__)
141	177	#endif
142	178
143		−#ifndef ${xchg}
144		−#define ${xchg}(...) \\
145		− __atomic_op_fence(${xchg}, __VA_ARGS__)
	179	+#ifdef arch_${xchg}
	180	+#define raw_${xchg} arch_${xchg}
	181	+#else
	182	+#define raw_${xchg}(...) \\
	183	+ __atomic_op_fence(arch_${xchg}, __VA_ARGS__)
146	184	#endif
147	185
148	186	EOF
149	187	}
150	188
151		−gen_xchg_fallbacks()
	189	+gen_xchg_order_fallback()
152	190	{
153	191	local xchg="$1"; shift
154		− printf "#ifndef ${xchg}_relaxed\n"
	192	+ local order="$1"; shift
	193	+ local forder="${order:-_fence}"
155	194
156		− gen_basic_fallbacks ${xchg}
	195	+ printf "#if defined(arch_${xchg}${order})\n"
	196	+ printf "#define raw_${xchg}${order} arch_${xchg}${order}\n"
157	197
158		− printf "#else /* ${xchg}_relaxed */\n"
	198	+ if [ "${order}" != "_relaxed" ]; then
	199	+ printf "#elif defined(arch_${xchg}_relaxed)\n"
	200	+ printf "#define raw_${xchg}${order}(...) \\\\\n"
	201	+ printf " __atomic_op${forder}(arch_${xchg}, __VA_ARGS__)\n"
	202	+ fi
	203	+
	204	+ if [ ! -z "${order}" ]; then
	205	+ printf "#elif defined(arch_${xchg})\n"
	206	+ printf "#define raw_${xchg}${order} arch_${xchg}\n"
	207	+ fi
159	208
160		− gen_order_fallbacks ${xchg}
	209	+ printf "#else\n"
	210	+ printf "extern void raw_${xchg}${order}_not_implemented(void);\n"
	211	+ printf "#define raw_${xchg}${order}(...) raw_${xchg}${order}_not_implemented()\n"
	212	+ printf "#endif\n\n"
	213	+}
	214	+
	215	+gen_xchg_fallbacks()
	216	+{
	217	+ local xchg="$1"; shift
161	218
162		− printf "#endif /* ${xchg}_relaxed */\n\n"
	219	+ for order in "" "_acquire" "_release" "_relaxed"; do
	220	+ gen_xchg_order_fallback "${xchg}" "${order}"
	221	+ done
163	222	}
164	223
165	224	gen_try_cmpxchg_fallback()
@@ -168,40 +227,61 @@ gen_try_cmpxchg_fallback() · 2 unmodified lines
168	227	local order="$1"; shift;
169	228
170	229	cat <<EOF
171		−#ifndef arch_try_${cmpxchg}${order}
172		−#define arch_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
	230	+#define raw_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
173	231	({ \\
174	232	typeof((_ptr)) ___op = (_oldp), ___o = *___op, ___r; \\
175		− ___r = arch_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
	233	+ ___r = raw_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
176	234	if (unlikely(___r != ___o)) \\
177	235	*___op = ___r; \\
178	236	likely(___r == ___o); \\
179	237	})
180		−#endif /* arch_try_${cmpxchg}${order} */
181		−
182	238	EOF
183	239	}
184	240
185		−gen_try_cmpxchg_fallbacks()
	241	+gen_try_cmpxchg_order_fallback()
186	242	{
187		− local cmpxchg="$1"; shift;
	243	+ local cmpxchg="$1"; shift
	244	+ local order="$1"; shift
	245	+ local forder="${order:-_fence}"
188	246
189		− printf "#ifndef arch_try_${cmpxchg}_relaxed\n"
190		− printf "#ifdef arch_try_${cmpxchg}\n"
	247	+ printf "#if defined(arch_try_${cmpxchg}${order})\n"
	248	+ printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}${order}\n"
	249	+
	250	+ if [ "${order}" != "_relaxed" ]; then
	251	+ printf "#elif defined(arch_try_${cmpxchg}_relaxed)\n"
	252	+ printf "#define raw_try_${cmpxchg}${order}(...) \\\\\n"
	253	+ printf " __atomic_op${forder}(arch_try_${cmpxchg}, __VA_ARGS__)\n"
	254	+ fi
191	255
192		− gen_basic_fallbacks "arch_try_${cmpxchg}"
	256	+ if [ ! -z "${order}" ]; then
	257	+ printf "#elif defined(arch_try_${cmpxchg})\n"
	258	+ printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}\n"
	259	+ fi
193	260
194		− printf "#endif /* arch_try_${cmpxchg} */\n\n"
	261	+ printf "#else\n"
	262	+ gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
	263	+ printf "#endif\n\n"
	264	+}
	265	+
	266	+gen_try_cmpxchg_fallbacks()
	267	+{
	268	+ local cmpxchg="$1"; shift;
195	269
196	270	for order in "" "_acquire" "_release" "_relaxed"; do
197		− gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
	271	+ gen_try_cmpxchg_order_fallback "${cmpxchg}" "${order}"
198	272	done
	273	+}
199	274
200		− printf "#else /* arch_try_${cmpxchg}_relaxed */\n"
201		−
202		− gen_order_fallbacks "arch_try_${cmpxchg}"
	275	+gen_cmpxchg_local_fallbacks()
	276	+{
	277	+ local cmpxchg="$1"; shift
203	278
204		− printf "#endif /* arch_try_${cmpxchg}_relaxed */\n\n"
	279	+ printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
	280	+ printf "#ifdef arch_try_${cmpxchg}\n"
	281	+ printf "#define raw_try_${cmpxchg} arch_try_${cmpxchg}\n"
	282	+ printf "#else\n"
	283	+ gen_try_cmpxchg_fallback "${cmpxchg}" ""
	284	+ printf "#endif\n\n"
205	285	}
206	286
207	287	cat << EOF
@@ -217,16 +297,20 @@ cat << EOF · 9 unmodified lines
217	297
218	298	EOF
219	299
220		−for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do
	300	+for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128"; do
221	301	gen_xchg_fallbacks "${xchg}"
222	302	done
223	303
224		−for cmpxchg in "cmpxchg" "cmpxchg64"; do
	304	+for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
225	305	gen_try_cmpxchg_fallbacks "${cmpxchg}"
226	306	done
227	307
228		−for cmpxchg in "cmpxchg_local" "cmpxchg64_local"; do
229		− gen_try_cmpxchg_fallback "${cmpxchg}" ""
	308	+for cmpxchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local"; do
	309	+ gen_cmpxchg_local_fallbacks "${cmpxchg}" ""
	310	+done
	311	+
	312	+for cmpxchg in "sync_cmpxchg"; do
	313	+ printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
230	314	done
231	315
232	316	grep '^[a-z]' "$1" \| while read name meta args; do

scripts/atomic/gen-atomic-instrumented.sh+16−26

@@ -68,12 +68,14 @@ gen_proto_order_variant() · 67 unmodified lines
68	68	local args="$(gen_args "$@")"
69	69	local retstmt="$(gen_ret_stmt "${meta}")"
70	70
	71	+ gen_kerneldoc "" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
	72	+
71	73	cat <<EOF
72	74	static __always_inline ${ret}
73	75	${atomicname}(${params})
74	76	{
75	77	${checks}
76		− ${retstmt}arch_${atomicname}(${args});
	78	+ ${retstmt}raw_${atomicname}(${args});
77	79	}
78	80	EOF
79	81
@@ -84,7 +86,6 @@ gen_xchg() · 4 unmodified lines
84	86	{
85	87	local xchg="$1"; shift
86	88	local order="$1"; shift
87		− local mult="$1"; shift
88	89
89	90	kcsan_barrier=""
90	91	if [ "${xchg%_local}" = "${xchg}" ]; then
@@ -104,9 +105,9 @@ cat <<EOF · 13 unmodified lines
104	105	EOF
105	106	[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
106	107	cat <<EOF
107		− instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
108		− instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
109		− arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
	108	+ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
	109	+ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\
	110	+ raw_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
110	111	})
111	112	EOF
112	113
@@ -119,8 +120,8 @@ cat <<EOF · 6 unmodified lines
119	120	EOF
120	121	[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
121	122	cat <<EOF
122		− instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
123		− arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\
	123	+ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
	124	+ raw_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\
124	125	})
125	126	EOF
126	127
@@ -134,15 +135,10 @@ cat << EOF · 7 unmodified lines
134	135	// DO NOT MODIFY THIS FILE DIRECTLY
135	136
136	137	/*
137		− * This file provides wrappers with KASAN instrumentation for atomic operations.
138		− * To use this functionality an arch's atomic.h file needs to define all
139		− * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
140		− * this file at the end. This file provides atomic_read() that forwards to
141		− * arch_atomic_read() for actual atomic operation.
142		− * Note: if an arch atomic operation is implemented by means of other atomic
143		− * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
144		− * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
145		− * double instrumentation.
	138	+ * This file provoides atomic operations with explicit instrumentation (e.g.
	139	+ * KASAN, KCSAN), which should be used unless it is necessary to avoid
	140	+ * instrumentation. Where it is necessary to aovid instrumenation, the
	141	+ * raw_atomic*() operations should be used.
146	142	*/
147	143	#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
148	144	#define _LINUX_ATOMIC_INSTRUMENTED_H
@@ -166,24 +162,18 @@ grep '^[a-z]' "$1" \| while read name meta args; do · 17 unmodified lines
166	162	done
167	163
168	164
169		−for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do
	165	+for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
170	166	for order in "" "_acquire" "_release" "_relaxed"; do
171		− gen_xchg "${xchg}" "${order}" ""
	167	+ gen_xchg "${xchg}" "${order}"
172	168	printf "\n"
173	169	done
174	170	done
175	171
176		−for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do
177		− gen_xchg "${xchg}" "" ""
	172	+for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do
	173	+ gen_xchg "${xchg}" ""
178	174	printf "\n"
179	175	done
180	176
181		−gen_xchg "cmpxchg_double" "" "2 * "
182		−
183		−printf "\n\n"
184		−
185		−gen_xchg "cmpxchg_double_local" "" "2 * "
186		−
187	177	cat <<EOF
188	178
189	179	#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */

scripts/atomic/gen-atomic-long.sh+18−20

@@ -32,24 +32,34 @@ gen_args_cast() · 31 unmodified lines
32	32	done
33	33	}
34	34
35		−#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...)
	35	+#gen_proto_order_variant(meta, pfx, name, sfx, order, arg...)
36	36	gen_proto_order_variant()
37	37	{
38	38	local meta="$1"; shift
39		− local name="$1$2$3$4"; shift; shift; shift; shift
40		− local atomic="$1"; shift
41		− local int="$1"; shift
	39	+ local pfx="$1"; shift
	40	+ local name="$1"; shift
	41	+ local sfx="$1"; shift
	42	+ local order="$1"; shift
	43	+
	44	+ local atomicname="${pfx}${name}${sfx}${order}"
42	45
43	46	local ret="$(gen_ret_type "${meta}" "long")"
44	47	local params="$(gen_params "long" "atomic_long" "$@")"
45		− local argscast="$(gen_args_cast "${int}" "${atomic}" "$@")"
	48	+ local argscast_32="$(gen_args_cast "int" "atomic" "$@")"
	49	+ local argscast_64="$(gen_args_cast "s64" "atomic64" "$@")"
46	50	local retstmt="$(gen_ret_stmt "${meta}")"
47	51
	52	+ gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "atomic_long" "long" "$@"
	53	+
48	54	cat <<EOF
49	55	static __always_inline ${ret}
50		−arch_atomic_long_${name}(${params})
	56	+raw_atomic_long_${atomicname}(${params})
51	57	{
52		− ${retstmt}arch_${atomic}_${name}(${argscast});
	58	+#ifdef CONFIG_64BIT
	59	+ ${retstmt}raw_atomic64_${atomicname}(${argscast_64});
	60	+#else
	61	+ ${retstmt}raw_atomic_${atomicname}(${argscast_32});
	62	+#endif
53	63	}
54	64
55	65	EOF
@@ -79,24 +89,12 @@ typedef atomic_t atomic_long_t; · 23 unmodified lines
79	89	#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
80	90	#endif
81	91
82		−#ifdef CONFIG_64BIT
83		−
84		−EOF
85		−
86		−grep '^[a-z]' "$1" \| while read name meta args; do
87		− gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
88		−done
89		−
90		−cat <<EOF
91		−#else /* CONFIG_64BIT */
92		−
93	92	EOF
94	93
95	94	grep '^[a-z]' "$1" \| while read name meta args; do
96		− gen_proto "${meta}" "${name}" "atomic" "int" ${args}
	95	+ gen_proto "${meta}" "${name}" ${args}
97	96	done
98	97
99	98	cat <<EOF
100		−#endif /* CONFIG_64BIT */
101	99	#endif /* _LINUX_ATOMIC_LONG_H */
102	100	EOF

scripts/atomic/kerneldoc/add+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic add with ${desc_order} ordering
	4	+ * @i: ${int} value to add
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v + @i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * ${desc_return}
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/add_negative+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic add and test if negative with ${desc_order} ordering
	4	+ * @i: ${int} value to add
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v + @i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * Return: @true if the resulting value of @v is negative, @false otherwise.
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/add_unless+18−0

@@ -0,0 +1,18 @@
	1	+if [ -z "${pfx}" ]; then
	2	+ desc_return="Return: @true if @v was updated, @false otherwise."
	3	+fi
	4	+
	5	+cat <<EOF
	6	+/**
	7	+ * ${class}${atomicname}() - atomic add unless value with ${desc_order} ordering
	8	+ * @v: pointer to ${atomic}_t
	9	+ * @a: ${int} value to add
	10	+ * @u: ${int} value to compare with
	11	+ *
	12	+ * If (@v != @u), atomically updates @v to (@v + @a) with ${desc_order} ordering.
	13	+ *
	14	+ * ${desc_noinstr}
	15	+ *
	16	+ * ${desc_return}
	17	+ */
	18	+EOF

scripts/atomic/kerneldoc/and+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic bitwise AND with ${desc_order} ordering
	4	+ * @i: ${int} value
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v & @i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * ${desc_return}
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/andnot+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic bitwise AND NOT with ${desc_order} ordering
	4	+ * @i: ${int} value
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v & ~@i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * ${desc_return}
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/cmpxchg+14−0

@@ -0,0 +1,14 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ * @old: ${int} value to compare with
	6	+ * @new: ${int} value to assign
	7	+ *
	8	+ * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
	9	+ *
	10	+ * ${desc_noinstr}
	11	+ *
	12	+ * Return: The original value of @v.
	13	+ */
	14	+EOF

scripts/atomic/kerneldoc/dec+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic decrement with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * Atomically updates @v to (@v - 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * ${desc_return}
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/dec_and_test+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic decrement and test if zero with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * Atomically updates @v to (@v - 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/dec_if_positive+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic decrement if positive with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * If (@v > 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/dec_unless_positive+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic decrement unless positive with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * If (@v <= 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * Return: @true if @v was updated, @false otherwise.
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/inc+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic increment with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * Atomically updates @v to (@v + 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * ${desc_return}
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/inc_and_test+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic increment and test if zero with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * Atomically updates @v to (@v + 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/inc_not_zero+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic increment unless zero with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * If (@v != 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * Return: @true if @v was updated, @false otherwise.
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/inc_unless_negative+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic increment unless negative with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * If (@v >= 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * Return: @true if @v was updated, @false otherwise.
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/or+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic bitwise OR with ${desc_order} ordering
	4	+ * @i: ${int} value
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v \| @i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * ${desc_return}
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/read+12−0

@@ -0,0 +1,12 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic load with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ *
	6	+ * Atomically loads the value of @v with ${desc_order} ordering.
	7	+ *
	8	+ * ${desc_noinstr}
	9	+ *
	10	+ * Return: The value loaded from @v.
	11	+ */
	12	+EOF

scripts/atomic/kerneldoc/set+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic set with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ * @i: ${int} value to assign
	6	+ *
	7	+ * Atomically sets @v to @i with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * Return: Nothing.
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/sub+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic subtract with ${desc_order} ordering
	4	+ * @i: ${int} value to subtract
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v - @i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * ${desc_return}
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/sub_and_test+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic subtract and test if zero with ${desc_order} ordering
	4	+ * @i: ${int} value to add
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v - @i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * Return: @true if the resulting value of @v is zero, @false otherwise.
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/try_cmpxchg+15−0

@@ -0,0 +1,15 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ * @old: pointer to ${int} value to compare with
	6	+ * @new: ${int} value to assign
	7	+ *
	8	+ * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
	9	+ * Otherwise, updates @old to the current value of @v.
	10	+ *
	11	+ * ${desc_noinstr}
	12	+ *
	13	+ * Return: @true if the exchange occured, @false otherwise.
	14	+ */
	15	+EOF

scripts/atomic/kerneldoc/xchg+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic exchange with ${desc_order} ordering
	4	+ * @v: pointer to ${atomic}_t
	5	+ * @new: ${int} value to assign
	6	+ *
	7	+ * Atomically updates @v to @new with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * Return: The original value of @v.
	12	+ */
	13	+EOF

scripts/atomic/kerneldoc/xor+13−0

@@ -0,0 +1,13 @@
	1	+cat <<EOF
	2	+/**
	3	+ * ${class}${atomicname}() - atomic bitwise XOR with ${desc_order} ordering
	4	+ * @i: ${int} value
	5	+ * @v: pointer to ${atomic}_t
	6	+ *
	7	+ * Atomically updates @v to (@v ^ @i) with ${desc_order} ordering.
	8	+ *
	9	+ * ${desc_noinstr}
	10	+ *
	11	+ * ${desc_return}
	12	+ */
	13	+EOF

scripts/kernel-doc+1−1

@@ -64,7 +64,7 @@ my $type_constant = '\b``([^\`]+)``\b'; · 63 unmodified lines
64	64	my $type_constant2 = '\%([-_\w]+)';
65	65	my $type_func = '(\w+)';
66	66	my $type_param = '\@(\w((\.\w+)\|(->\w+))(\.\.\.)?)';
67		−my $type_param_ref = '([\!]?)\@(\w((\.\w+)\|(->\w+))(\.\.\.)?)';
	67	+my $type_param_ref = '([\!~]?)\@(\w((\.\w+)\|(->\w+))(\.\.\.)?)';
68	68	my $type_fp_param = '\@(\w+)'; # Special RST handling for func ptr params
69	69	my $type_fp_param2 = '\@(\w+->\S+)'; # Special RST handling for structs with func ptr params
70	70	my $type_env = '(\$\w+)';

scripts/min-tool-version.sh+5−1

@@ -17,7 +17,11 @@ binutils) · 16 unmodified lines
17	17	echo 2.25.0
18	18	;;
19	19	gcc)
20		− echo 5.1.0
	20	+ if [ "$SRCARCH" = parisc ]; then
	21	+ echo 11.0.0
	22	+ else
	23	+ echo 5.1.0
	24	+ fi
21	25	;;
22	26	llvm)
23	27	if [ "$SRCARCH" = s390 ]; then

@@ -64,7 +64,7 @@ my $type_constant = '\b``([^\`]+)``\b'; · 63 unmodified lines
64	64	my $type_constant2 = '\%([-_\w]+)';
65	65	my $type_func = '(\w+)\(\)';
66	66	my $type_param = '\@(\w((\.\w+)\|(->\w+))(\.\.\.)?)';
67		−my $type_param_ref = '([\!]?)\@(\w((\.\w+)\|(->\w+))(\.\.\.)?)';
	67	+my $type_param_ref = '([\!~]?)\@(\w((\.\w+)\|(->\w+))(\.\.\.)?)';
68	68	my $type_fp_param = '\@(\w+)\(\)'; # Special RST handling for func ptr params
69	69	my $type_fp_param2 = '\@(\w+->\S+)\(\)'; # Special RST handling for structs with func ptr params
70	70	my $type_env = '(\$\w+)';