History

arch: Remove cmpxchg_double

No moar users, remove the monster.

Peter Zijlstra committed 3 years ago

Commit febe9501 parent 6801be415 files · +5 −371Browse files

Unified · Split

15 files
  1. Documentation/core-api/this_cpu_ops.rst +0 −2
  2. arch/arm64/include/asm/atomic_ll_sc.h +0 −33
  3. arch/arm64/include/asm/atomic_lse.h +0 −36
  4. arch/arm64/include/asm/cmpxchg.h +0 −46
  5. arch/arm64/include/asm/percpu.h +0 −10
  6. arch/s390/include/asm/cmpxchg.h +0 −34
  7. arch/s390/include/asm/percpu.h +0 −18
  8. arch/x86/include/asm/cmpxchg.h +0 −25
  9. arch/x86/include/asm/cmpxchg_32.h +0 −1
  10. arch/x86/include/asm/cmpxchg_64.h +0 −1
  11. arch/x86/include/asm/percpu.h +0 −42
  12. include/asm-generic/percpu.h +0 −58
  13. include/linux/atomic/atomic-instrumented.h +1 −16
  14. include/linux/percpu-defs.h +0 −38
  15. scripts/atomic/gen-atomic-instrumented.sh +4 −11
Documentation/core-api/this_cpu_ops.rst+0−2
@@ -53,7 +53,6 @@ preemption and interrupts:: · 52 unmodified lines
53 this_cpu_add_return(pcp, val)53 this_cpu_add_return(pcp, val)
54 this_cpu_xchg(pcp, nval)54 this_cpu_xchg(pcp, nval)
55 this_cpu_cmpxchg(pcp, oval, nval)55 this_cpu_cmpxchg(pcp, oval, nval)
56 this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
57 this_cpu_sub(pcp, val)56 this_cpu_sub(pcp, val)
58 this_cpu_inc(pcp)57 this_cpu_inc(pcp)
59 this_cpu_dec(pcp)58 this_cpu_dec(pcp)
@@ -242,7 +241,6 @@ safe:: · 182 unmodified lines
242 __this_cpu_add_return(pcp, val)241 __this_cpu_add_return(pcp, val)
243 __this_cpu_xchg(pcp, nval)242 __this_cpu_xchg(pcp, nval)
244 __this_cpu_cmpxchg(pcp, oval, nval)243 __this_cpu_cmpxchg(pcp, oval, nval)
245 __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
246 __this_cpu_sub(pcp, val)244 __this_cpu_sub(pcp, val)
247 __this_cpu_inc(pcp)245 __this_cpu_inc(pcp)
248 __this_cpu_dec(pcp)246 __this_cpu_dec(pcp)
arch/arm64/include/asm/atomic_ll_sc.h+0−33
@@ -294,39 +294,6 @@ __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) · 293 unmodified lines
294 294
295#undef __CMPXCHG_CASE295#undef __CMPXCHG_CASE
296 296
297#define __CMPXCHG_DBL(name, mb, rel, cl) \
298static __always_inline long \
299__ll_sc__cmpxchg_double##name(unsigned long old1, \
300 unsigned long old2, \
301 unsigned long new1, \
302 unsigned long new2, \
303 volatile void *ptr) \
304{ \
305 unsigned long tmp, ret; \
306 \
307 asm volatile("// __cmpxchg_double" #name "\n" \
308 " prfm pstl1strm, %2\n" \
309 "1: ldxp %0, %1, %2\n" \
310 " eor %0, %0, %3\n" \
311 " eor %1, %1, %4\n" \
312 " orr %1, %0, %1\n" \
313 " cbnz %1, 2f\n" \
314 " st" #rel "xp %w0, %5, %6, %2\n" \
315 " cbnz %w0, 1b\n" \
316 " " #mb "\n" \
317 "2:" \
318 : "=&r" (tmp), "=&r" (ret), "+Q" (*(__uint128_t *)ptr) \
319 : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \
320 : cl); \
321 \
322 return ret; \
323}
324
325__CMPXCHG_DBL( , , , )
326__CMPXCHG_DBL(_mb, dmb ish, l, "memory")
327
328#undef __CMPXCHG_DBL
329
330union __u128_halves {297union __u128_halves {
331 u128 full;298 u128 full;
332 struct {299 struct {
arch/arm64/include/asm/atomic_lse.h+0−36
@@ -281,42 +281,6 @@ __CMPXCHG_CASE(x, , mb_, 64, al, "memory") · 280 unmodified lines
281 281
282#undef __CMPXCHG_CASE282#undef __CMPXCHG_CASE
283 283
284#define __CMPXCHG_DBL(name, mb, cl...) \
285static __always_inline long \
286__lse__cmpxchg_double##name(unsigned long old1, \
287 unsigned long old2, \
288 unsigned long new1, \
289 unsigned long new2, \
290 volatile void *ptr) \
291{ \
292 unsigned long oldval1 = old1; \
293 unsigned long oldval2 = old2; \
294 register unsigned long x0 asm ("x0") = old1; \
295 register unsigned long x1 asm ("x1") = old2; \
296 register unsigned long x2 asm ("x2") = new1; \
297 register unsigned long x3 asm ("x3") = new2; \
298 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
299 \
300 asm volatile( \
301 __LSE_PREAMBLE \
302 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
303 " eor %[old1], %[old1], %[oldval1]\n" \
304 " eor %[old2], %[old2], %[oldval2]\n" \
305 " orr %[old1], %[old1], %[old2]" \
306 : [old1] "+&r" (x0), [old2] "+&r" (x1), \
307 [v] "+Q" (*(__uint128_t *)ptr) \
308 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
309 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
310 : cl); \
311 \
312 return x0; \
313}
314
315__CMPXCHG_DBL( , )
316__CMPXCHG_DBL(_mb, al, "memory")
317
318#undef __CMPXCHG_DBL
319
320#define __CMPXCHG128(name, mb, cl...) \284#define __CMPXCHG128(name, mb, cl...) \
321static __always_inline u128 \285static __always_inline u128 \
322__lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \286__lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
arch/arm64/include/asm/cmpxchg.h+0−46
@@ -130,22 +130,6 @@ __CMPXCHG_CASE(mb_, 64) · 129 unmodified lines
130 130
131#undef __CMPXCHG_CASE131#undef __CMPXCHG_CASE
132 132
133#define __CMPXCHG_DBL(name) \
134static inline long __cmpxchg_double##name(unsigned long old1, \
135 unsigned long old2, \
136 unsigned long new1, \
137 unsigned long new2, \
138 volatile void *ptr) \
139{ \
140 return __lse_ll_sc_body(_cmpxchg_double##name, \
141 old1, old2, new1, new2, ptr); \
142}
143
144__CMPXCHG_DBL( )
145__CMPXCHG_DBL(_mb)
146
147#undef __CMPXCHG_DBL
148
149#define __CMPXCHG128(name) \133#define __CMPXCHG128(name) \
150static inline u128 __cmpxchg128##name(volatile u128 *ptr, \134static inline u128 __cmpxchg128##name(volatile u128 *ptr, \
151 u128 old, u128 new) \135 u128 old, u128 new) \
@@ -211,36 +195,6 @@ __CMPXCHG_GEN(_mb) · 59 unmodified lines
211#define arch_cmpxchg64 arch_cmpxchg195#define arch_cmpxchg64 arch_cmpxchg
212#define arch_cmpxchg64_local arch_cmpxchg_local196#define arch_cmpxchg64_local arch_cmpxchg_local
213 197
214/* cmpxchg_double */
215#define system_has_cmpxchg_double() 1
216
217#define __cmpxchg_double_check(ptr1, ptr2) \
218({ \
219 if (sizeof(*(ptr1)) != 8) \
220 BUILD_BUG(); \
221 VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
222})
223
224#define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
225({ \
226 int __ret; \
227 __cmpxchg_double_check(ptr1, ptr2); \
228 __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
229 (unsigned long)(n1), (unsigned long)(n2), \
230 ptr1); \
231 __ret; \
232})
233
234#define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
235({ \
236 int __ret; \
237 __cmpxchg_double_check(ptr1, ptr2); \
238 __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
239 (unsigned long)(n1), (unsigned long)(n2), \
240 ptr1); \
241 __ret; \
242})
243
244/* cmpxchg128 */198/* cmpxchg128 */
245#define system_has_cmpxchg128() 1199#define system_has_cmpxchg128() 1
246 200
arch/arm64/include/asm/percpu.h+0−10
@@ -145,16 +145,6 @@ PERCPU_RET_OP(add, add, ldadd) · 144 unmodified lines
145 * preemption point when TIF_NEED_RESCHED gets set while preemption is145 * preemption point when TIF_NEED_RESCHED gets set while preemption is
146 * disabled.146 * disabled.
147 */147 */
148#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
149({ \
150 int __ret; \
151 preempt_disable_notrace(); \
152 __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
153 raw_cpu_ptr(&(ptr2)), \
154 o1, o2, n1, n2); \
155 preempt_enable_notrace(); \
156 __ret; \
157})
158 148
159#define _pcp_protect(op, pcp, ...) \149#define _pcp_protect(op, pcp, ...) \
160({ \150({ \
arch/s390/include/asm/cmpxchg.h+0−34
@@ -190,40 +190,6 @@ static __always_inline unsigned long __cmpxchg(unsigned long address, · 189 unmodified lines
190#define arch_cmpxchg_local arch_cmpxchg190#define arch_cmpxchg_local arch_cmpxchg
191#define arch_cmpxchg64_local arch_cmpxchg191#define arch_cmpxchg64_local arch_cmpxchg
192 192
193#define system_has_cmpxchg_double() 1
194
195static __always_inline int __cmpxchg_double(unsigned long p1, unsigned long p2,
196 unsigned long o1, unsigned long o2,
197 unsigned long n1, unsigned long n2)
198{
199 union register_pair old = { .even = o1, .odd = o2, };
200 union register_pair new = { .even = n1, .odd = n2, };
201 int cc;
202
203 asm volatile(
204 " cdsg %[old],%[new],%[ptr]\n"
205 " ipm %[cc]\n"
206 " srl %[cc],28\n"
207 : [cc] "=&d" (cc), [old] "+&d" (old.pair)
208 : [new] "d" (new.pair),
209 [ptr] "QS" (*(unsigned long *)p1), "Q" (*(unsigned long *)p2)
210 : "memory", "cc");
211 return !cc;
212}
213
214#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \
215({ \
216 typeof(p1) __p1 = (p1); \
217 typeof(p2) __p2 = (p2); \
218 \
219 BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
220 BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
221 VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\
222 __cmpxchg_double((unsigned long)__p1, (unsigned long)__p2, \
223 (unsigned long)(o1), (unsigned long)(o2), \
224 (unsigned long)(n1), (unsigned long)(n2)); \
225})
226
227#define system_has_cmpxchg128() 1193#define system_has_cmpxchg128() 1
228 194
229static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)195static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
arch/s390/include/asm/percpu.h+0−18
@@ -180,24 +180,6 @@ · 179 unmodified lines
180#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)180#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
181#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)181#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
182 182
183#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \
184({ \
185 typeof(pcp1) *p1__; \
186 typeof(pcp2) *p2__; \
187 int ret__; \
188 \
189 preempt_disable_notrace(); \
190 p1__ = raw_cpu_ptr(&(pcp1)); \
191 p2__ = raw_cpu_ptr(&(pcp2)); \
192 ret__ = __cmpxchg_double((unsigned long)p1__, (unsigned long)p2__, \
193 (unsigned long)(o1), (unsigned long)(o2), \
194 (unsigned long)(n1), (unsigned long)(n2)); \
195 preempt_enable_notrace(); \
196 ret__; \
197})
198
199#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
200
201#include <asm-generic/percpu.h>183#include <asm-generic/percpu.h>
202 184
203#endif /* __ARCH_S390_PERCPU__ */185#endif /* __ARCH_S390_PERCPU__ */
arch/x86/include/asm/cmpxchg.h+0−25
@@ -239,29 +239,4 @@ extern void __add_wrong_size(void) · 238 unmodified lines
239#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)239#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
240#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)240#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
241 241
242#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
243({ \
244 bool __ret; \
245 __typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \
246 __typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \
247 BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
248 BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
249 VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \
250 VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \
251 asm volatile(pfx "cmpxchg%c5b %1" \
252 CC_SET(e) \
253 : CC_OUT(e) (__ret), \
254 "+m" (*(p1)), "+m" (*(p2)), \
255 "+a" (__old1), "+d" (__old2) \
256 : "i" (2 * sizeof(long)), \
257 "b" (__new1), "c" (__new2)); \
258 __ret; \
259})
260
261#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \
262 __cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)
263
264#define arch_cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
265 __cmpxchg_double(, p1, p2, o1, o2, n1, n2)
266
267#endif /* ASM_X86_CMPXCHG_H */242#endif /* ASM_X86_CMPXCHG_H */
arch/x86/include/asm/cmpxchg_32.h+0−1
@@ -103,7 +103,6 @@ static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new) · 102 unmodified lines
103 103
104#endif104#endif
105 105
106#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)
107#define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8)106#define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8)
108 107
109#endif /* _ASM_X86_CMPXCHG_32_H */108#endif /* _ASM_X86_CMPXCHG_32_H */
arch/x86/include/asm/cmpxchg_64.h+0−1
@@ -81,7 +81,6 @@ static __always_inline bool arch_try_cmpxchg128_local(volatile u128 *ptr, u128 * · 80 unmodified lines
81 return __arch_try_cmpxchg128(ptr, oldp, new,);81 return __arch_try_cmpxchg128(ptr, oldp, new,);
82}82}
83 83
84#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16)
85#define system_has_cmpxchg128() boot_cpu_has(X86_FEATURE_CX16)84#define system_has_cmpxchg128() boot_cpu_has(X86_FEATURE_CX16)
86 85
87#endif /* _ASM_X86_CMPXCHG_64_H */86#endif /* _ASM_X86_CMPXCHG_64_H */
arch/x86/include/asm/percpu.h+0−42
@@ -351,23 +351,6 @@ do { \ · 350 unmodified lines
351#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)351#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
352#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)352#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
353 353
354#ifdef CONFIG_X86_CMPXCHG64
355#define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2) \
356({ \
357 bool __ret; \
358 typeof(pcp1) __o1 = (o1), __n1 = (n1); \
359 typeof(pcp2) __o2 = (o2), __n2 = (n2); \
360 asm volatile("cmpxchg8b "__percpu_arg(1) \
361 CC_SET(z) \
362 : CC_OUT(z) (__ret), "+m" (pcp1), "+m" (pcp2), "+a" (__o1), "+d" (__o2) \
363 : "b" (__n1), "c" (__n2)); \
364 __ret; \
365})
366
367#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
368#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
369#endif /* CONFIG_X86_CMPXCHG64 */
370
371/*354/*
372 * Per cpu atomic 64 bit operations are only available under 64 bit.355 * Per cpu atomic 64 bit operations are only available under 64 bit.
373 * 32 bit must fall back to generic operations.356 * 32 bit must fall back to generic operations.
@@ -390,31 +373,6 @@ do { \ · 16 unmodified lines
390#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)373#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
391#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval)374#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval)
392#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)375#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
393
394/*
395 * Pretty complex macro to generate cmpxchg16 instruction. The instruction
396 * is not supported on early AMD64 processors so we must be able to emulate
397 * it in software. The address used in the cmpxchg16 instruction must be
398 * aligned to a 16 byte boundary.
399 */
400#define percpu_cmpxchg16b_double(pcp1, pcp2, o1, o2, n1, n2) \
401({ \
402 bool __ret; \
403 typeof(pcp1) __o1 = (o1), __n1 = (n1); \
404 typeof(pcp2) __o2 = (o2), __n2 = (n2); \
405 asm volatile (ALTERNATIVE("leaq %P1, %%rsi; call this_cpu_cmpxchg16b_emu", \
406 "cmpxchg16b " __percpu_arg(1), X86_FEATURE_CX16) \
407 "setz %0" \
408 : "=a" (__ret), "+m" (pcp1) \
409 : "b" (__n1), "c" (__n2), \
410 "a" (__o1), "d" (__o2) \
411 : "memory", "rsi"); \
412 __ret; \
413})
414
415#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
416#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
417
418#endif376#endif
419 377
420static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,378static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,
include/asm-generic/percpu.h+0−58
@@ -120,19 +120,6 @@ do { \ · 119 unmodified lines
120 __old; \120 __old; \
121})121})
122 122
123#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
124({ \
125 typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \
126 typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2)); \
127 int __ret = 0; \
128 if (*__p1 == (oval1) && *__p2 == (oval2)) { \
129 *__p1 = nval1; \
130 *__p2 = nval2; \
131 __ret = 1; \
132 } \
133 (__ret); \
134})
135
136#define __this_cpu_generic_read_nopreempt(pcp) \123#define __this_cpu_generic_read_nopreempt(pcp) \
137({ \124({ \
138 typeof(pcp) ___ret; \125 typeof(pcp) ___ret; \
@@ -211,17 +198,6 @@ do { \ · 72 unmodified lines
211 __ret; \198 __ret; \
212})199})
213 200
214#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
215({ \
216 int __ret; \
217 unsigned long __flags; \
218 raw_local_irq_save(__flags); \
219 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
220 oval1, oval2, nval1, nval2); \
221 raw_local_irq_restore(__flags); \
222 __ret; \
223})
224
225#ifndef raw_cpu_read_1201#ifndef raw_cpu_read_1
226#define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)202#define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)
227#endif203#endif
@@ -395,23 +371,6 @@ do { \ · 167 unmodified lines
395 raw_cpu_generic_cmpxchg(pcp, oval, nval)371 raw_cpu_generic_cmpxchg(pcp, oval, nval)
396#endif372#endif
397 373
398#ifndef raw_cpu_cmpxchg_double_1
399#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
400 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
401#endif
402#ifndef raw_cpu_cmpxchg_double_2
403#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
404 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
405#endif
406#ifndef raw_cpu_cmpxchg_double_4
407#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
408 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
409#endif
410#ifndef raw_cpu_cmpxchg_double_8
411#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
412 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
413#endif
414
415#ifndef this_cpu_read_1374#ifndef this_cpu_read_1
416#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)375#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
417#endif376#endif
@@ -585,21 +544,4 @@ do { \ · 167 unmodified lines
585 this_cpu_generic_cmpxchg(pcp, oval, nval)544 this_cpu_generic_cmpxchg(pcp, oval, nval)
586#endif545#endif
587 546
588#ifndef this_cpu_cmpxchg_double_1
589#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
590 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
591#endif
592#ifndef this_cpu_cmpxchg_double_2
593#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
594 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
595#endif
596#ifndef this_cpu_cmpxchg_double_4
597#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
598 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
599#endif
600#ifndef this_cpu_cmpxchg_double_8
601#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
602 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
603#endif
604
605#endif /* _ASM_GENERIC_PERCPU_H_ */547#endif /* _ASM_GENERIC_PERCPU_H_ */
include/linux/atomic/atomic-instrumented.h+1−16
@@ -2234,21 +2234,6 @@ atomic_long_dec_if_positive(atomic_long_t *v) · 2233 unmodified lines
2234 arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \2234 arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2235})2235})
2236 2236
2237#define cmpxchg_double(ptr, ...) \
2238({ \
2239 typeof(ptr) __ai_ptr = (ptr); \
2240 kcsan_mb(); \
2241 instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2242 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2243})
2244
2245
2246#define cmpxchg_double_local(ptr, ...) \
2247({ \
2248 typeof(ptr) __ai_ptr = (ptr); \
2249 instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2250 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2251})
2252 2237
2253#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */2238#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2254// 82d1be694fab30414527d0877c29fa75ed5a0b742239// 3611991b015450e119bcd7417a9431af7f3ba13c
include/linux/percpu-defs.h+0−38
@@ -358,33 +358,6 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } · 357 unmodified lines
358 pscr2_ret__; \358 pscr2_ret__; \
359})359})
360 360
361/*
362 * Special handling for cmpxchg_double. cmpxchg_double is passed two
363 * percpu variables. The first has to be aligned to a double word
364 * boundary and the second has to follow directly thereafter.
365 * We enforce this on all architectures even if they don't support
366 * a double cmpxchg instruction, since it's a cheap requirement, and it
367 * avoids breaking the requirement for architectures with the instruction.
368 */
369#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
370({ \
371 bool pdcrb_ret__; \
372 __verify_pcpu_ptr(&(pcp1)); \
373 BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
374 VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
375 VM_BUG_ON((unsigned long)(&(pcp2)) != \
376 (unsigned long)(&(pcp1)) + sizeof(pcp1)); \
377 switch(sizeof(pcp1)) { \
378 case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
379 case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
380 case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
381 case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
382 default: \
383 __bad_size_call_parameter(); break; \
384 } \
385 pdcrb_ret__; \
386})
387
388#define __pcpu_size_call(stem, variable, ...) \361#define __pcpu_size_call(stem, variable, ...) \
389do { \362do { \
390 __verify_pcpu_ptr(&(variable)); \363 __verify_pcpu_ptr(&(variable)); \
@@ -443,9 +416,6 @@ do { \ · 52 unmodified lines
443 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)416 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
444#define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \417#define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \
445 __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)418 __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)
446#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
447 __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
448
449#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))419#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
450#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)420#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
451#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)421#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
@@ -505,11 +475,6 @@ do { \ · 53 unmodified lines
505 raw_cpu_cmpxchg(pcp, oval, nval); \475 raw_cpu_cmpxchg(pcp, oval, nval); \
506})476})
507 477
508#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
509({ __this_cpu_preempt_check("cmpxchg_double"); \
510 raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
511})
512
513#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))478#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
514#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)479#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
515#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)480#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
@@ -532,9 +497,6 @@ do { \ · 16 unmodified lines
532 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)497 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
533#define this_cpu_try_cmpxchg(pcp, ovalp, nval) \498#define this_cpu_try_cmpxchg(pcp, ovalp, nval) \
534 __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)499 __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)
535#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
536 __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
537
538#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))500#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
539#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)501#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
540#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)502#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
scripts/atomic/gen-atomic-instrumented.sh+4−11
@@ -84,7 +84,6 @@ gen_xchg() · 83 unmodified lines
84{84{
85 local xchg="$1"; shift85 local xchg="$1"; shift
86 local order="$1"; shift86 local order="$1"; shift
87 local mult="$1"; shift
88 87
89 kcsan_barrier=""88 kcsan_barrier=""
90 if [ "${xchg%_local}" = "${xchg}" ]; then89 if [ "${xchg%_local}" = "${xchg}" ]; then
@@ -104,8 +103,8 @@ cat <<EOF · 13 unmodified lines
104EOF103EOF
105[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"104[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
106cat <<EOF105cat <<EOF
107 instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\106 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
108 instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\107 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\
109 arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\108 arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
110})109})
111EOF110EOF
@@ -168,22 +167,16 @@ done · 56 unmodified lines
168 167
169for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do168for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
170 for order in "" "_acquire" "_release" "_relaxed"; do169 for order in "" "_acquire" "_release" "_relaxed"; do
171 gen_xchg "${xchg}" "${order}" ""170 gen_xchg "${xchg}" "${order}"
172 printf "\n"171 printf "\n"
173 done172 done
174done173done
175 174
176for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do175for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do
177 gen_xchg "${xchg}" "" ""176 gen_xchg "${xchg}" ""
178 printf "\n"177 printf "\n"
179done178done
180 179
181gen_xchg "cmpxchg_double" "" "2 * "
182
183printf "\n\n"
184
185gen_xchg "cmpxchg_double_local" "" "2 * "
186
187cat <<EOF180cat <<EOF
188 181
189#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */182#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t