History

arch: Remove cmpxchg_double

No moar users, remove the monster.

Peter Zijlstra committed 3 years ago

Commit febe9501 parent 6801be415 files · +5 −371Browse files

Unified · Split

15 files
  1. Documentation/core-api/this_cpu_ops.rst +0 −2
  2. arch/arm64/include/asm/atomic_ll_sc.h +0 −33
  3. arch/arm64/include/asm/atomic_lse.h +0 −36
  4. arch/arm64/include/asm/cmpxchg.h +0 −46
  5. arch/arm64/include/asm/percpu.h +0 −10
  6. arch/s390/include/asm/cmpxchg.h +0 −34
  7. arch/s390/include/asm/percpu.h +0 −18
  8. arch/x86/include/asm/cmpxchg.h +0 −25
  9. arch/x86/include/asm/cmpxchg_32.h +0 −1
  10. arch/x86/include/asm/cmpxchg_64.h +0 −1
  11. arch/x86/include/asm/percpu.h +0 −42
  12. include/asm-generic/percpu.h +0 −58
  13. include/linux/atomic/atomic-instrumented.h +1 −16
  14. include/linux/percpu-defs.h +0 −38
  15. scripts/atomic/gen-atomic-instrumented.sh +4 −11
Documentation/core-api/this_cpu_ops.rst+0−2
@@ -53,7 +53,6 @@ preemption and interrupts:: · 52 unmodified lines
5353 this_cpu_add_return(pcp, val)
5454 this_cpu_xchg(pcp, nval)
5555 this_cpu_cmpxchg(pcp, oval, nval)
56− this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
5756 this_cpu_sub(pcp, val)
5857 this_cpu_inc(pcp)
5958 this_cpu_dec(pcp)
@@ -242,7 +241,6 @@ safe:: · 182 unmodified lines
242241 __this_cpu_add_return(pcp, val)
243242 __this_cpu_xchg(pcp, nval)
244243 __this_cpu_cmpxchg(pcp, oval, nval)
245− __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
246244 __this_cpu_sub(pcp, val)
247245 __this_cpu_inc(pcp)
248246 __this_cpu_dec(pcp)
arch/arm64/include/asm/atomic_ll_sc.h+0−33
@@ -294,39 +294,6 @@ __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) · 293 unmodified lines
294294
295295 #undef __CMPXCHG_CASE
296296
297−#define __CMPXCHG_DBL(name, mb, rel, cl) \
298−static __always_inline long \
299−__ll_sc__cmpxchg_double##name(unsigned long old1, \
300− unsigned long old2, \
301− unsigned long new1, \
302− unsigned long new2, \
303− volatile void *ptr) \
304−{ \
305− unsigned long tmp, ret; \
306− \
307− asm volatile("// __cmpxchg_double" #name "\n" \
308− " prfm pstl1strm, %2\n" \
309− "1: ldxp %0, %1, %2\n" \
310− " eor %0, %0, %3\n" \
311− " eor %1, %1, %4\n" \
312− " orr %1, %0, %1\n" \
313− " cbnz %1, 2f\n" \
314− " st" #rel "xp %w0, %5, %6, %2\n" \
315− " cbnz %w0, 1b\n" \
316− " " #mb "\n" \
317− "2:" \
318− : "=&r" (tmp), "=&r" (ret), "+Q" (*(__uint128_t *)ptr) \
319− : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \
320− : cl); \
321− \
322− return ret; \
323−}
324−
325−__CMPXCHG_DBL( , , , )
326−__CMPXCHG_DBL(_mb, dmb ish, l, "memory")
327−
328−#undef __CMPXCHG_DBL
329−
330297 union __u128_halves {
331298 u128 full;
332299 struct {
arch/arm64/include/asm/atomic_lse.h+0−36
@@ -281,42 +281,6 @@ __CMPXCHG_CASE(x, , mb_, 64, al, "memory") · 280 unmodified lines
281281
282282 #undef __CMPXCHG_CASE
283283
284−#define __CMPXCHG_DBL(name, mb, cl...) \
285−static __always_inline long \
286−__lse__cmpxchg_double##name(unsigned long old1, \
287− unsigned long old2, \
288− unsigned long new1, \
289− unsigned long new2, \
290− volatile void *ptr) \
291−{ \
292− unsigned long oldval1 = old1; \
293− unsigned long oldval2 = old2; \
294− register unsigned long x0 asm ("x0") = old1; \
295− register unsigned long x1 asm ("x1") = old2; \
296− register unsigned long x2 asm ("x2") = new1; \
297− register unsigned long x3 asm ("x3") = new2; \
298− register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
299− \
300− asm volatile( \
301− __LSE_PREAMBLE \
302− " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
303− " eor %[old1], %[old1], %[oldval1]\n" \
304− " eor %[old2], %[old2], %[oldval2]\n" \
305− " orr %[old1], %[old1], %[old2]" \
306− : [old1] "+&r" (x0), [old2] "+&r" (x1), \
307− [v] "+Q" (*(__uint128_t *)ptr) \
308− : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
309− [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
310− : cl); \
311− \
312− return x0; \
313−}
314−
315−__CMPXCHG_DBL( , )
316−__CMPXCHG_DBL(_mb, al, "memory")
317−
318−#undef __CMPXCHG_DBL
319−
320284 #define __CMPXCHG128(name, mb, cl...) \
321285 static __always_inline u128 \
322286 __lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \
arch/arm64/include/asm/cmpxchg.h+0−46
@@ -130,22 +130,6 @@ __CMPXCHG_CASE(mb_, 64) · 129 unmodified lines
130130
131131 #undef __CMPXCHG_CASE
132132
133−#define __CMPXCHG_DBL(name) \
134−static inline long __cmpxchg_double##name(unsigned long old1, \
135− unsigned long old2, \
136− unsigned long new1, \
137− unsigned long new2, \
138− volatile void *ptr) \
139−{ \
140− return __lse_ll_sc_body(_cmpxchg_double##name, \
141− old1, old2, new1, new2, ptr); \
142−}
143−
144−__CMPXCHG_DBL( )
145−__CMPXCHG_DBL(_mb)
146−
147−#undef __CMPXCHG_DBL
148−
149133 #define __CMPXCHG128(name) \
150134 static inline u128 __cmpxchg128##name(volatile u128 *ptr, \
151135 u128 old, u128 new) \
@@ -211,36 +195,6 @@ __CMPXCHG_GEN(_mb) · 59 unmodified lines
211195 #define arch_cmpxchg64 arch_cmpxchg
212196 #define arch_cmpxchg64_local arch_cmpxchg_local
213197
214−/* cmpxchg_double */
215−#define system_has_cmpxchg_double() 1
216−
217−#define __cmpxchg_double_check(ptr1, ptr2) \
218−({ \
219− if (sizeof(*(ptr1)) != 8) \
220− BUILD_BUG(); \
221− VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
222−})
223−
224−#define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
225−({ \
226− int __ret; \
227− __cmpxchg_double_check(ptr1, ptr2); \
228− __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
229− (unsigned long)(n1), (unsigned long)(n2), \
230− ptr1); \
231− __ret; \
232−})
233−
234−#define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
235−({ \
236− int __ret; \
237− __cmpxchg_double_check(ptr1, ptr2); \
238− __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
239− (unsigned long)(n1), (unsigned long)(n2), \
240− ptr1); \
241− __ret; \
242−})
243−
244198 /* cmpxchg128 */
245199 #define system_has_cmpxchg128() 1
246200
arch/arm64/include/asm/percpu.h+0−10
@@ -145,16 +145,6 @@ PERCPU_RET_OP(add, add, ldadd) · 144 unmodified lines
145145 * preemption point when TIF_NEED_RESCHED gets set while preemption is
146146 * disabled.
147147 */
148−#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
149−({ \
150− int __ret; \
151− preempt_disable_notrace(); \
152− __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
153− raw_cpu_ptr(&(ptr2)), \
154− o1, o2, n1, n2); \
155− preempt_enable_notrace(); \
156− __ret; \
157−})
158148
159149 #define _pcp_protect(op, pcp, ...) \
160150 ({ \
arch/s390/include/asm/cmpxchg.h+0−34
@@ -190,40 +190,6 @@ static __always_inline unsigned long __cmpxchg(unsigned long address, · 189 unmodified lines
190190 #define arch_cmpxchg_local arch_cmpxchg
191191 #define arch_cmpxchg64_local arch_cmpxchg
192192
193−#define system_has_cmpxchg_double() 1
194−
195−static __always_inline int __cmpxchg_double(unsigned long p1, unsigned long p2,
196− unsigned long o1, unsigned long o2,
197− unsigned long n1, unsigned long n2)
198−{
199− union register_pair old = { .even = o1, .odd = o2, };
200− union register_pair new = { .even = n1, .odd = n2, };
201− int cc;
202−
203− asm volatile(
204− " cdsg %[old],%[new],%[ptr]\n"
205− " ipm %[cc]\n"
206− " srl %[cc],28\n"
207− : [cc] "=&d" (cc), [old] "+&d" (old.pair)
208− : [new] "d" (new.pair),
209− [ptr] "QS" (*(unsigned long *)p1), "Q" (*(unsigned long *)p2)
210− : "memory", "cc");
211− return !cc;
212−}
213−
214−#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \
215−({ \
216− typeof(p1) __p1 = (p1); \
217− typeof(p2) __p2 = (p2); \
218− \
219− BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
220− BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
221− VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\
222− __cmpxchg_double((unsigned long)__p1, (unsigned long)__p2, \
223− (unsigned long)(o1), (unsigned long)(o2), \
224− (unsigned long)(n1), (unsigned long)(n2)); \
225−})
226−
227193 #define system_has_cmpxchg128() 1
228194
229195 static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)
arch/s390/include/asm/percpu.h+0−18
@@ -180,24 +180,6 @@ · 179 unmodified lines
180180 #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
181181 #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
182182
183−#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \
184−({ \
185− typeof(pcp1) *p1__; \
186− typeof(pcp2) *p2__; \
187− int ret__; \
188− \
189− preempt_disable_notrace(); \
190− p1__ = raw_cpu_ptr(&(pcp1)); \
191− p2__ = raw_cpu_ptr(&(pcp2)); \
192− ret__ = __cmpxchg_double((unsigned long)p1__, (unsigned long)p2__, \
193− (unsigned long)(o1), (unsigned long)(o2), \
194− (unsigned long)(n1), (unsigned long)(n2)); \
195− preempt_enable_notrace(); \
196− ret__; \
197−})
198−
199−#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
200−
201183 #include <asm-generic/percpu.h>
202184
203185 #endif /* __ARCH_S390_PERCPU__ */
arch/x86/include/asm/cmpxchg.h+0−25
@@ -239,29 +239,4 @@ extern void __add_wrong_size(void) · 238 unmodified lines
239239 #define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
240240 #define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
241241
242−#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
243−({ \
244− bool __ret; \
245− __typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \
246− __typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \
247− BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
248− BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
249− VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \
250− VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \
251− asm volatile(pfx "cmpxchg%c5b %1" \
252− CC_SET(e) \
253− : CC_OUT(e) (__ret), \
254− "+m" (*(p1)), "+m" (*(p2)), \
255− "+a" (__old1), "+d" (__old2) \
256− : "i" (2 * sizeof(long)), \
257− "b" (__new1), "c" (__new2)); \
258− __ret; \
259−})
260−
261−#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \
262− __cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)
263−
264−#define arch_cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
265− __cmpxchg_double(, p1, p2, o1, o2, n1, n2)
266−
267242 #endif /* ASM_X86_CMPXCHG_H */
arch/x86/include/asm/cmpxchg_32.h+0−1
@@ -103,7 +103,6 @@ static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new) · 102 unmodified lines
103103
104104 #endif
105105
106−#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)
107106 #define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8)
108107
109108 #endif /* _ASM_X86_CMPXCHG_32_H */
arch/x86/include/asm/cmpxchg_64.h+0−1
@@ -81,7 +81,6 @@ static __always_inline bool arch_try_cmpxchg128_local(volatile u128 *ptr, u128 * · 80 unmodified lines
8181 return __arch_try_cmpxchg128(ptr, oldp, new,);
8282 }
8383
84−#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX16)
8584 #define system_has_cmpxchg128() boot_cpu_has(X86_FEATURE_CX16)
8685
8786 #endif /* _ASM_X86_CMPXCHG_64_H */
arch/x86/include/asm/percpu.h+0−42
@@ -351,23 +351,6 @@ do { \ · 350 unmodified lines
351351 #define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
352352 #define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
353353
354−#ifdef CONFIG_X86_CMPXCHG64
355−#define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2) \
356−({ \
357− bool __ret; \
358− typeof(pcp1) __o1 = (o1), __n1 = (n1); \
359− typeof(pcp2) __o2 = (o2), __n2 = (n2); \
360− asm volatile("cmpxchg8b "__percpu_arg(1) \
361− CC_SET(z) \
362− : CC_OUT(z) (__ret), "+m" (pcp1), "+m" (pcp2), "+a" (__o1), "+d" (__o2) \
363− : "b" (__n1), "c" (__n2)); \
364− __ret; \
365−})
366−
367−#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
368−#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double
369−#endif /* CONFIG_X86_CMPXCHG64 */
370−
371354 /*
372355 * Per cpu atomic 64 bit operations are only available under 64 bit.
373356 * 32 bit must fall back to generic operations.
@@ -390,31 +373,6 @@ do { \ · 16 unmodified lines
390373 #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
391374 #define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(8, volatile, pcp, nval)
392375 #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
393−
394−/*
395− * Pretty complex macro to generate cmpxchg16 instruction. The instruction
396− * is not supported on early AMD64 processors so we must be able to emulate
397− * it in software. The address used in the cmpxchg16 instruction must be
398− * aligned to a 16 byte boundary.
399− */
400−#define percpu_cmpxchg16b_double(pcp1, pcp2, o1, o2, n1, n2) \
401−({ \
402− bool __ret; \
403− typeof(pcp1) __o1 = (o1), __n1 = (n1); \
404− typeof(pcp2) __o2 = (o2), __n2 = (n2); \
405− asm volatile (ALTERNATIVE("leaq %P1, %%rsi; call this_cpu_cmpxchg16b_emu", \
406− "cmpxchg16b " __percpu_arg(1), X86_FEATURE_CX16) \
407− "setz %0" \
408− : "=a" (__ret), "+m" (pcp1) \
409− : "b" (__n1), "c" (__n2), \
410− "a" (__o1), "d" (__o2) \
411− : "memory", "rsi"); \
412− __ret; \
413−})
414−
415−#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
416−#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double
417−
418376 #endif
419377
420378 static __always_inline bool x86_this_cpu_constant_test_bit(unsigned int nr,
include/asm-generic/percpu.h+0−58
@@ -120,19 +120,6 @@ do { \ · 119 unmodified lines
120120 __old; \
121121 })
122122
123−#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
124−({ \
125− typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \
126− typeof(pcp2) *__p2 = raw_cpu_ptr(&(pcp2)); \
127− int __ret = 0; \
128− if (*__p1 == (oval1) && *__p2 == (oval2)) { \
129− *__p1 = nval1; \
130− *__p2 = nval2; \
131− __ret = 1; \
132− } \
133− (__ret); \
134−})
135−
136123 #define __this_cpu_generic_read_nopreempt(pcp) \
137124 ({ \
138125 typeof(pcp) ___ret; \
@@ -211,17 +198,6 @@ do { \ · 72 unmodified lines
211198 __ret; \
212199 })
213200
214−#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
215−({ \
216− int __ret; \
217− unsigned long __flags; \
218− raw_local_irq_save(__flags); \
219− __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
220− oval1, oval2, nval1, nval2); \
221− raw_local_irq_restore(__flags); \
222− __ret; \
223−})
224−
225201 #ifndef raw_cpu_read_1
226202 #define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)
227203 #endif
@@ -395,23 +371,6 @@ do { \ · 167 unmodified lines
395371 raw_cpu_generic_cmpxchg(pcp, oval, nval)
396372 #endif
397373
398−#ifndef raw_cpu_cmpxchg_double_1
399−#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
400− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
401−#endif
402−#ifndef raw_cpu_cmpxchg_double_2
403−#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
404− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
405−#endif
406−#ifndef raw_cpu_cmpxchg_double_4
407−#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
408− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
409−#endif
410−#ifndef raw_cpu_cmpxchg_double_8
411−#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
412− raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
413−#endif
414−
415374 #ifndef this_cpu_read_1
416375 #define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
417376 #endif
@@ -585,21 +544,4 @@ do { \ · 167 unmodified lines
585544 this_cpu_generic_cmpxchg(pcp, oval, nval)
586545 #endif
587546
588−#ifndef this_cpu_cmpxchg_double_1
589−#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
590− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
591−#endif
592−#ifndef this_cpu_cmpxchg_double_2
593−#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
594− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
595−#endif
596−#ifndef this_cpu_cmpxchg_double_4
597−#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
598− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
599−#endif
600−#ifndef this_cpu_cmpxchg_double_8
601−#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
602− this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
603−#endif
604−
605547 #endif /* _ASM_GENERIC_PERCPU_H_ */
include/linux/atomic/atomic-instrumented.h+1−16
@@ -2234,21 +2234,6 @@ atomic_long_dec_if_positive(atomic_long_t *v) · 2233 unmodified lines
22342234 arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
22352235 })
22362236
2237−#define cmpxchg_double(ptr, ...) \
2238−({ \
2239− typeof(ptr) __ai_ptr = (ptr); \
2240− kcsan_mb(); \
2241− instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2242− arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2243−})
2244−
2245−
2246−#define cmpxchg_double_local(ptr, ...) \
2247−({ \
2248− typeof(ptr) __ai_ptr = (ptr); \
2249− instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2250− arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2251−})
22522237
22532238 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2254−// 82d1be694fab30414527d0877c29fa75ed5a0b74
2239+// 3611991b015450e119bcd7417a9431af7f3ba13c
include/linux/percpu-defs.h+0−38
@@ -358,33 +358,6 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } · 357 unmodified lines
358358 pscr2_ret__; \
359359 })
360360
361−/*
362− * Special handling for cmpxchg_double. cmpxchg_double is passed two
363− * percpu variables. The first has to be aligned to a double word
364− * boundary and the second has to follow directly thereafter.
365− * We enforce this on all architectures even if they don't support
366− * a double cmpxchg instruction, since it's a cheap requirement, and it
367− * avoids breaking the requirement for architectures with the instruction.
368− */
369−#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
370−({ \
371− bool pdcrb_ret__; \
372− __verify_pcpu_ptr(&(pcp1)); \
373− BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
374− VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
375− VM_BUG_ON((unsigned long)(&(pcp2)) != \
376− (unsigned long)(&(pcp1)) + sizeof(pcp1)); \
377− switch(sizeof(pcp1)) { \
378− case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
379− case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
380− case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
381− case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
382− default: \
383− __bad_size_call_parameter(); break; \
384− } \
385− pdcrb_ret__; \
386−})
387−
388361 #define __pcpu_size_call(stem, variable, ...) \
389362 do { \
390363 __verify_pcpu_ptr(&(variable)); \
@@ -443,9 +416,6 @@ do { \ · 52 unmodified lines
443416 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
444417 #define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \
445418 __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)
446−#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
447− __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
448−
449419 #define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
450420 #define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
451421 #define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
@@ -505,11 +475,6 @@ do { \ · 53 unmodified lines
505475 raw_cpu_cmpxchg(pcp, oval, nval); \
506476 })
507477
508−#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
509−({ __this_cpu_preempt_check("cmpxchg_double"); \
510− raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
511−})
512−
513478 #define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
514479 #define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
515480 #define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
@@ -532,9 +497,6 @@ do { \ · 16 unmodified lines
532497 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
533498 #define this_cpu_try_cmpxchg(pcp, ovalp, nval) \
534499 __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)
535−#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
536− __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
537−
538500 #define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
539501 #define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
540502 #define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
scripts/atomic/gen-atomic-instrumented.sh+4−11
@@ -84,7 +84,6 @@ gen_xchg() · 83 unmodified lines
8484 {
8585 local xchg="$1"; shift
8686 local order="$1"; shift
87− local mult="$1"; shift
8887
8988 kcsan_barrier=""
9089 if [ "${xchg%_local}" = "${xchg}" ]; then
@@ -104,8 +103,8 @@ cat <<EOF · 13 unmodified lines
104103 EOF
105104 [ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
106105 cat <<EOF
107− instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
108− instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
106+ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
107+ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\
109108 arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
110109 })
111110 EOF
@@ -168,22 +167,16 @@ done · 56 unmodified lines
168167
169168 for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
170169 for order in "" "_acquire" "_release" "_relaxed"; do
171− gen_xchg "${xchg}" "${order}" ""
170+ gen_xchg "${xchg}" "${order}"
172171 printf "\n"
173172 done
174173 done
175174
176175 for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do
177− gen_xchg "${xchg}" "" ""
176+ gen_xchg "${xchg}" ""
178177 printf "\n"
179178 done
180179
181−gen_xchg "cmpxchg_double" "" "2 * "
182−
183−printf "\n\n"
184−
185−gen_xchg "cmpxchg_double_local" "" "2 * "
186−
187180 cat <<EOF
188181
189182 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t