History

percpu: Wire up cmpxchg128

In order to replace cmpxchg_double() with the newly minted cmpxchg128() family of functions, wire it up in this_cpu_cmpxchg().

Peter Zijlstra committed 3 years ago

Commit 6d12c8d1 parent c5c0ba97 files · +240 −39Browse files

Unified · Split

7 files
  1. arch/arm64/include/asm/percpu.h +20 −0
  2. arch/s390/include/asm/percpu.h +16 −0
  3. arch/x86/include/asm/percpu.h +68 −6
  4. arch/x86/lib/Makefile +2 −1
  5. arch/x86/lib/cmpxchg16b_emu.S +25 −18
  6. arch/x86/lib/cmpxchg8b_emu.S +53 −14
  7. include/asm-generic/percpu.h +56 −0
arch/arm64/include/asm/percpu.h+20−0
@@ -140,6 +140,10 @@ PERCPU_RET_OP(add, add, ldadd) · 139 unmodified lines
140 * re-enabling preemption for preemptible kernels, but doing that in a way140 * re-enabling preemption for preemptible kernels, but doing that in a way
141 * which builds inside a module would mean messing directly with the preempt141 * which builds inside a module would mean messing directly with the preempt
142 * count. If you do this, peterz and tglx will hunt you down.142 * count. If you do this, peterz and tglx will hunt you down.
143 *
144 * Not to mention it'll break the actual preemption model for missing a
145 * preemption point when TIF_NEED_RESCHED gets set while preemption is
146 * disabled.
143 */147 */
144#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \148#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
145({ \149({ \
@@ -240,6 +244,22 @@ PERCPU_RET_OP(add, add, ldadd) · 94 unmodified lines
240#define this_cpu_cmpxchg_8(pcp, o, n) \244#define this_cpu_cmpxchg_8(pcp, o, n) \
241 _pcp_protect_return(cmpxchg_relaxed, pcp, o, n)245 _pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
242 246
247#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
248
249#define this_cpu_cmpxchg128(pcp, o, n) \
250({ \
251 typedef typeof(pcp) pcp_op_T__; \
252 u128 old__, new__, ret__; \
253 pcp_op_T__ *ptr__; \
254 old__ = o; \
255 new__ = n; \
256 preempt_disable_notrace(); \
257 ptr__ = raw_cpu_ptr(&(pcp)); \
258 ret__ = cmpxchg128_local((void *)ptr__, old__, new__); \
259 preempt_enable_notrace(); \
260 ret__; \
261})
262
243#ifdef __KVM_NVHE_HYPERVISOR__263#ifdef __KVM_NVHE_HYPERVISOR__
244extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);264extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
245#define __per_cpu_offset265#define __per_cpu_offset
arch/s390/include/asm/percpu.h+16−0
@@ -148,6 +148,22 @@ · 147 unmodified lines
148#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)148#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)149#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
150 150
151#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
152
153#define this_cpu_cmpxchg128(pcp, oval, nval) \
154({ \
155 typedef typeof(pcp) pcp_op_T__; \
156 u128 old__, new__, ret__; \
157 pcp_op_T__ *ptr__; \
158 old__ = oval; \
159 new__ = nval; \
160 preempt_disable_notrace(); \
161 ptr__ = raw_cpu_ptr(&(pcp)); \
162 ret__ = cmpxchg128((void *)ptr__, old__, new__); \
163 preempt_enable_notrace(); \
164 ret__; \
165})
166
151#define arch_this_cpu_xchg(pcp, nval) \167#define arch_this_cpu_xchg(pcp, nval) \
152({ \168({ \
153 typeof(pcp) *ptr__; \169 typeof(pcp) *ptr__; \
arch/x86/include/asm/percpu.h+68−6
@@ -210,6 +210,67 @@ do { \ · 209 unmodified lines
210 (typeof(_var))(unsigned long) pco_old__; \210 (typeof(_var))(unsigned long) pco_old__; \
211})211})
212 212
213#if defined(CONFIG_X86_32) && !defined(CONFIG_UML)
214#define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \
215({ \
216 union { \
217 u64 var; \
218 struct { \
219 u32 low, high; \
220 }; \
221 } old__, new__; \
222 \
223 old__.var = _oval; \
224 new__.var = _nval; \
225 \
226 asm qual (ALTERNATIVE("leal %P[var], %%esi; call this_cpu_cmpxchg8b_emu", \
227 "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
228 : [var] "+m" (_var), \
229 "+a" (old__.low), \
230 "+d" (old__.high) \
231 : "b" (new__.low), \
232 "c" (new__.high) \
233 : "memory", "esi"); \
234 \
235 old__.var; \
236})
237
238#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval)
239#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
240#endif
241
242#ifdef CONFIG_X86_64
243#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval);
244#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
245
246#define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval) \
247({ \
248 union { \
249 u128 var; \
250 struct { \
251 u64 low, high; \
252 }; \
253 } old__, new__; \
254 \
255 old__.var = _oval; \
256 new__.var = _nval; \
257 \
258 asm qual (ALTERNATIVE("leaq %P[var], %%rsi; call this_cpu_cmpxchg16b_emu", \
259 "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
260 : [var] "+m" (_var), \
261 "+a" (old__.low), \
262 "+d" (old__.high) \
263 : "b" (new__.low), \
264 "c" (new__.high) \
265 : "memory", "rsi"); \
266 \
267 old__.var; \
268})
269
270#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval)
271#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
272#endif
273
213/*274/*
214 * this_cpu_read() makes gcc load the percpu variable every time it is275 * this_cpu_read() makes gcc load the percpu variable every time it is
215 * accessed while this_cpu_read_stable() allows the value to be cached.276 * accessed while this_cpu_read_stable() allows the value to be cached.
@@ -341,12 +402,13 @@ do { \ · 125 unmodified lines
341 bool __ret; \402 bool __ret; \
342 typeof(pcp1) __o1 = (o1), __n1 = (n1); \403 typeof(pcp1) __o1 = (o1), __n1 = (n1); \
343 typeof(pcp2) __o2 = (o2), __n2 = (n2); \404 typeof(pcp2) __o2 = (o2), __n2 = (n2); \
344 alternative_io("leaq %P1,%%rsi\n\tcall this_cpu_cmpxchg16b_emu\n\t", \405 asm volatile (ALTERNATIVE("leaq %P1, %%rsi; call this_cpu_cmpxchg16b_emu", \
345 "cmpxchg16b " __percpu_arg(1) "\n\tsetz %0\n\t", \406 "cmpxchg16b " __percpu_arg(1), X86_FEATURE_CX16) \
346 X86_FEATURE_CX16, \407 "setz %0" \
347 ASM_OUTPUT2("=a" (__ret), "+m" (pcp1), \408 : "=a" (__ret), "+m" (pcp1) \
348 "+m" (pcp2), "+d" (__o2)), \409 : "b" (__n1), "c" (__n2), \
349 "b" (__n1), "c" (__n2), "a" (__o1) : "rsi"); \410 "a" (__o1), "d" (__o2) \
411 : "memory", "rsi"); \
350 __ret; \412 __ret; \
351})413})
352 414
arch/x86/lib/Makefile+2−1
@@ -61,8 +61,9 @@ ifeq ($(CONFIG_X86_32),y) · 60 unmodified lines
61 lib-y += strstr_32.o61 lib-y += strstr_32.o
62 lib-y += string_32.o62 lib-y += string_32.o
63 lib-y += memmove_32.o63 lib-y += memmove_32.o
64 lib-y += cmpxchg8b_emu.o
64ifneq ($(CONFIG_X86_CMPXCHG64),y)65ifneq ($(CONFIG_X86_CMPXCHG64),y)
65 lib-y += cmpxchg8b_emu.o atomic64_386_32.o66 lib-y += atomic64_386_32.o
66endif67endif
67else68else
68 obj-y += iomap_copy_64.o69 obj-y += iomap_copy_64.o
arch/x86/lib/cmpxchg16b_emu.S+25−18
@@ -1,47 +1,54 @@
1/* SPDX-License-Identifier: GPL-2.0-only */1/* SPDX-License-Identifier: GPL-2.0-only */
2#include <linux/linkage.h>2#include <linux/linkage.h>
3#include <asm/percpu.h>3#include <asm/percpu.h>
4#include <asm/processor-flags.h>
4 5
5.text6.text
6 7
7/*8/*
9 * Emulate 'cmpxchg16b %gs:(%rsi)'
10 *
8 * Inputs:11 * Inputs:
9 * %rsi : memory location to compare12 * %rsi : memory location to compare
10 * %rax : low 64 bits of old value13 * %rax : low 64 bits of old value
11 * %rdx : high 64 bits of old value14 * %rdx : high 64 bits of old value
12 * %rbx : low 64 bits of new value15 * %rbx : low 64 bits of new value
13 * %rcx : high 64 bits of new value16 * %rcx : high 64 bits of new value
14 * %al : Operation successful17 *
18 * Notably this is not LOCK prefixed and is not safe against NMIs
15 */19 */
16SYM_FUNC_START(this_cpu_cmpxchg16b_emu)20SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
17 21
18#
19# Emulate 'cmpxchg16b %gs:(%rsi)' except we return the result in %al not
20# via the ZF. Caller will access %al to get result.
21#
22# Note that this is only useful for a cpuops operation. Meaning that we
23# do *not* have a fully atomic operation but just an operation that is
24# *atomic* on a single cpu (as provided by the this_cpu_xx class of
25# macros).
26#
27 pushfq22 pushfq
28 cli23 cli
29 24
30 cmpq PER_CPU_VAR((%rsi)), %rax25 /* if (*ptr == old) */
31 jne .Lnot_same26 cmpq PER_CPU_VAR(0(%rsi)), %rax
32 cmpq PER_CPU_VAR(8(%rsi)), %rdx27 jne .Lnot_same
33 jne .Lnot_same28 cmpq PER_CPU_VAR(8(%rsi)), %rdx
29 jne .Lnot_same
34 30
35 movq %rbx, PER_CPU_VAR((%rsi))31 /* *ptr = new */
36 movq %rcx, PER_CPU_VAR(8(%rsi))32 movq %rbx, PER_CPU_VAR(0(%rsi))
33 movq %rcx, PER_CPU_VAR(8(%rsi))
34
35 /* set ZF in EFLAGS to indicate success */
36 orl $X86_EFLAGS_ZF, (%rsp)
37 37
38 popfq38 popfq
39 mov $1, %al
40 RET39 RET
41 40
42.Lnot_same:41.Lnot_same:
42 /* *ptr != old */
43
44 /* old = *ptr */
45 movq PER_CPU_VAR(0(%rsi)), %rax
46 movq PER_CPU_VAR(8(%rsi)), %rdx
47
48 /* clear ZF in EFLAGS to indicate failure */
49 andl $(~X86_EFLAGS_ZF), (%rsp)
50
43 popfq51 popfq
44 xor %al,%al
45 RET52 RET
46 53
47SYM_FUNC_END(this_cpu_cmpxchg16b_emu)54SYM_FUNC_END(this_cpu_cmpxchg16b_emu)
arch/x86/lib/cmpxchg8b_emu.S+53−14
@@ -2,10 +2,16 @@ · 1 unmodified line
2 2
3#include <linux/linkage.h>3#include <linux/linkage.h>
4#include <asm/export.h>4#include <asm/export.h>
5#include <asm/percpu.h>
6#include <asm/processor-flags.h>
5 7
6.text8.text
7 9
10#ifndef CONFIG_X86_CMPXCHG64
11
8/*12/*
13 * Emulate 'cmpxchg8b (%esi)' on UP
14 *
9 * Inputs:15 * Inputs:
10 * %esi : memory location to compare16 * %esi : memory location to compare
11 * %eax : low 32 bits of old value17 * %eax : low 32 bits of old value
@@ -15,32 +21,65 @@ · 3 unmodified lines
15 */21 */
16SYM_FUNC_START(cmpxchg8b_emu)22SYM_FUNC_START(cmpxchg8b_emu)
17 23
18#
19# Emulate 'cmpxchg8b (%esi)' on UP except we don't
20# set the whole ZF thing (caller will just compare
21# eax:edx with the expected value)
22#
23 pushfl24 pushfl
24 cli25 cli
25 26
26 cmpl (%esi), %eax27 cmpl 0(%esi), %eax
27 jne .Lnot_same28 jne .Lnot_same
28 cmpl 4(%esi), %edx29 cmpl 4(%esi), %edx
29 jne .Lhalf_same30 jne .Lnot_same
31
32 movl %ebx, 0(%esi)
33 movl %ecx, 4(%esi)
30 34
31 movl %ebx, (%esi)35 orl $X86_EFLAGS_ZF, (%esp)
32 movl %ecx, 4(%esi)
33 36
34 popfl37 popfl
35 RET38 RET
36 39
37.Lnot_same:40.Lnot_same:
38 movl (%esi), %eax41 movl 0(%esi), %eax
39.Lhalf_same:42 movl 4(%esi), %edx
40 movl 4(%esi), %edx43
44 andl $(~X86_EFLAGS_ZF), (%esp)
41 45
42 popfl46 popfl
43 RET47 RET
44 48
45SYM_FUNC_END(cmpxchg8b_emu)49SYM_FUNC_END(cmpxchg8b_emu)
46EXPORT_SYMBOL(cmpxchg8b_emu)50EXPORT_SYMBOL(cmpxchg8b_emu)
51
52#endif
53
54#ifndef CONFIG_UML
55
56SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
57
58 pushfl
59 cli
60
61 cmpl PER_CPU_VAR(0(%esi)), %eax
62 jne .Lnot_same2
63 cmpl PER_CPU_VAR(4(%esi)), %edx
64 jne .Lnot_same2
65
66 movl %ebx, PER_CPU_VAR(0(%esi))
67 movl %ecx, PER_CPU_VAR(4(%esi))
68
69 orl $X86_EFLAGS_ZF, (%esp)
70
71 popfl
72 RET
73
74.Lnot_same2:
75 movl PER_CPU_VAR(0(%esi)), %eax
76 movl PER_CPU_VAR(4(%esi)), %edx
77
78 andl $(~X86_EFLAGS_ZF), (%esp)
79
80 popfl
81 RET
82
83SYM_FUNC_END(this_cpu_cmpxchg8b_emu)
84
85#endif
include/asm-generic/percpu.h+56−0
@@ -350,6 +350,25 @@ do { \ · 349 unmodified lines
350#endif350#endif
351#endif351#endif
352 352
353#ifndef raw_cpu_try_cmpxchg64
354#ifdef raw_cpu_cmpxchg64
355#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
356 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64)
357#else
358#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
359 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
360#endif
361#endif
362#ifndef raw_cpu_try_cmpxchg128
363#ifdef raw_cpu_cmpxchg128
364#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
365 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128)
366#else
367#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
368 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
369#endif
370#endif
371
353#ifndef raw_cpu_cmpxchg_1372#ifndef raw_cpu_cmpxchg_1
354#define raw_cpu_cmpxchg_1(pcp, oval, nval) \373#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
355 raw_cpu_generic_cmpxchg(pcp, oval, nval)374 raw_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -367,6 +386,15 @@ do { \ · 11 unmodified lines
367 raw_cpu_generic_cmpxchg(pcp, oval, nval)386 raw_cpu_generic_cmpxchg(pcp, oval, nval)
368#endif387#endif
369 388
389#ifndef raw_cpu_cmpxchg64
390#define raw_cpu_cmpxchg64(pcp, oval, nval) \
391 raw_cpu_generic_cmpxchg(pcp, oval, nval)
392#endif
393#ifndef raw_cpu_cmpxchg128
394#define raw_cpu_cmpxchg128(pcp, oval, nval) \
395 raw_cpu_generic_cmpxchg(pcp, oval, nval)
396#endif
397
370#ifndef raw_cpu_cmpxchg_double_1398#ifndef raw_cpu_cmpxchg_double_1
371#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \399#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
372 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)400 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
@@ -512,6 +540,25 @@ do { \ · 139 unmodified lines
512#endif540#endif
513#endif541#endif
514 542
543#ifndef this_cpu_try_cmpxchg64
544#ifdef this_cpu_cmpxchg64
545#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
546 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64)
547#else
548#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
549 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
550#endif
551#endif
552#ifndef this_cpu_try_cmpxchg128
553#ifdef this_cpu_cmpxchg128
554#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
555 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128)
556#else
557#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
558 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
559#endif
560#endif
561
515#ifndef this_cpu_cmpxchg_1562#ifndef this_cpu_cmpxchg_1
516#define this_cpu_cmpxchg_1(pcp, oval, nval) \563#define this_cpu_cmpxchg_1(pcp, oval, nval) \
517 this_cpu_generic_cmpxchg(pcp, oval, nval)564 this_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -529,6 +576,15 @@ do { \ · 11 unmodified lines
529 this_cpu_generic_cmpxchg(pcp, oval, nval)576 this_cpu_generic_cmpxchg(pcp, oval, nval)
530#endif577#endif
531 578
579#ifndef this_cpu_cmpxchg64
580#define this_cpu_cmpxchg64(pcp, oval, nval) \
581 this_cpu_generic_cmpxchg(pcp, oval, nval)
582#endif
583#ifndef this_cpu_cmpxchg128
584#define this_cpu_cmpxchg128(pcp, oval, nval) \
585 this_cpu_generic_cmpxchg(pcp, oval, nval)
586#endif
587
532#ifndef this_cpu_cmpxchg_double_1588#ifndef this_cpu_cmpxchg_double_1
533#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \589#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
534 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)590 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t