percpu: Wire up cmpxchg128

In order to replace cmpxchg_double() with the newly minted cmpxchg128() family of functions, wire it up in this_cpu_cmpxchg().

Peter Zijlstra committed 3 years ago

Commit 6d12c8d1 parent c5c0ba97 files · +240 −39Browse files

Unified · Split

7 files

arch/arm64/include/asm/percpu.h +20 −0
arch/s390/include/asm/percpu.h +16 −0
arch/x86/include/asm/percpu.h +68 −6
arch/x86/lib/Makefile +2 −1
arch/x86/lib/cmpxchg16b_emu.S +25 −18
arch/x86/lib/cmpxchg8b_emu.S +53 −14
include/asm-generic/percpu.h +56 −0

arch/arm64/include/asm/percpu.h+20−0

@@ -140,6 +140,10 @@ PERCPU_RET_OP(add, add, ldadd) · 139 unmodified lines
140	140	* re-enabling preemption for preemptible kernels, but doing that in a way
141	141	* which builds inside a module would mean messing directly with the preempt
142	142	* count. If you do this, peterz and tglx will hunt you down.
	143	+ *
	144	+ * Not to mention it'll break the actual preemption model for missing a
	145	+ * preemption point when TIF_NEED_RESCHED gets set while preemption is
	146	+ * disabled.
143	147	*/
144	148	#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
145	149	({ \
@@ -240,6 +244,22 @@ PERCPU_RET_OP(add, add, ldadd) · 94 unmodified lines
240	244	#define this_cpu_cmpxchg_8(pcp, o, n) \
241	245	_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
242	246
	247	+#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
	248	+
	249	+#define this_cpu_cmpxchg128(pcp, o, n) \
	250	+({ \
	251	+ typedef typeof(pcp) pcp_op_T__; \
	252	+ u128 old__, new__, ret__; \
	253	+ pcp_op_T__ *ptr__; \
	254	+ old__ = o; \
	255	+ new__ = n; \
	256	+ preempt_disable_notrace(); \
	257	+ ptr__ = raw_cpu_ptr(&(pcp)); \
	258	+ ret__ = cmpxchg128_local((void *)ptr__, old__, new__); \
	259	+ preempt_enable_notrace(); \
	260	+ ret__; \
	261	+})
	262	+
243	263	#ifdef __KVM_NVHE_HYPERVISOR__
244	264	extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
245	265	#define __per_cpu_offset

arch/s390/include/asm/percpu.h+16−0

@@ -148,6 +148,22 @@ · 147 unmodified lines
148	148	#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149	149	#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
150	150
	151	+#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
	152	+
	153	+#define this_cpu_cmpxchg128(pcp, oval, nval) \
	154	+({ \
	155	+ typedef typeof(pcp) pcp_op_T__; \
	156	+ u128 old__, new__, ret__; \
	157	+ pcp_op_T__ *ptr__; \
	158	+ old__ = oval; \
	159	+ new__ = nval; \
	160	+ preempt_disable_notrace(); \
	161	+ ptr__ = raw_cpu_ptr(&(pcp)); \
	162	+ ret__ = cmpxchg128((void *)ptr__, old__, new__); \
	163	+ preempt_enable_notrace(); \
	164	+ ret__; \
	165	+})
	166	+
151	167	#define arch_this_cpu_xchg(pcp, nval) \
152	168	({ \
153	169	typeof(pcp) *ptr__; \

arch/x86/include/asm/percpu.h+68−6

@@ -210,6 +210,67 @@ do { \ · 209 unmodified lines
210	210	(typeof(_var))(unsigned long) pco_old__; \
211	211	})
212	212
	213	+#if defined(CONFIG_X86_32) && !defined(CONFIG_UML)
	214	+#define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \
	215	+({ \
	216	+ union { \
	217	+ u64 var; \
	218	+ struct { \
	219	+ u32 low, high; \
	220	+ }; \
	221	+ } old__, new__; \
	222	+ \
	223	+ old__.var = _oval; \
	224	+ new__.var = _nval; \
	225	+ \
	226	+ asm qual (ALTERNATIVE("leal %P[var], %%esi; call this_cpu_cmpxchg8b_emu", \
	227	+ "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
	228	+ : [var] "+m" (_var), \
	229	+ "+a" (old__.low), \
	230	+ "+d" (old__.high) \
	231	+ : "b" (new__.low), \
	232	+ "c" (new__.high) \
	233	+ : "memory", "esi"); \
	234	+ \
	235	+ old__.var; \
	236	+})
	237	+
	238	+#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval)
	239	+#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
	240	+#endif
	241	+
	242	+#ifdef CONFIG_X86_64
	243	+#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval);
	244	+#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
	245	+
	246	+#define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval) \
	247	+({ \
	248	+ union { \
	249	+ u128 var; \
	250	+ struct { \
	251	+ u64 low, high; \
	252	+ }; \
	253	+ } old__, new__; \
	254	+ \
	255	+ old__.var = _oval; \
	256	+ new__.var = _nval; \
	257	+ \
	258	+ asm qual (ALTERNATIVE("leaq %P[var], %%rsi; call this_cpu_cmpxchg16b_emu", \
	259	+ "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
	260	+ : [var] "+m" (_var), \
	261	+ "+a" (old__.low), \
	262	+ "+d" (old__.high) \
	263	+ : "b" (new__.low), \
	264	+ "c" (new__.high) \
	265	+ : "memory", "rsi"); \
	266	+ \
	267	+ old__.var; \
	268	+})
	269	+
	270	+#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval)
	271	+#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
	272	+#endif
	273	+
213	274	/*
214	275	* this_cpu_read() makes gcc load the percpu variable every time it is
215	276	* accessed while this_cpu_read_stable() allows the value to be cached.
@@ -341,12 +402,13 @@ do { \ · 125 unmodified lines
341	402	bool __ret; \
342	403	typeof(pcp1) __o1 = (o1), __n1 = (n1); \
343	404	typeof(pcp2) __o2 = (o2), __n2 = (n2); \
344		− alternative_io("leaq %P1,%%rsi\n\tcall this_cpu_cmpxchg16b_emu\n\t", \
345		− "cmpxchg16b " __percpu_arg(1) "\n\tsetz %0\n\t", \
346		− X86_FEATURE_CX16, \
347		− ASM_OUTPUT2("=a" (__ret), "+m" (pcp1), \
348		− "+m" (pcp2), "+d" (__o2)), \
349		− "b" (__n1), "c" (__n2), "a" (__o1) : "rsi"); \
	405	+ asm volatile (ALTERNATIVE("leaq %P1, %%rsi; call this_cpu_cmpxchg16b_emu", \
	406	+ "cmpxchg16b " __percpu_arg(1), X86_FEATURE_CX16) \
	407	+ "setz %0" \
	408	+ : "=a" (__ret), "+m" (pcp1) \
	409	+ : "b" (__n1), "c" (__n2), \
	410	+ "a" (__o1), "d" (__o2) \
	411	+ : "memory", "rsi"); \
350	412	__ret; \
351	413	})
352	414

arch/x86/lib/Makefile+2−1

@@ -61,8 +61,9 @@ ifeq ($(CONFIG_X86_32),y) · 60 unmodified lines
61	61	lib-y += strstr_32.o
62	62	lib-y += string_32.o
63	63	lib-y += memmove_32.o
	64	+ lib-y += cmpxchg8b_emu.o
64	65	ifneq ($(CONFIG_X86_CMPXCHG64),y)
65		− lib-y += cmpxchg8b_emu.o atomic64_386_32.o
	66	+ lib-y += atomic64_386_32.o
66	67	endif
67	68	else
68	69	obj-y += iomap_copy_64.o

arch/x86/lib/cmpxchg16b_emu.S+25−18

@@ -1,47 +1,54 @@
1	1	/* SPDX-License-Identifier: GPL-2.0-only */
2	2	#include <linux/linkage.h>
3	3	#include <asm/percpu.h>
	4	+#include <asm/processor-flags.h>
4	5
5	6	.text
6	7
7	8	/*
	9	+ * Emulate 'cmpxchg16b %gs:(%rsi)'
	10	+ *
8	11	* Inputs:
9	12	* %rsi : memory location to compare
10	13	* %rax : low 64 bits of old value
11	14	* %rdx : high 64 bits of old value
12	15	* %rbx : low 64 bits of new value
13	16	* %rcx : high 64 bits of new value
14		− * %al : Operation successful
	17	+ *
	18	+ * Notably this is not LOCK prefixed and is not safe against NMIs
15	19	*/
16	20	SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
17	21
18		−#
19		−# Emulate 'cmpxchg16b %gs:(%rsi)' except we return the result in %al not
20		−# via the ZF. Caller will access %al to get result.
21		−#
22		−# Note that this is only useful for a cpuops operation. Meaning that we
23		−# do not have a fully atomic operation but just an operation that is
24		−# atomic on a single cpu (as provided by the this_cpu_xx class of
25		−# macros).
26		−#
27	22	pushfq
28	23	cli
29	24
30		− cmpq PER_CPU_VAR((%rsi)), %rax
31		− jne .Lnot_same
32		− cmpq PER_CPU_VAR(8(%rsi)), %rdx
33		− jne .Lnot_same
	25	+ /* if (ptr == old) /
	26	+ cmpq PER_CPU_VAR(0(%rsi)), %rax
	27	+ jne .Lnot_same
	28	+ cmpq PER_CPU_VAR(8(%rsi)), %rdx
	29	+ jne .Lnot_same
34	30
35		− movq %rbx, PER_CPU_VAR((%rsi))
36		− movq %rcx, PER_CPU_VAR(8(%rsi))
	31	+ /* ptr = new /
	32	+ movq %rbx, PER_CPU_VAR(0(%rsi))
	33	+ movq %rcx, PER_CPU_VAR(8(%rsi))
	34	+
	35	+ /* set ZF in EFLAGS to indicate success */
	36	+ orl $X86_EFLAGS_ZF, (%rsp)
37	37
38	38	popfq
39		− mov $1, %al
40	39	RET
41	40
42	41	.Lnot_same:
	42	+ /* ptr != old /
	43	+
	44	+ /* old = ptr /
	45	+ movq PER_CPU_VAR(0(%rsi)), %rax
	46	+ movq PER_CPU_VAR(8(%rsi)), %rdx
	47	+
	48	+ /* clear ZF in EFLAGS to indicate failure */
	49	+ andl $(~X86_EFLAGS_ZF), (%rsp)
	50	+
43	51	popfq
44		− xor %al,%al
45	52	RET
46	53
47	54	SYM_FUNC_END(this_cpu_cmpxchg16b_emu)

arch/x86/lib/cmpxchg8b_emu.S+53−14

@@ -2,10 +2,16 @@ · 1 unmodified line
2	2
3	3	#include <linux/linkage.h>
4	4	#include <asm/export.h>
	5	+#include <asm/percpu.h>
	6	+#include <asm/processor-flags.h>
5	7
6	8	.text
7	9
	10	+#ifndef CONFIG_X86_CMPXCHG64
	11	+
8	12	/*
	13	+ * Emulate 'cmpxchg8b (%esi)' on UP
	14	+ *
9	15	* Inputs:
10	16	* %esi : memory location to compare
11	17	* %eax : low 32 bits of old value
@@ -15,32 +21,65 @@ · 3 unmodified lines
15	21	*/
16	22	SYM_FUNC_START(cmpxchg8b_emu)
17	23
18		−#
19		−# Emulate 'cmpxchg8b (%esi)' on UP except we don't
20		−# set the whole ZF thing (caller will just compare
21		−# eax:edx with the expected value)
22		−#
23	24	pushfl
24	25	cli
25	26
26		− cmpl (%esi), %eax
27		− jne .Lnot_same
28		− cmpl 4(%esi), %edx
29		− jne .Lhalf_same
	27	+ cmpl 0(%esi), %eax
	28	+ jne .Lnot_same
	29	+ cmpl 4(%esi), %edx
	30	+ jne .Lnot_same
	31	+
	32	+ movl %ebx, 0(%esi)
	33	+ movl %ecx, 4(%esi)
30	34
31		− movl %ebx, (%esi)
32		− movl %ecx, 4(%esi)
	35	+ orl $X86_EFLAGS_ZF, (%esp)
33	36
34	37	popfl
35	38	RET
36	39
37	40	.Lnot_same:
38		− movl (%esi), %eax
39		−.Lhalf_same:
40		− movl 4(%esi), %edx
	41	+ movl 0(%esi), %eax
	42	+ movl 4(%esi), %edx
	43	+
	44	+ andl $(~X86_EFLAGS_ZF), (%esp)
41	45
42	46	popfl
43	47	RET
44	48
45	49	SYM_FUNC_END(cmpxchg8b_emu)
46	50	EXPORT_SYMBOL(cmpxchg8b_emu)
	51	+
	52	+#endif
	53	+
	54	+#ifndef CONFIG_UML
	55	+
	56	+SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
	57	+
	58	+ pushfl
	59	+ cli
	60	+
	61	+ cmpl PER_CPU_VAR(0(%esi)), %eax
	62	+ jne .Lnot_same2
	63	+ cmpl PER_CPU_VAR(4(%esi)), %edx
	64	+ jne .Lnot_same2
	65	+
	66	+ movl %ebx, PER_CPU_VAR(0(%esi))
	67	+ movl %ecx, PER_CPU_VAR(4(%esi))
	68	+
	69	+ orl $X86_EFLAGS_ZF, (%esp)
	70	+
	71	+ popfl
	72	+ RET
	73	+
	74	+.Lnot_same2:
	75	+ movl PER_CPU_VAR(0(%esi)), %eax
	76	+ movl PER_CPU_VAR(4(%esi)), %edx
	77	+
	78	+ andl $(~X86_EFLAGS_ZF), (%esp)
	79	+
	80	+ popfl
	81	+ RET
	82	+
	83	+SYM_FUNC_END(this_cpu_cmpxchg8b_emu)
	84	+
	85	+#endif

include/asm-generic/percpu.h+56−0

@@ -350,6 +350,25 @@ do { \ · 349 unmodified lines
350	350	#endif
351	351	#endif
352	352
	353	+#ifndef raw_cpu_try_cmpxchg64
	354	+#ifdef raw_cpu_cmpxchg64
	355	+#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	356	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64)
	357	+#else
	358	+#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	359	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	360	+#endif
	361	+#endif
	362	+#ifndef raw_cpu_try_cmpxchg128
	363	+#ifdef raw_cpu_cmpxchg128
	364	+#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	365	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128)
	366	+#else
	367	+#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	368	+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	369	+#endif
	370	+#endif
	371	+
353	372	#ifndef raw_cpu_cmpxchg_1
354	373	#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
355	374	raw_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -367,6 +386,15 @@ do { \ · 11 unmodified lines
367	386	raw_cpu_generic_cmpxchg(pcp, oval, nval)
368	387	#endif
369	388
	389	+#ifndef raw_cpu_cmpxchg64
	390	+#define raw_cpu_cmpxchg64(pcp, oval, nval) \
	391	+ raw_cpu_generic_cmpxchg(pcp, oval, nval)
	392	+#endif
	393	+#ifndef raw_cpu_cmpxchg128
	394	+#define raw_cpu_cmpxchg128(pcp, oval, nval) \
	395	+ raw_cpu_generic_cmpxchg(pcp, oval, nval)
	396	+#endif
	397	+
370	398	#ifndef raw_cpu_cmpxchg_double_1
371	399	#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
372	400	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
@@ -512,6 +540,25 @@ do { \ · 139 unmodified lines
512	540	#endif
513	541	#endif
514	542
	543	+#ifndef this_cpu_try_cmpxchg64
	544	+#ifdef this_cpu_cmpxchg64
	545	+#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	546	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64)
	547	+#else
	548	+#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
	549	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	550	+#endif
	551	+#endif
	552	+#ifndef this_cpu_try_cmpxchg128
	553	+#ifdef this_cpu_cmpxchg128
	554	+#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	555	+ __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128)
	556	+#else
	557	+#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
	558	+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
	559	+#endif
	560	+#endif
	561	+
515	562	#ifndef this_cpu_cmpxchg_1
516	563	#define this_cpu_cmpxchg_1(pcp, oval, nval) \
517	564	this_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -529,6 +576,15 @@ do { \ · 11 unmodified lines
529	576	this_cpu_generic_cmpxchg(pcp, oval, nval)
530	577	#endif
531	578
	579	+#ifndef this_cpu_cmpxchg64
	580	+#define this_cpu_cmpxchg64(pcp, oval, nval) \
	581	+ this_cpu_generic_cmpxchg(pcp, oval, nval)
	582	+#endif
	583	+#ifndef this_cpu_cmpxchg128
	584	+#define this_cpu_cmpxchg128(pcp, oval, nval) \
	585	+ this_cpu_generic_cmpxchg(pcp, oval, nval)
	586	+#endif
	587	+
532	588	#ifndef this_cpu_cmpxchg_double_1
533	589	#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
534	590	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)

percpu: Wire up cmpxchg128

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code