/* SPDX-License-Identifier: LGPL-2.1 OR MIT *//* * rseq-ppc-bits.h * * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> */ #include "rseq-bits-template.h" #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) static inline __attribute__((always_inline))int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu){ RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])#ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4)#ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])#endif /* final store */ RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) RSEQ_INJECT_ASM(5) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) RSEQ_INJECT_INPUT : "memory", "cc", "r17" RSEQ_INJECT_CLOBBER : abort, cmpfail#ifdef RSEQ_COMPARE_TWICE , error1, error2#endif ); rseq_after_asm_goto(); return 0;abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1;cmpfail: rseq_after_asm_goto(); return 1;#ifdef RSEQ_COMPARE_TWICEerror1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed");error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed");#endif} static inline __attribute__((always_inline))int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, int cpu){ RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])#ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v not equal to @expectnot */ RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) RSEQ_INJECT_ASM(4)#ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v not equal to @expectnot */ RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])#endif /* load the value of @v */ RSEQ_ASM_OP_R_LOAD(v) /* store it in @load */ RSEQ_ASM_OP_R_STORE(load) /* dereference voffp(v) */ RSEQ_ASM_OP_R_LOADX(voffp) /* final store the value at voffp(v) */ RSEQ_ASM_OP_R_FINAL_STORE(v, 2) RSEQ_INJECT_ASM(5) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), [voffp] "b" (voffp), [load] "m" (*load) RSEQ_INJECT_INPUT : "memory", "cc", "r17" RSEQ_INJECT_CLOBBER : abort, cmpfail#ifdef RSEQ_COMPARE_TWICE , error1, error2#endif ); rseq_after_asm_goto(); return 0;abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1;cmpfail: rseq_after_asm_goto(); return 1;#ifdef RSEQ_COMPARE_TWICEerror1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed");error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed");#endif} static inline __attribute__((always_inline))int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu){ RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */#ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3)#ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])#endif /* load the value of @v */ RSEQ_ASM_OP_R_LOAD(v) /* add @count to it */ RSEQ_ASM_OP_R_ADD(count) /* final store */ RSEQ_ASM_OP_R_FINAL_STORE(v, 2) RSEQ_INJECT_ASM(4) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [count] "r" (count) RSEQ_INJECT_INPUT : "memory", "cc", "r17" RSEQ_INJECT_CLOBBER : abort#ifdef RSEQ_COMPARE_TWICE , error1#endif ); rseq_after_asm_goto(); return 0;abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1;#ifdef RSEQ_COMPARE_TWICEerror1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed");#endif} static inline __attribute__((always_inline))int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t *v2, intptr_t expect2, intptr_t newv, int cpu){ RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])#ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) /* cmp @v2 equal to @expct2 */ RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) RSEQ_INJECT_ASM(5)#ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) /* cmp @v2 equal to @expct2 */ RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])#endif /* final store */ RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) RSEQ_INJECT_ASM(6) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), /* final store input */ [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) RSEQ_INJECT_INPUT : "memory", "cc", "r17" RSEQ_INJECT_CLOBBER : abort, cmpfail#ifdef RSEQ_COMPARE_TWICE , error1, error2, error3#endif ); rseq_after_asm_goto(); return 0;abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1;cmpfail: rseq_after_asm_goto(); return 1;#ifdef RSEQ_COMPARE_TWICEerror1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed");error2: rseq_after_asm_goto(); rseq_bug("1st expected value comparison failed");error3: rseq_after_asm_goto(); rseq_bug("2nd expected value comparison failed");#endif} #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) static inline __attribute__((always_inline))int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, intptr_t *v2, intptr_t newv2, intptr_t newv, int cpu){ RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])#ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4)#ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])#endif /* try store */ RSEQ_ASM_OP_STORE(newv2, v2) RSEQ_INJECT_ASM(5)#ifdef RSEQ_TEMPLATE_MO_RELEASE /* for 'release' */ "lwsync\n\t"#endif /* final store */ RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) RSEQ_INJECT_ASM(6) RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), /* final store input */ [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) RSEQ_INJECT_INPUT : "memory", "cc", "r17" RSEQ_INJECT_CLOBBER : abort, cmpfail#ifdef RSEQ_COMPARE_TWICE , error1, error2#endif ); rseq_after_asm_goto(); return 0;abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1;cmpfail: rseq_after_asm_goto(); return 1;#ifdef RSEQ_COMPARE_TWICEerror1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed");error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed");#endif} static inline __attribute__((always_inline))int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, void *dst, void *src, size_t len, intptr_t newv, int cpu){ RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])#ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])#endif /* setup for mempcy */ "mr %%r19, %[len]\n\t" "mr %%r20, %[src]\n\t" "mr %%r21, %[dst]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4)#ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])#endif /* try memcpy */ RSEQ_ASM_OP_R_MEMCPY() RSEQ_INJECT_ASM(5)#ifdef RSEQ_TEMPLATE_MO_RELEASE /* for 'release' */ "lwsync\n\t"#endif /* final store */ RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) RSEQ_INJECT_ASM(6) /* teardown */ RSEQ_ASM_DEFINE_ABORT(4, abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv), /* try memcpy input */ [dst] "r" (dst), [src] "r" (src), [len] "r" (len) RSEQ_INJECT_INPUT : "memory", "cc", "r17", "r18", "r19", "r20", "r21" RSEQ_INJECT_CLOBBER : abort, cmpfail#ifdef RSEQ_COMPARE_TWICE , error1, error2#endif ); rseq_after_asm_goto(); return 0;abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1;cmpfail: rseq_after_asm_goto(); return 1;#ifdef RSEQ_COMPARE_TWICEerror1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed");error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed");#endif} #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ #include "rseq-bits-reset.h"
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
/*
* rseq-ppc-bits.h
*
* (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
* (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
*/
#include "rseq-bits-template.h"
#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
{
RSEQ_INJECT_C(9)
__asm__ __volatile__ goto (
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
/* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
RSEQ_INJECT_ASM(3)
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
/* final store */
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
RSEQ_INJECT_ASM(5)
RSEQ_ASM_DEFINE_ABORT(4, abort)
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
[v] "m" (*v),
[expect] "r" (expect),
[newv] "r" (newv)
RSEQ_INJECT_INPUT
: "memory", "cc", "r17"
RSEQ_INJECT_CLOBBER
: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
, error1, error2
#endif
);
rseq_after_asm_goto();
return 0;
abort:
rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
long voffp, intptr_t *load, int cpu)
{
RSEQ_INJECT_C(9)
__asm__ __volatile__ goto (
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
/* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
RSEQ_INJECT_ASM(3)
/* cmp @v not equal to @expectnot */
RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
/* cmp @v not equal to @expectnot */
RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
#endif
/* load the value of @v */
RSEQ_ASM_OP_R_LOAD(v)
/* store it in @load */
RSEQ_ASM_OP_R_STORE(load)
/* dereference voffp(v) */
RSEQ_ASM_OP_R_LOADX(voffp)
/* final store the value at voffp(v) */
RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
RSEQ_INJECT_ASM(5)
RSEQ_ASM_DEFINE_ABORT(4, abort)
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* final store input */
[v] "m" (*v),
[expectnot] "r" (expectnot),
[voffp] "b" (voffp),
[load] "m" (*load)
RSEQ_INJECT_INPUT
: "memory", "cc", "r17"
RSEQ_INJECT_CLOBBER
: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
, error1, error2
#endif
);
rseq_after_asm_goto();
return 0;
abort:
rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
{
RSEQ_INJECT_C(9)
__asm__ __volatile__ goto (
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
#ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
#endif
/* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
#endif
/* load the value of @v */
RSEQ_ASM_OP_R_LOAD(v)
/* add @count to it */
RSEQ_ASM_OP_R_ADD(count)
/* final store */
RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
RSEQ_INJECT_ASM(4)
RSEQ_ASM_DEFINE_ABORT(4, abort)
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* final store input */
[v] "m" (*v),
[count] "r" (count)
RSEQ_INJECT_INPUT
: "memory", "cc", "r17"
RSEQ_INJECT_CLOBBER
: abort
#ifdef RSEQ_COMPARE_TWICE
, error1
#endif
);
rseq_after_asm_goto();
return 0;
abort:
rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
#ifdef RSEQ_COMPARE_TWICE
error1:
rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
#endif
}
static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
intptr_t *v2, intptr_t expect2,
intptr_t newv, int cpu)
{
RSEQ_INJECT_C(9)
__asm__ __volatile__ goto (
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
#endif
/* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
RSEQ_INJECT_ASM(3)
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
RSEQ_INJECT_ASM(4)
/* cmp @v2 equal to @expct2 */
RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
RSEQ_INJECT_ASM(5)
#ifdef RSEQ_COMPARE_TWICE
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
/* cmp @v2 equal to @expct2 */
RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
#endif
/* final store */
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
RSEQ_INJECT_ASM(6)
RSEQ_ASM_DEFINE_ABORT(4, abort)
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* cmp2 input */
[v2] "m" (*v2),
[expect2] "r" (expect2),
/* final store input */
[v] "m" (*v),
[expect] "r" (expect),
[newv] "r" (newv)
RSEQ_INJECT_INPUT
: "memory", "cc", "r17"
RSEQ_INJECT_CLOBBER
: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
, error1, error2, error3
#endif
);
rseq_after_asm_goto();
return 0;
abort:
rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
rseq_after_asm_goto();
rseq_bug("1st expected value comparison failed");
error3:
rseq_after_asm_goto();
rseq_bug("2nd expected value comparison failed");
#endif
}
#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
intptr_t *v2, intptr_t newv2,
intptr_t newv, int cpu)
{
RSEQ_INJECT_C(9)
__asm__ __volatile__ goto (
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
/* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
RSEQ_INJECT_ASM(3)
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
/* try store */
RSEQ_ASM_OP_STORE(newv2, v2)
RSEQ_INJECT_ASM(5)
#ifdef RSEQ_TEMPLATE_MO_RELEASE
/* for 'release' */
"lwsync\n\t"
#endif
/* final store */
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
RSEQ_INJECT_ASM(6)
RSEQ_ASM_DEFINE_ABORT(4, abort)
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* try store input */
[v2] "m" (*v2),
[newv2] "r" (newv2),
/* final store input */
[v] "m" (*v),
[expect] "r" (expect),
[newv] "r" (newv)
RSEQ_INJECT_INPUT
: "memory", "cc", "r17"
RSEQ_INJECT_CLOBBER
: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
, error1, error2
#endif
);
rseq_after_asm_goto();
return 0;
abort:
rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
void *dst, void *src, size_t len,
intptr_t newv, int cpu)
{
RSEQ_INJECT_C(9)
__asm__ __volatile__ goto (
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
/* setup for mempcy */
"mr %%r19, %[len]\n\t"
"mr %%r20, %[src]\n\t"
"mr %%r21, %[dst]\n\t"
/* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
RSEQ_INJECT_ASM(3)
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
/* cmp cpuid */
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
/* cmp @v equal to @expect */
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
/* try memcpy */
RSEQ_ASM_OP_R_MEMCPY()
RSEQ_INJECT_ASM(5)
#ifdef RSEQ_TEMPLATE_MO_RELEASE
/* for 'release' */
"lwsync\n\t"
#endif
/* final store */
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
RSEQ_INJECT_ASM(6)
/* teardown */
RSEQ_ASM_DEFINE_ABORT(4, abort)
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* final store input */
[v] "m" (*v),
[expect] "r" (expect),
[newv] "r" (newv),
/* try memcpy input */
[dst] "r" (dst),
[src] "r" (src),
[len] "r" (len)
RSEQ_INJECT_INPUT
: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
RSEQ_INJECT_CLOBBER
: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
, error1, error2
#endif
);
rseq_after_asm_goto();
return 0;
abort:
rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
#include "rseq-bits-reset.h"