// SPDX-License-Identifier: LGPL-2.1#define _GNU_SOURCE#include <assert.h>#include <linux/membarrier.h>#include <pthread.h>#include <sched.h>#include <stdatomic.h>#include <stdint.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <syscall.h>#include <unistd.h>#include <poll.h>#include <sys/types.h>#include <signal.h>#include <errno.h>#include <stddef.h>#include <stdbool.h> static inline pid_t rseq_gettid(void){	return syscall(__NR_gettid);} #define NR_INJECT	9static int loop_cnt[NR_INJECT + 1]; static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used)); static int opt_modulo, verbose; static int opt_yield, opt_signal, opt_sleep,		opt_disable_rseq, opt_threads = 200,		opt_disable_mod = 0, opt_test = 's';static bool opt_rseq_legacy;static long long opt_reps = 5000; static __thread __attribute__((tls_model("initial-exec")))unsigned int signals_delivered; #ifndef BENCHMARK static __thread __attribute__((tls_model("initial-exec"), unused))unsigned int yield_mod_cnt, nr_abort; #define printf_verbose(fmt, ...)			\	do {						\		if (verbose)				\			printf(fmt, ## __VA_ARGS__);	\	} while (0) #ifdef __i386__ #define INJECT_ASM_REG	"eax" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \	"jz 333f\n\t" \	"222:\n\t" \	"dec %%" INJECT_ASM_REG "\n\t" \	"jnz 222b\n\t" \	"333:\n\t" #elif defined(__x86_64__) #define INJECT_ASM_REG_P	"rax"#define INJECT_ASM_REG		"eax" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG_P \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \	"mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \	"jz 333f\n\t" \	"222:\n\t" \	"dec %%" INJECT_ASM_REG "\n\t" \	"jnz 222b\n\t" \	"333:\n\t" #elif defined(__s390__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"r12" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \	"je 333f\n\t" \	"222:\n\t" \	"ahi %%" INJECT_ASM_REG ", -1\n\t" \	"jnz 222b\n\t" \	"333:\n\t" #elif defined(__ARMEL__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"r4" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"cmp " INJECT_ASM_REG ", #0\n\t" \	"beq 333f\n\t" \	"222:\n\t" \	"subs " INJECT_ASM_REG ", #1\n\t" \	"bne 222b\n\t" \	"333:\n\t" #elif defined(__AARCH64EL__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1] "Qo" (loop_cnt[1]) \	, [loop_cnt_2] "Qo" (loop_cnt[2]) \	, [loop_cnt_3] "Qo" (loop_cnt[3]) \	, [loop_cnt_4] "Qo" (loop_cnt[4]) \	, [loop_cnt_5] "Qo" (loop_cnt[5]) \	, [loop_cnt_6] "Qo" (loop_cnt[6]) #define INJECT_ASM_REG	RSEQ_ASM_TMP_REG32 #define RSEQ_INJECT_ASM(n) \	"	ldr	" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n"	\	"	cbz	" INJECT_ASM_REG ", 333f\n"			\	"222:\n"							\	"	sub	" INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n"	\	"	cbnz	" INJECT_ASM_REG ", 222b\n"			\	"333:\n" #elif defined(__PPC__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"r18" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"cmpwi %%" INJECT_ASM_REG ", 0\n\t" \	"beq 333f\n\t" \	"222:\n\t" \	"subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \	"bne 222b\n\t" \	"333:\n\t" #elif defined(__mips__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"$5" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"beqz " INJECT_ASM_REG ", 333f\n\t" \	"222:\n\t" \	"addiu " INJECT_ASM_REG ", -1\n\t" \	"bnez " INJECT_ASM_REG ", 222b\n\t" \	"333:\n\t"#elif defined(__riscv) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"t1" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n)					\	"lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t"		\	"beqz " INJECT_ASM_REG ", 333f\n\t"			\	"222:\n\t"						\	"addi  " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t"	\	"bnez " INJECT_ASM_REG ", 222b\n\t"			\	"333:\n\t"#elif defined(__or1k__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"r31" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n)					\	"l.lwz   " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t"	\	"l.sfeqi " INJECT_ASM_REG ", 0\n\t"			\	"l.bf 333f\n\t"						\	" l.nop\n\t"						\	"222:\n\t"						\	"l.addi  " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t"	\	"l.sfeqi " INJECT_ASM_REG ", 0\n\t"			\	"l.bf 222f\n\t"						\	" l.nop\n\t"						\	"333:\n\t"#else#error unsupported target#endif #define RSEQ_INJECT_FAILED \	nr_abort++; #define RSEQ_INJECT_C(n) \{ \	int loc_i, loc_nr_loops = loop_cnt[n]; \	\	for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \		rseq_barrier(); \	} \	if (loc_nr_loops == -1 && opt_modulo) { \		if (yield_mod_cnt == opt_modulo - 1) { \			if (opt_sleep > 0) \				poll(NULL, 0, opt_sleep); \			if (opt_yield) \				sched_yield(); \			if (opt_signal) \				raise(SIGUSR1); \			yield_mod_cnt = 0; \		} else { \			yield_mod_cnt++; \		} \	} \} #define rseq_no_glibc			true #else #define printf_verbose(fmt, ...)#define rseq_no_glibc			false #endif /* BENCHMARK */ #include "rseq.h" static enum rseq_mo opt_mo = RSEQ_MO_RELAXED; #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV#define TEST_MEMBARRIER static int sys_membarrier(int cmd, int flags, int cpu_id){	return syscall(__NR_membarrier, cmd, flags, cpu_id);}#endif #ifdef BUILDOPT_RSEQ_PERCPU_MM_CID# define RSEQ_PERCPU	RSEQ_PERCPU_MM_CIDstaticint get_current_cpu_id(void){	return rseq_current_mm_cid();}staticbool rseq_validate_cpu_id(void){	return rseq_mm_cid_available();}staticbool rseq_use_cpu_index(void){	return false;	/* Use mm_cid */}# ifdef TEST_MEMBARRIER/* * Membarrier does not currently support targeting a mm_cid, so * issue the barrier on all cpus. */staticint rseq_membarrier_expedited(int cpu){	return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,			      0, 0);}# endif /* TEST_MEMBARRIER */#else# define RSEQ_PERCPU	RSEQ_PERCPU_CPU_IDstaticint get_current_cpu_id(void){	return rseq_cpu_start();}staticbool rseq_validate_cpu_id(void){	return rseq_current_cpu_raw() >= 0;}staticbool rseq_use_cpu_index(void){	return true;	/* Use cpu_id as index. */}# ifdef TEST_MEMBARRIERstaticint rseq_membarrier_expedited(int cpu){	return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,			      MEMBARRIER_CMD_FLAG_CPU, cpu);}# endif /* TEST_MEMBARRIER */#endif struct percpu_lock_entry {	intptr_t v;} __attribute__((aligned(128))); struct percpu_lock {	struct percpu_lock_entry c[CPU_SETSIZE];}; struct test_data_entry {	intptr_t count;} __attribute__((aligned(128))); struct spinlock_test_data {	struct percpu_lock lock;	struct test_data_entry c[CPU_SETSIZE];}; struct spinlock_thread_test_data {	struct spinlock_test_data *data;	long long reps;	int reg;}; struct inc_test_data {	struct test_data_entry c[CPU_SETSIZE];}; struct inc_thread_test_data {	struct inc_test_data *data;	long long reps;	int reg;}; struct percpu_list_node {	intptr_t data;	struct percpu_list_node *next;}; struct percpu_list_entry {	struct percpu_list_node *head;} __attribute__((aligned(128))); struct percpu_list {	struct percpu_list_entry c[CPU_SETSIZE];}; #define BUFFER_ITEM_PER_CPU	100 struct percpu_buffer_node {	intptr_t data;}; struct percpu_buffer_entry {	intptr_t offset;	intptr_t buflen;	struct percpu_buffer_node **array;} __attribute__((aligned(128))); struct percpu_buffer {	struct percpu_buffer_entry c[CPU_SETSIZE];}; #define MEMCPY_BUFFER_ITEM_PER_CPU	100 struct percpu_memcpy_buffer_node {	intptr_t data1;	uint64_t data2;}; struct percpu_memcpy_buffer_entry {	intptr_t offset;	intptr_t buflen;	struct percpu_memcpy_buffer_node *array;} __attribute__((aligned(128))); struct percpu_memcpy_buffer {	struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];}; /* A simple percpu spinlock. Grabs lock on current cpu. */static int rseq_this_cpu_lock(struct percpu_lock *lock){	int cpu; 	for (;;) {		int ret; 		cpu = get_current_cpu_id();		if (cpu < 0) {			fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n",					getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu);			abort();		}		ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU,					 &lock->c[cpu].v,					 0, 1, cpu);		if (rseq_likely(!ret))			break;		/* Retry if comparison fails or rseq aborts. */	}	/*	 * Acquire semantic when taking lock after control dependency.	 * Matches rseq_smp_store_release().	 */	rseq_smp_acquire__after_ctrl_dep();	return cpu;} static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu){	assert(lock->c[cpu].v == 1);	/*	 * Release lock, with release semantic. Matches	 * rseq_smp_acquire__after_ctrl_dep().	 */	rseq_smp_store_release(&lock->c[cpu].v, 0);} void *test_percpu_spinlock_thread(void *arg){	struct spinlock_thread_test_data *thread_data = arg;	struct spinlock_test_data *data = thread_data->data;	long long i, reps; 	if (!opt_disable_rseq && thread_data->reg &&	    __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy))		abort();	reps = thread_data->reps;	for (i = 0; i < reps; i++) {		int cpu = rseq_this_cpu_lock(&data->lock);		data->c[cpu].count++;		rseq_percpu_unlock(&data->lock, cpu);#ifndef BENCHMARK		if (i != 0 && !(i % (reps / 10)))			printf_verbose("tid %d: count %lld\n",				       (int) rseq_gettid(), i);#endif	}	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",

Lines 1–500 of 1667. Load more