// SPDX-License-Identifier: LGPL-2.1#define _GNU_SOURCE#include <assert.h>#include <linux/membarrier.h>#include <pthread.h>#include <sched.h>#include <stdatomic.h>#include <stdint.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <syscall.h>#include <unistd.h>#include <poll.h>#include <sys/types.h>#include <signal.h>#include <errno.h>#include <stddef.h> static inline pid_t rseq_gettid(void){	return syscall(__NR_gettid);} #define NR_INJECT	9static int loop_cnt[NR_INJECT + 1]; static int loop_cnt_1 asm("asm_loop_cnt_1") __attribute__((used));static int loop_cnt_2 asm("asm_loop_cnt_2") __attribute__((used));static int loop_cnt_3 asm("asm_loop_cnt_3") __attribute__((used));static int loop_cnt_4 asm("asm_loop_cnt_4") __attribute__((used));static int loop_cnt_5 asm("asm_loop_cnt_5") __attribute__((used));static int loop_cnt_6 asm("asm_loop_cnt_6") __attribute__((used)); static int opt_modulo, verbose; static int opt_yield, opt_signal, opt_sleep,		opt_disable_rseq, opt_threads = 200,		opt_disable_mod = 0, opt_test = 's', opt_mb = 0; #ifndef RSEQ_SKIP_FASTPATHstatic long long opt_reps = 5000;#elsestatic long long opt_reps = 100;#endif static __thread __attribute__((tls_model("initial-exec")))unsigned int signals_delivered; #ifndef BENCHMARK static __thread __attribute__((tls_model("initial-exec"), unused))unsigned int yield_mod_cnt, nr_abort; #define printf_verbose(fmt, ...)			\	do {						\		if (verbose)				\			printf(fmt, ## __VA_ARGS__);	\	} while (0) #ifdef __i386__ #define INJECT_ASM_REG	"eax" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \	"jz 333f\n\t" \	"222:\n\t" \	"dec %%" INJECT_ASM_REG "\n\t" \	"jnz 222b\n\t" \	"333:\n\t" #elif defined(__x86_64__) #define INJECT_ASM_REG_P	"rax"#define INJECT_ASM_REG		"eax" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG_P \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \	"mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \	"jz 333f\n\t" \	"222:\n\t" \	"dec %%" INJECT_ASM_REG "\n\t" \	"jnz 222b\n\t" \	"333:\n\t" #elif defined(__s390__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"r12" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \	"je 333f\n\t" \	"222:\n\t" \	"ahi %%" INJECT_ASM_REG ", -1\n\t" \	"jnz 222b\n\t" \	"333:\n\t" #elif defined(__ARMEL__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"r4" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"cmp " INJECT_ASM_REG ", #0\n\t" \	"beq 333f\n\t" \	"222:\n\t" \	"subs " INJECT_ASM_REG ", #1\n\t" \	"bne 222b\n\t" \	"333:\n\t" #elif defined(__AARCH64EL__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1] "Qo" (loop_cnt[1]) \	, [loop_cnt_2] "Qo" (loop_cnt[2]) \	, [loop_cnt_3] "Qo" (loop_cnt[3]) \	, [loop_cnt_4] "Qo" (loop_cnt[4]) \	, [loop_cnt_5] "Qo" (loop_cnt[5]) \	, [loop_cnt_6] "Qo" (loop_cnt[6]) #define INJECT_ASM_REG	RSEQ_ASM_TMP_REG32 #define RSEQ_INJECT_ASM(n) \	"	ldr	" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n"	\	"	cbz	" INJECT_ASM_REG ", 333f\n"			\	"222:\n"							\	"	sub	" INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n"	\	"	cbnz	" INJECT_ASM_REG ", 222b\n"			\	"333:\n" #elif __PPC__ #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"r18" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"cmpwi %%" INJECT_ASM_REG ", 0\n\t" \	"beq 333f\n\t" \	"222:\n\t" \	"subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \	"bne 222b\n\t" \	"333:\n\t" #elif defined(__mips__) #define RSEQ_INJECT_INPUT \	, [loop_cnt_1]"m"(loop_cnt[1]) \	, [loop_cnt_2]"m"(loop_cnt[2]) \	, [loop_cnt_3]"m"(loop_cnt[3]) \	, [loop_cnt_4]"m"(loop_cnt[4]) \	, [loop_cnt_5]"m"(loop_cnt[5]) \	, [loop_cnt_6]"m"(loop_cnt[6]) #define INJECT_ASM_REG	"$5" #define RSEQ_INJECT_CLOBBER \	, INJECT_ASM_REG #define RSEQ_INJECT_ASM(n) \	"lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \	"beqz " INJECT_ASM_REG ", 333f\n\t" \	"222:\n\t" \	"addiu " INJECT_ASM_REG ", -1\n\t" \	"bnez " INJECT_ASM_REG ", 222b\n\t" \	"333:\n\t" #else#error unsupported target#endif #define RSEQ_INJECT_FAILED \	nr_abort++; #define RSEQ_INJECT_C(n) \{ \	int loc_i, loc_nr_loops = loop_cnt[n]; \	\	for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \		rseq_barrier(); \	} \	if (loc_nr_loops == -1 && opt_modulo) { \		if (yield_mod_cnt == opt_modulo - 1) { \			if (opt_sleep > 0) \				poll(NULL, 0, opt_sleep); \			if (opt_yield) \				sched_yield(); \			if (opt_signal) \				raise(SIGUSR1); \			yield_mod_cnt = 0; \		} else { \			yield_mod_cnt++; \		} \	} \} #else #define printf_verbose(fmt, ...) #endif /* BENCHMARK */ #include "rseq.h" struct percpu_lock_entry {	intptr_t v;} __attribute__((aligned(128))); struct percpu_lock {	struct percpu_lock_entry c[CPU_SETSIZE];}; struct test_data_entry {	intptr_t count;} __attribute__((aligned(128))); struct spinlock_test_data {	struct percpu_lock lock;	struct test_data_entry c[CPU_SETSIZE];}; struct spinlock_thread_test_data {	struct spinlock_test_data *data;	long long reps;	int reg;}; struct inc_test_data {	struct test_data_entry c[CPU_SETSIZE];}; struct inc_thread_test_data {	struct inc_test_data *data;	long long reps;	int reg;}; struct percpu_list_node {	intptr_t data;	struct percpu_list_node *next;}; struct percpu_list_entry {	struct percpu_list_node *head;} __attribute__((aligned(128))); struct percpu_list {	struct percpu_list_entry c[CPU_SETSIZE];}; #define BUFFER_ITEM_PER_CPU	100 struct percpu_buffer_node {	intptr_t data;}; struct percpu_buffer_entry {	intptr_t offset;	intptr_t buflen;	struct percpu_buffer_node **array;} __attribute__((aligned(128))); struct percpu_buffer {	struct percpu_buffer_entry c[CPU_SETSIZE];}; #define MEMCPY_BUFFER_ITEM_PER_CPU	100 struct percpu_memcpy_buffer_node {	intptr_t data1;	uint64_t data2;}; struct percpu_memcpy_buffer_entry {	intptr_t offset;	intptr_t buflen;	struct percpu_memcpy_buffer_node *array;} __attribute__((aligned(128))); struct percpu_memcpy_buffer {	struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];}; /* A simple percpu spinlock. Grabs lock on current cpu. */static int rseq_this_cpu_lock(struct percpu_lock *lock){	int cpu; 	for (;;) {		int ret; 		cpu = rseq_cpu_start();		ret = rseq_cmpeqv_storev(&lock->c[cpu].v,					 0, 1, cpu);		if (rseq_likely(!ret))			break;		/* Retry if comparison fails or rseq aborts. */	}	/*	 * Acquire semantic when taking lock after control dependency.	 * Matches rseq_smp_store_release().	 */	rseq_smp_acquire__after_ctrl_dep();	return cpu;} static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu){	assert(lock->c[cpu].v == 1);	/*	 * Release lock, with release semantic. Matches	 * rseq_smp_acquire__after_ctrl_dep().	 */	rseq_smp_store_release(&lock->c[cpu].v, 0);} void *test_percpu_spinlock_thread(void *arg){	struct spinlock_thread_test_data *thread_data = arg;	struct spinlock_test_data *data = thread_data->data;	long long i, reps; 	if (!opt_disable_rseq && thread_data->reg &&	    rseq_register_current_thread())		abort();	reps = thread_data->reps;	for (i = 0; i < reps; i++) {		int cpu = rseq_cpu_start(); 		cpu = rseq_this_cpu_lock(&data->lock);		data->c[cpu].count++;		rseq_percpu_unlock(&data->lock, cpu);#ifndef BENCHMARK		if (i != 0 && !(i % (reps / 10)))			printf_verbose("tid %d: count %lld\n",				       (int) rseq_gettid(), i);#endif	}	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",		       (int) rseq_gettid(), nr_abort, signals_delivered);	if (!opt_disable_rseq && thread_data->reg &&	    rseq_unregister_current_thread())		abort();	return NULL;} /* * A simple test which implements a sharded counter using a per-cpu * lock.  Obviously real applications might prefer to simply use a * per-cpu increment; however, this is reasonable for a test and the * lock can be extended to synchronize more complicated operations. */void test_percpu_spinlock(void){	const int num_threads = opt_threads;	int i, ret;	uint64_t sum;	pthread_t test_threads[num_threads];	struct spinlock_test_data data;	struct spinlock_thread_test_data thread_data[num_threads]; 	memset(&data, 0, sizeof(data));	for (i = 0; i < num_threads; i++) {		thread_data[i].reps = opt_reps;		if (opt_disable_mod <= 0 || (i % opt_disable_mod))			thread_data[i].reg = 1;		else			thread_data[i].reg = 0;		thread_data[i].data = &data;		ret = pthread_create(&test_threads[i], NULL,				     test_percpu_spinlock_thread,				     &thread_data[i]);		if (ret) {			errno = ret;			perror("pthread_create");			abort();		}	} 	for (i = 0; i < num_threads; i++) {		ret = pthread_join(test_threads[i], NULL);		if (ret) {			errno = ret;			perror("pthread_join");			abort();		}	} 	sum = 0;	for (i = 0; i < CPU_SETSIZE; i++)		sum += data.c[i].count; 	assert(sum == (uint64_t)opt_reps * num_threads);} void *test_percpu_inc_thread(void *arg){	struct inc_thread_test_data *thread_data = arg;	struct inc_test_data *data = thread_data->data;	long long i, reps; 	if (!opt_disable_rseq && thread_data->reg &&	    rseq_register_current_thread())		abort();	reps = thread_data->reps;	for (i = 0; i < reps; i++) {		int ret; 		do {			int cpu; 			cpu = rseq_cpu_start();			ret = rseq_addv(&data->c[cpu].count, 1, cpu);		} while (rseq_unlikely(ret));#ifndef BENCHMARK		if (i != 0 && !(i % (reps / 10)))			printf_verbose("tid %d: count %lld\n",				       (int) rseq_gettid(), i);#endif	}	printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",		       (int) rseq_gettid(), nr_abort, signals_delivered);	if (!opt_disable_rseq && thread_data->reg &&	    rseq_unregister_current_thread())		abort();	return NULL;} void test_percpu_inc(void){	const int num_threads = opt_threads;	int i, ret;	uint64_t sum;	pthread_t test_threads[num_threads];	struct inc_test_data data;	struct inc_thread_test_data thread_data[num_threads]; 	memset(&data, 0, sizeof(data));	for (i = 0; i < num_threads; i++) {		thread_data[i].reps = opt_reps;		if (opt_disable_mod <= 0 || (i % opt_disable_mod))			thread_data[i].reg = 1;		else			thread_data[i].reg = 0;		thread_data[i].data = &data;		ret = pthread_create(&test_threads[i], NULL,				     test_percpu_inc_thread,				     &thread_data[i]);		if (ret) {			errno = ret;			perror("pthread_create");			abort();		}	} 	for (i = 0; i < num_threads; i++) {		ret = pthread_join(test_threads[i], NULL);

Lines 1–500 of 1552. Load more