// SPDX-License-Identifier: GPL-2.0-only/* * Stress userfaultfd syscall. * *  Copyright (C) 2015  Red Hat, Inc. * * This test allocates two virtual areas and bounces the physical * memory across the two virtual areas (from area_src to area_dst) * using userfaultfd. * * There are three threads running per CPU: * * 1) one per-CPU thread takes a per-page pthread_mutex in a random *    page of the area_dst (while the physical page may still be in *    area_src), and increments a per-page counter in the same page, *    and checks its value against a verification region. * * 2) another per-CPU thread handles the userfaults generated by *    thread 1 above. userfaultfd blocking reads or poll() modes are *    exercised interleaved. * * 3) one last per-CPU thread transfers the memory in the background *    at maximum bandwidth (if not already transferred by thread *    2). Each cpu thread takes cares of transferring a portion of the *    area. * * When all threads of type 3 completed the transfer, one bounce is * complete. area_src and area_dst are then swapped. All threads are * respawned and so the bounce is immediately restarted in the * opposite direction. * * per-CPU threads 1 by triggering userfaults inside * pthread_mutex_lock will also verify the atomicity of the memory * transfer (UFFDIO_COPY). */ #define _GNU_SOURCE#include <stdio.h>#include <errno.h>#include <unistd.h>#include <stdlib.h>#include <sys/types.h>#include <sys/stat.h>#include <fcntl.h>#include <time.h>#include <signal.h>#include <poll.h>#include <string.h>#include <sys/mman.h>#include <sys/syscall.h>#include <sys/ioctl.h>#include <sys/wait.h>#include <pthread.h>#include <linux/userfaultfd.h>#include <setjmp.h>#include <stdbool.h>#include <assert.h>#include <inttypes.h>#include <stdint.h> #include "../kselftest.h" #ifdef __NR_userfaultfd static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size; #define BOUNCE_RANDOM		(1<<0)#define BOUNCE_RACINGFAULTS	(1<<1)#define BOUNCE_VERIFY		(1<<2)#define BOUNCE_POLL		(1<<3)static int bounces; #define TEST_ANON	1#define TEST_HUGETLB	2#define TEST_SHMEM	3static int test_type; /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */#define ALARM_INTERVAL_SECS 10static volatile bool test_uffdio_copy_eexist = true;static volatile bool test_uffdio_zeropage_eexist = true;/* Whether to test uffd write-protection */static bool test_uffdio_wp = false;/* Whether to test uffd minor faults */static bool test_uffdio_minor = false; static bool map_shared;static int huge_fd;static char *huge_fd_off0;static unsigned long long *count_verify;static int uffd, uffd_flags, finished, *pipefd;static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;static char *zeropage;pthread_attr_t attr; /* Userfaultfd test statistics */struct uffd_stats {	int cpu;	unsigned long missing_faults;	unsigned long wp_faults;	unsigned long minor_faults;}; /* pthread_mutex_t starts at page offset 0 */#define area_mutex(___area, ___nr)					\	((pthread_mutex_t *) ((___area) + (___nr)*page_size))/* * count is placed in the page after pthread_mutex_t naturally aligned * to avoid non alignment faults on non-x86 archs. */#define area_count(___area, ___nr)					\	((volatile unsigned long long *) ((unsigned long)		\				 ((___area) + (___nr)*page_size +	\				  sizeof(pthread_mutex_t) +		\				  sizeof(unsigned long long) - 1) &	\				 ~(unsigned long)(sizeof(unsigned long long) \						  -  1))) const char *examples =    "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"    "./userfaultfd anon 100 99999\n\n"    "# Run share memory test on 1GiB region with 99 bounces:\n"    "./userfaultfd shmem 1000 99\n\n"    "# Run hugetlb memory test on 256MiB region with 50 bounces (using /dev/hugepages/hugefile):\n"    "./userfaultfd hugetlb 256 50 /dev/hugepages/hugefile\n\n"    "# Run the same hugetlb test but using shmem:\n"    "./userfaultfd hugetlb_shared 256 50 /dev/hugepages/hugefile\n\n"    "# 10MiB-~6GiB 999 bounces anonymous test, "    "continue forever unless an error triggers\n"    "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n"; static void usage(void){	fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "		"[hugetlbfs_file]\n\n");	fprintf(stderr, "Supported <test type>: anon, hugetlb, "		"hugetlb_shared, shmem\n\n");	fprintf(stderr, "Examples:\n\n");	fprintf(stderr, "%s", examples);	exit(1);} #define uffd_error(code, fmt, ...)                                             \	do {                                                                   \		fprintf(stderr, fmt, ##__VA_ARGS__);                           \		fprintf(stderr, ": %" PRId64 "\n", (int64_t)(code));           \		exit(1);                                                       \	} while (0) static void uffd_stats_reset(struct uffd_stats *uffd_stats,			     unsigned long n_cpus){	int i; 	for (i = 0; i < n_cpus; i++) {		uffd_stats[i].cpu = i;		uffd_stats[i].missing_faults = 0;		uffd_stats[i].wp_faults = 0;		uffd_stats[i].minor_faults = 0;	}} static void uffd_stats_report(struct uffd_stats *stats, int n_cpus){	int i;	unsigned long long miss_total = 0, wp_total = 0, minor_total = 0; 	for (i = 0; i < n_cpus; i++) {		miss_total += stats[i].missing_faults;		wp_total += stats[i].wp_faults;		minor_total += stats[i].minor_faults;	} 	printf("userfaults: %llu missing (", miss_total);	for (i = 0; i < n_cpus; i++)		printf("%lu+", stats[i].missing_faults);	printf("\b), %llu wp (", wp_total);	for (i = 0; i < n_cpus; i++)		printf("%lu+", stats[i].wp_faults);	printf("\b), %llu minor (", minor_total);	for (i = 0; i < n_cpus; i++)		printf("%lu+", stats[i].minor_faults);	printf("\b)\n");} static int anon_release_pages(char *rel_area){	int ret = 0; 	if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) {		perror("madvise");		ret = 1;	} 	return ret;} static void anon_allocate_area(void **alloc_area){	if (posix_memalign(alloc_area, page_size, nr_pages * page_size)) {		fprintf(stderr, "out of memory\n");		*alloc_area = NULL;	}} static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset){} /* HugeTLB memory */static int hugetlb_release_pages(char *rel_area){	int ret = 0; 	if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,				rel_area == huge_fd_off0 ? 0 :				nr_pages * page_size,				nr_pages * page_size)) {		perror("fallocate");		ret = 1;	} 	return ret;} static void hugetlb_allocate_area(void **alloc_area){	void *area_alias = NULL;	char **alloc_area_alias; 	*alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,			   (map_shared ? MAP_SHARED : MAP_PRIVATE) |			   MAP_HUGETLB,			   huge_fd, *alloc_area == area_src ? 0 :			   nr_pages * page_size);	if (*alloc_area == MAP_FAILED) {		perror("mmap of hugetlbfs file failed");		goto fail;	} 	if (map_shared) {		area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,				  MAP_SHARED | MAP_HUGETLB,				  huge_fd, *alloc_area == area_src ? 0 :				  nr_pages * page_size);		if (area_alias == MAP_FAILED) {			perror("mmap of hugetlb file alias failed");			goto fail_munmap;		}	} 	if (*alloc_area == area_src) {		huge_fd_off0 = *alloc_area;		alloc_area_alias = &area_src_alias;	} else {		alloc_area_alias = &area_dst_alias;	}	if (area_alias)		*alloc_area_alias = area_alias; 	return; fail_munmap:	if (munmap(*alloc_area, nr_pages * page_size) < 0) {		perror("hugetlb munmap");		exit(1);	}fail:	*alloc_area = NULL;} static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset){	if (!map_shared)		return;	/*	 * We can't zap just the pagetable with hugetlbfs because	 * MADV_DONTEED won't work. So exercise -EEXIST on a alias	 * mapping where the pagetables are not established initially,	 * this way we'll exercise the -EEXEC at the fs level.	 */	*start = (unsigned long) area_dst_alias + offset;} /* Shared memory */static int shmem_release_pages(char *rel_area){	int ret = 0; 	if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) {		perror("madvise");		ret = 1;	} 	return ret;} static void shmem_allocate_area(void **alloc_area){	*alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,			   MAP_ANONYMOUS | MAP_SHARED, -1, 0);	if (*alloc_area == MAP_FAILED) {		fprintf(stderr, "shared memory mmap failed\n");		*alloc_area = NULL;	}} struct uffd_test_ops {	unsigned long expected_ioctls;	void (*allocate_area)(void **alloc_area);	int (*release_pages)(char *rel_area);	void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);}; #define SHMEM_EXPECTED_IOCTLS		((1 << _UFFDIO_WAKE) | \					 (1 << _UFFDIO_COPY) | \					 (1 << _UFFDIO_ZEROPAGE)) #define ANON_EXPECTED_IOCTLS		((1 << _UFFDIO_WAKE) | \					 (1 << _UFFDIO_COPY) | \					 (1 << _UFFDIO_ZEROPAGE) | \					 (1 << _UFFDIO_WRITEPROTECT)) static struct uffd_test_ops anon_uffd_test_ops = {	.expected_ioctls = ANON_EXPECTED_IOCTLS,	.allocate_area	= anon_allocate_area,	.release_pages	= anon_release_pages,	.alias_mapping = noop_alias_mapping,}; static struct uffd_test_ops shmem_uffd_test_ops = {	.expected_ioctls = SHMEM_EXPECTED_IOCTLS,	.allocate_area	= shmem_allocate_area,	.release_pages	= shmem_release_pages,	.alias_mapping = noop_alias_mapping,}; static struct uffd_test_ops hugetlb_uffd_test_ops = {	.expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC & ~(1 << _UFFDIO_CONTINUE),	.allocate_area	= hugetlb_allocate_area,	.release_pages	= hugetlb_release_pages,	.alias_mapping = hugetlb_alias_mapping,}; static struct uffd_test_ops *uffd_test_ops; static int my_bcmp(char *str1, char *str2, size_t n){	unsigned long i;	for (i = 0; i < n; i++)		if (str1[i] != str2[i])			return 1;	return 0;} static void wp_range(int ufd, __u64 start, __u64 len, bool wp){	struct uffdio_writeprotect prms; 	/* Write protection page faults */	prms.range.start = start;	prms.range.len = len;	/* Undo write-protect, do wakeup after that */	prms.mode = wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0; 	if (ioctl(ufd, UFFDIO_WRITEPROTECT, &prms)) {		fprintf(stderr, "clear WP failed for address 0x%" PRIx64 "\n",			(uint64_t)start);		exit(1);	}} static void continue_range(int ufd, __u64 start, __u64 len){	struct uffdio_continue req; 	req.range.start = start;	req.range.len = len;	req.mode = 0; 	if (ioctl(ufd, UFFDIO_CONTINUE, &req)) {		fprintf(stderr,			"UFFDIO_CONTINUE failed for address 0x%" PRIx64 "\n",			(uint64_t)start);		exit(1);	}} static void *locking_thread(void *arg){	unsigned long cpu = (unsigned long) arg;	struct random_data rand;	unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */	int32_t rand_nr;	unsigned long long count;	char randstate[64];	unsigned int seed;	time_t start; 	if (bounces & BOUNCE_RANDOM) {		seed = (unsigned int) time(NULL) - bounces;		if (!(bounces & BOUNCE_RACINGFAULTS))			seed += cpu;		bzero(&rand, sizeof(rand));		bzero(&randstate, sizeof(randstate));		if (initstate_r(seed, randstate, sizeof(randstate), &rand)) {			fprintf(stderr, "srandom_r error\n");			exit(1);		}	} else {		page_nr = -bounces;		if (!(bounces & BOUNCE_RACINGFAULTS))			page_nr += cpu * nr_pages_per_cpu;	} 	while (!finished) {		if (bounces & BOUNCE_RANDOM) {			if (random_r(&rand, &rand_nr)) {				fprintf(stderr, "random_r 1 error\n");				exit(1);			}			page_nr = rand_nr;			if (sizeof(page_nr) > sizeof(rand_nr)) {				if (random_r(&rand, &rand_nr)) {					fprintf(stderr, "random_r 2 error\n");					exit(1);				}				page_nr |= (((unsigned long) rand_nr) << 16) <<					   16;			}		} else			page_nr += 1;		page_nr %= nr_pages; 		start = time(NULL);		if (bounces & BOUNCE_VERIFY) {			count = *area_count(area_dst, page_nr);			if (!count) {				fprintf(stderr,					"page_nr %lu wrong count %Lu %Lu\n",					page_nr, count,					count_verify[page_nr]);				exit(1);			}  			/*			 * We can't use bcmp (or memcmp) because that			 * returns 0 erroneously if the memory is			 * changing under it (even if the end of the			 * page is never changing and always			 * different).			 */#if 1			if (!my_bcmp(area_dst + page_nr * page_size, zeropage,				     page_size)) {				fprintf(stderr,					"my_bcmp page_nr %lu wrong count %Lu %Lu\n",					page_nr, count, count_verify[page_nr]);				exit(1);			}#else			unsigned long loops; 			loops = 0;			/* uncomment the below line to test with mutex */			/* pthread_mutex_lock(area_mutex(area_dst, page_nr)); */			while (!bcmp(area_dst + page_nr * page_size, zeropage,				     page_size)) {				loops += 1;				if (loops > 10)					break;			}			/* uncomment below line to test with mutex */			/* pthread_mutex_unlock(area_mutex(area_dst, page_nr)); */			if (loops) {				fprintf(stderr,					"page_nr %lu all zero thread %lu %p %lu\n",					page_nr, cpu, area_dst + page_nr * page_size,					loops);				if (loops > 10)					exit(1);			}#endif		} 		pthread_mutex_lock(area_mutex(area_dst, page_nr));		count = *area_count(area_dst, page_nr);		if (count != count_verify[page_nr]) {			fprintf(stderr,				"page_nr %lu memory corruption %Lu %Lu\n",				page_nr, count,				count_verify[page_nr]); exit(1);		}		count++;		*area_count(area_dst, page_nr) = count_verify[page_nr] = count;		pthread_mutex_unlock(area_mutex(area_dst, page_nr)); 		if (time(NULL) - start > 1)			fprintf(stderr,

Lines 1–500 of 1700. Load more