/* * Stress userfaultfd syscall. * * Copyright (C) 2015 Red Hat, Inc. * * This work is licensed under the terms of the GNU GPL, version 2. See * the COPYING file in the top-level directory. * * This test allocates two virtual areas and bounces the physical * memory across the two virtual areas (from area_src to area_dst) * using userfaultfd. * * There are three threads running per CPU: * * 1) one per-CPU thread takes a per-page pthread_mutex in a random * page of the area_dst (while the physical page may still be in * area_src), and increments a per-page counter in the same page, * and checks its value against a verification region. * * 2) another per-CPU thread handles the userfaults generated by * thread 1 above. userfaultfd blocking reads or poll() modes are * exercised interleaved. * * 3) one last per-CPU thread transfers the memory in the background * at maximum bandwidth (if not already transferred by thread * 2). Each cpu thread takes cares of transferring a portion of the * area. * * When all threads of type 3 completed the transfer, one bounce is * complete. area_src and area_dst are then swapped. All threads are * respawned and so the bounce is immediately restarted in the * opposite direction. * * per-CPU threads 1 by triggering userfaults inside * pthread_mutex_lock will also verify the atomicity of the memory * transfer (UFFDIO_COPY). * * The program takes two parameters: the amounts of physical memory in * megabytes (MiB) of the area and the number of bounces to execute. * * # 100MiB 99999 bounces * ./userfaultfd 100 99999 * * # 1GiB 99 bounces * ./userfaultfd 1000 99 * * # 10MiB-~6GiB 999 bounces, continue forever unless an error triggers * while ./userfaultfd $[RANDOM % 6000 + 10] 999; do true; done */ #define _GNU_SOURCE#include <stdio.h>#include <errno.h>#include <unistd.h>#include <stdlib.h>#include <sys/types.h>#include <sys/stat.h>#include <fcntl.h>#include <time.h>#include <signal.h>#include <poll.h>#include <string.h>#include <sys/mman.h>#include <sys/syscall.h>#include <sys/ioctl.h>#include <sys/wait.h>#include <pthread.h>#include <linux/userfaultfd.h> #ifdef __NR_userfaultfd static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size; #define BOUNCE_RANDOM (1<<0)#define BOUNCE_RACINGFAULTS (1<<1)#define BOUNCE_VERIFY (1<<2)#define BOUNCE_POLL (1<<3)static int bounces; #define TEST_ANON 1#define TEST_HUGETLB 2#define TEST_SHMEM 3static int test_type; static int huge_fd;static char *huge_fd_off0;static unsigned long long *count_verify;static int uffd, uffd_flags, finished, *pipefd;static char *area_src, *area_dst;static char *zeropage;pthread_attr_t attr; /* pthread_mutex_t starts at page offset 0 */#define area_mutex(___area, ___nr) \ ((pthread_mutex_t *) ((___area) + (___nr)*page_size))/* * count is placed in the page after pthread_mutex_t naturally aligned * to avoid non alignment faults on non-x86 archs. */#define area_count(___area, ___nr) \ ((volatile unsigned long long *) ((unsigned long) \ ((___area) + (___nr)*page_size + \ sizeof(pthread_mutex_t) + \ sizeof(unsigned long long) - 1) & \ ~(unsigned long)(sizeof(unsigned long long) \ - 1))) static int anon_release_pages(char *rel_area){ int ret = 0; if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) { perror("madvise"); ret = 1; } return ret;} static void anon_allocate_area(void **alloc_area){ if (posix_memalign(alloc_area, page_size, nr_pages * page_size)) { fprintf(stderr, "out of memory\n"); *alloc_area = NULL; }} /* HugeTLB memory */static int hugetlb_release_pages(char *rel_area){ int ret = 0; if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, rel_area == huge_fd_off0 ? 0 : nr_pages * page_size, nr_pages * page_size)) { perror("fallocate"); ret = 1; } return ret;} static void hugetlb_allocate_area(void **alloc_area){ *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_HUGETLB, huge_fd, *alloc_area == area_src ? 0 : nr_pages * page_size); if (*alloc_area == MAP_FAILED) { fprintf(stderr, "mmap of hugetlbfs file failed\n"); *alloc_area = NULL; } if (*alloc_area == area_src) huge_fd_off0 = *alloc_area;} /* Shared memory */static int shmem_release_pages(char *rel_area){ int ret = 0; if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) { perror("madvise"); ret = 1; } return ret;} static void shmem_allocate_area(void **alloc_area){ *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, -1, 0); if (*alloc_area == MAP_FAILED) { fprintf(stderr, "shared memory mmap failed\n"); *alloc_area = NULL; }} struct uffd_test_ops { unsigned long expected_ioctls; void (*allocate_area)(void **alloc_area); int (*release_pages)(char *rel_area);}; #define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \ (1 << _UFFDIO_COPY) | \ (1 << _UFFDIO_ZEROPAGE)) static struct uffd_test_ops anon_uffd_test_ops = { .expected_ioctls = ANON_EXPECTED_IOCTLS, .allocate_area = anon_allocate_area, .release_pages = anon_release_pages,}; static struct uffd_test_ops shmem_uffd_test_ops = { .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC, .allocate_area = shmem_allocate_area, .release_pages = shmem_release_pages,}; static struct uffd_test_ops hugetlb_uffd_test_ops = { .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC, .allocate_area = hugetlb_allocate_area, .release_pages = hugetlb_release_pages,}; static struct uffd_test_ops *uffd_test_ops; static int my_bcmp(char *str1, char *str2, size_t n){ unsigned long i; for (i = 0; i < n; i++) if (str1[i] != str2[i]) return 1; return 0;} static void *locking_thread(void *arg){ unsigned long cpu = (unsigned long) arg; struct random_data rand; unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */ int32_t rand_nr; unsigned long long count; char randstate[64]; unsigned int seed; time_t start; if (bounces & BOUNCE_RANDOM) { seed = (unsigned int) time(NULL) - bounces; if (!(bounces & BOUNCE_RACINGFAULTS)) seed += cpu; bzero(&rand, sizeof(rand)); bzero(&randstate, sizeof(randstate)); if (initstate_r(seed, randstate, sizeof(randstate), &rand)) fprintf(stderr, "srandom_r error\n"), exit(1); } else { page_nr = -bounces; if (!(bounces & BOUNCE_RACINGFAULTS)) page_nr += cpu * nr_pages_per_cpu; } while (!finished) { if (bounces & BOUNCE_RANDOM) { if (random_r(&rand, &rand_nr)) fprintf(stderr, "random_r 1 error\n"), exit(1); page_nr = rand_nr; if (sizeof(page_nr) > sizeof(rand_nr)) { if (random_r(&rand, &rand_nr)) fprintf(stderr, "random_r 2 error\n"), exit(1); page_nr |= (((unsigned long) rand_nr) << 16) << 16; } } else page_nr += 1; page_nr %= nr_pages; start = time(NULL); if (bounces & BOUNCE_VERIFY) { count = *area_count(area_dst, page_nr); if (!count) fprintf(stderr, "page_nr %lu wrong count %Lu %Lu\n", page_nr, count, count_verify[page_nr]), exit(1); /* * We can't use bcmp (or memcmp) because that * returns 0 erroneously if the memory is * changing under it (even if the end of the * page is never changing and always * different). */#if 1 if (!my_bcmp(area_dst + page_nr * page_size, zeropage, page_size)) fprintf(stderr, "my_bcmp page_nr %lu wrong count %Lu %Lu\n", page_nr, count, count_verify[page_nr]), exit(1);#else unsigned long loops; loops = 0; /* uncomment the below line to test with mutex */ /* pthread_mutex_lock(area_mutex(area_dst, page_nr)); */ while (!bcmp(area_dst + page_nr * page_size, zeropage, page_size)) { loops += 1; if (loops > 10) break; } /* uncomment below line to test with mutex */ /* pthread_mutex_unlock(area_mutex(area_dst, page_nr)); */ if (loops) { fprintf(stderr, "page_nr %lu all zero thread %lu %p %lu\n", page_nr, cpu, area_dst + page_nr * page_size, loops); if (loops > 10) exit(1); }#endif } pthread_mutex_lock(area_mutex(area_dst, page_nr)); count = *area_count(area_dst, page_nr); if (count != count_verify[page_nr]) { fprintf(stderr, "page_nr %lu memory corruption %Lu %Lu\n", page_nr, count, count_verify[page_nr]), exit(1); } count++; *area_count(area_dst, page_nr) = count_verify[page_nr] = count; pthread_mutex_unlock(area_mutex(area_dst, page_nr)); if (time(NULL) - start > 1) fprintf(stderr, "userfault too slow %ld " "possible false positive with overcommit\n", time(NULL) - start); } return NULL;} static int copy_page(int ufd, unsigned long offset){ struct uffdio_copy uffdio_copy; if (offset >= nr_pages * page_size) fprintf(stderr, "unexpected offset %lu\n", offset), exit(1); uffdio_copy.dst = (unsigned long) area_dst + offset; uffdio_copy.src = (unsigned long) area_src + offset; uffdio_copy.len = page_size; uffdio_copy.mode = 0; uffdio_copy.copy = 0; if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) { /* real retval in ufdio_copy.copy */ if (uffdio_copy.copy != -EEXIST) fprintf(stderr, "UFFDIO_COPY error %Ld\n", uffdio_copy.copy), exit(1); } else if (uffdio_copy.copy != page_size) { fprintf(stderr, "UFFDIO_COPY unexpected copy %Ld\n", uffdio_copy.copy), exit(1); } else return 1; return 0;} static void *uffd_poll_thread(void *arg){ unsigned long cpu = (unsigned long) arg; struct pollfd pollfd[2]; struct uffd_msg msg; struct uffdio_register uffd_reg; int ret; unsigned long offset; char tmp_chr; unsigned long userfaults = 0; pollfd[0].fd = uffd; pollfd[0].events = POLLIN; pollfd[1].fd = pipefd[cpu*2]; pollfd[1].events = POLLIN; for (;;) { ret = poll(pollfd, 2, -1); if (!ret) fprintf(stderr, "poll error %d\n", ret), exit(1); if (ret < 0) perror("poll"), exit(1); if (pollfd[1].revents & POLLIN) { if (read(pollfd[1].fd, &tmp_chr, 1) != 1) fprintf(stderr, "read pipefd error\n"), exit(1); break; } if (!(pollfd[0].revents & POLLIN)) fprintf(stderr, "pollfd[0].revents %d\n", pollfd[0].revents), exit(1); ret = read(uffd, &msg, sizeof(msg)); if (ret < 0) { if (errno == EAGAIN) continue; perror("nonblocking read error"), exit(1); } switch (msg.event) { default: fprintf(stderr, "unexpected msg event %u\n", msg.event), exit(1); break; case UFFD_EVENT_PAGEFAULT: if (msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) fprintf(stderr, "unexpected write fault\n"), exit(1); offset = (char *)(unsigned long)msg.arg.pagefault.address - area_dst; offset &= ~(page_size-1); if (copy_page(uffd, offset)) userfaults++; break; case UFFD_EVENT_FORK: uffd = msg.arg.fork.ufd; pollfd[0].fd = uffd; break; case UFFD_EVENT_REMOVE: uffd_reg.range.start = msg.arg.remove.start; uffd_reg.range.len = msg.arg.remove.end - msg.arg.remove.start; if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range)) fprintf(stderr, "remove failure\n"), exit(1); break; case UFFD_EVENT_REMAP: area_dst = (char *)(unsigned long)msg.arg.remap.to; break; } } return (void *)userfaults;} pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER; static void *uffd_read_thread(void *arg){ unsigned long *this_cpu_userfaults; struct uffd_msg msg; unsigned long offset; int ret; this_cpu_userfaults = (unsigned long *) arg; *this_cpu_userfaults = 0; pthread_mutex_unlock(&uffd_read_mutex); /* from here cancellation is ok */ for (;;) { ret = read(uffd, &msg, sizeof(msg)); if (ret != sizeof(msg)) { if (ret < 0) perror("blocking read error"), exit(1); else fprintf(stderr, "short read\n"), exit(1); } if (msg.event != UFFD_EVENT_PAGEFAULT) fprintf(stderr, "unexpected msg event %u\n", msg.event), exit(1); if (bounces & BOUNCE_VERIFY && msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) fprintf(stderr, "unexpected write fault\n"), exit(1); offset = (char *)(unsigned long)msg.arg.pagefault.address - area_dst; offset &= ~(page_size-1); if (copy_page(uffd, offset)) (*this_cpu_userfaults)++; } return (void *)NULL;} static void *background_thread(void *arg){ unsigned long cpu = (unsigned long) arg; unsigned long page_nr; for (page_nr = cpu * nr_pages_per_cpu; page_nr < (cpu+1) * nr_pages_per_cpu; page_nr++) copy_page(uffd, page_nr * page_size); return NULL;} static int stress(unsigned long *userfaults){ unsigned long cpu; pthread_t locking_threads[nr_cpus]; pthread_t uffd_threads[nr_cpus]; pthread_t background_threads[nr_cpus]; void **_userfaults = (void **) userfaults; finished = 0; for (cpu = 0; cpu < nr_cpus; cpu++) { if (pthread_create(&locking_threads[cpu], &attr, locking_thread, (void *)cpu)) return 1; if (bounces & BOUNCE_POLL) { if (pthread_create(&uffd_threads[cpu], &attr, uffd_poll_thread, (void *)cpu)) return 1; } else { if (pthread_create(&uffd_threads[cpu], &attr, uffd_read_thread, &_userfaults[cpu]))Lines 1–500 of 1058. Load more
/* * Stress userfaultfd syscall. * * Copyright (C) 2015 Red Hat, Inc. * * This work is licensed under the terms of the GNU GPL, version 2. See * the COPYING file in the top-level directory. * * This test allocates two virtual areas and bounces the physical * memory across the two virtual areas (from area_src to area_dst) * using userfaultfd. * * There are three threads running per CPU: * * 1) one per-CPU thread takes a per-page pthread_mutex in a random * page of the area_dst (while the physical page may still be in * area_src), and increments a per-page counter in the same page, * and checks its value against a verification region. * * 2) another per-CPU thread handles the userfaults generated by * thread 1 above. userfaultfd blocking reads or poll() modes are * exercised interleaved. * * 3) one last per-CPU thread transfers the memory in the background * at maximum bandwidth (if not already transferred by thread * 2). Each cpu thread takes cares of transferring a portion of the * area. * * When all threads of type 3 completed the transfer, one bounce is * complete. area_src and area_dst are then swapped. All threads are * respawned and so the bounce is immediately restarted in the * opposite direction. * * per-CPU threads 1 by triggering userfaults inside * pthread_mutex_lock will also verify the atomicity of the memory * transfer (UFFDIO_COPY). * * The program takes two parameters: the amounts of physical memory in * megabytes (MiB) of the area and the number of bounces to execute. * * # 100MiB 99999 bounces * ./userfaultfd 100 99999 * * # 1GiB 99 bounces * ./userfaultfd 1000 99 * * # 10MiB-~6GiB 999 bounces, continue forever unless an error triggers * while ./userfaultfd $[RANDOM % 6000 + 10] 999; do true; done */ #define _GNU_SOURCE #include <stdio.h> #include <errno.h> #include <unistd.h> #include <stdlib.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #include <time.h> #include <signal.h> #include <poll.h> #include <string.h> #include <sys/mman.h> #include <sys/syscall.h> #include <sys/ioctl.h> #include <sys/wait.h> #include <pthread.h> #include <linux/userfaultfd.h> #ifdef __NR_userfaultfd static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size; #define BOUNCE_RANDOM (1<<0) #define BOUNCE_RACINGFAULTS (1<<1) #define BOUNCE_VERIFY (1<<2) #define BOUNCE_POLL (1<<3) static int bounces; #define TEST_ANON 1 #define TEST_HUGETLB 2 #define TEST_SHMEM 3 static int test_type; static int huge_fd; static char *huge_fd_off0; static unsigned long long *count_verify; static int uffd, uffd_flags, finished, *pipefd; static char *area_src, *area_dst; static char *zeropage; pthread_attr_t attr; /* pthread_mutex_t starts at page offset 0 */ #define area_mutex(___area, ___nr) \ ((pthread_mutex_t *) ((___area) + (___nr)*page_size)) /* * count is placed in the page after pthread_mutex_t naturally aligned * to avoid non alignment faults on non-x86 archs. */ #define area_count(___area, ___nr) \ ((volatile unsigned long long *) ((unsigned long) \ ((___area) + (___nr)*page_size + \ sizeof(pthread_mutex_t) + \ sizeof(unsigned long long) - 1) & \ ~(unsigned long)(sizeof(unsigned long long) \ - 1))) static int anon_release_pages(char *rel_area) { int ret = 0; if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) { perror("madvise"); ret = 1; } return ret; } static void anon_allocate_area(void **alloc_area) { if (posix_memalign(alloc_area, page_size, nr_pages * page_size)) { fprintf(stderr, "out of memory\n"); *alloc_area = NULL; } } /* HugeTLB memory */ static int hugetlb_release_pages(char *rel_area) { int ret = 0; if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, rel_area == huge_fd_off0 ? 0 : nr_pages * page_size, nr_pages * page_size)) { perror("fallocate"); ret = 1; } return ret; } static void hugetlb_allocate_area(void **alloc_area) { *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_HUGETLB, huge_fd, *alloc_area == area_src ? 0 : nr_pages * page_size); if (*alloc_area == MAP_FAILED) { fprintf(stderr, "mmap of hugetlbfs file failed\n"); *alloc_area = NULL; } if (*alloc_area == area_src) huge_fd_off0 = *alloc_area; } /* Shared memory */ static int shmem_release_pages(char *rel_area) { int ret = 0; if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) { perror("madvise"); ret = 1; } return ret; } static void shmem_allocate_area(void **alloc_area) { *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_SHARED, -1, 0); if (*alloc_area == MAP_FAILED) { fprintf(stderr, "shared memory mmap failed\n"); *alloc_area = NULL; } } struct uffd_test_ops { unsigned long expected_ioctls; void (*allocate_area)(void **alloc_area); int (*release_pages)(char *rel_area); }; #define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \ (1 << _UFFDIO_COPY) | \ (1 << _UFFDIO_ZEROPAGE)) static struct uffd_test_ops anon_uffd_test_ops = { .expected_ioctls = ANON_EXPECTED_IOCTLS, .allocate_area = anon_allocate_area, .release_pages = anon_release_pages, }; static struct uffd_test_ops shmem_uffd_test_ops = { .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC, .allocate_area = shmem_allocate_area, .release_pages = shmem_release_pages, }; static struct uffd_test_ops hugetlb_uffd_test_ops = { .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC, .allocate_area = hugetlb_allocate_area, .release_pages = hugetlb_release_pages, }; static struct uffd_test_ops *uffd_test_ops; static int my_bcmp(char *str1, char *str2, size_t n) { unsigned long i; for (i = 0; i < n; i++) if (str1[i] != str2[i]) return 1; return 0; } static void *locking_thread(void *arg) { unsigned long cpu = (unsigned long) arg; struct random_data rand; unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */ int32_t rand_nr; unsigned long long count; char randstate[64]; unsigned int seed; time_t start; if (bounces & BOUNCE_RANDOM) { seed = (unsigned int) time(NULL) - bounces; if (!(bounces & BOUNCE_RACINGFAULTS)) seed += cpu; bzero(&rand, sizeof(rand)); bzero(&randstate, sizeof(randstate)); if (initstate_r(seed, randstate, sizeof(randstate), &rand)) fprintf(stderr, "srandom_r error\n"), exit(1); } else { page_nr = -bounces; if (!(bounces & BOUNCE_RACINGFAULTS)) page_nr += cpu * nr_pages_per_cpu; } while (!finished) { if (bounces & BOUNCE_RANDOM) { if (random_r(&rand, &rand_nr)) fprintf(stderr, "random_r 1 error\n"), exit(1); page_nr = rand_nr; if (sizeof(page_nr) > sizeof(rand_nr)) { if (random_r(&rand, &rand_nr)) fprintf(stderr, "random_r 2 error\n"), exit(1); page_nr |= (((unsigned long) rand_nr) << 16) << 16; } } else page_nr += 1; page_nr %= nr_pages; start = time(NULL); if (bounces & BOUNCE_VERIFY) { count = *area_count(area_dst, page_nr); if (!count) fprintf(stderr, "page_nr %lu wrong count %Lu %Lu\n", page_nr, count, count_verify[page_nr]), exit(1); /* * We can't use bcmp (or memcmp) because that * returns 0 erroneously if the memory is * changing under it (even if the end of the * page is never changing and always * different). */ #if 1 if (!my_bcmp(area_dst + page_nr * page_size, zeropage, page_size)) fprintf(stderr, "my_bcmp page_nr %lu wrong count %Lu %Lu\n", page_nr, count, count_verify[page_nr]), exit(1); #else unsigned long loops; loops = 0; /* uncomment the below line to test with mutex */ /* pthread_mutex_lock(area_mutex(area_dst, page_nr)); */ while (!bcmp(area_dst + page_nr * page_size, zeropage, page_size)) { loops += 1; if (loops > 10) break; } /* uncomment below line to test with mutex */ /* pthread_mutex_unlock(area_mutex(area_dst, page_nr)); */ if (loops) { fprintf(stderr, "page_nr %lu all zero thread %lu %p %lu\n", page_nr, cpu, area_dst + page_nr * page_size, loops); if (loops > 10) exit(1); } #endif } pthread_mutex_lock(area_mutex(area_dst, page_nr)); count = *area_count(area_dst, page_nr); if (count != count_verify[page_nr]) { fprintf(stderr, "page_nr %lu memory corruption %Lu %Lu\n", page_nr, count, count_verify[page_nr]), exit(1); } count++; *area_count(area_dst, page_nr) = count_verify[page_nr] = count; pthread_mutex_unlock(area_mutex(area_dst, page_nr)); if (time(NULL) - start > 1) fprintf(stderr, "userfault too slow %ld " "possible false positive with overcommit\n", time(NULL) - start); } return NULL; } static int copy_page(int ufd, unsigned long offset) { struct uffdio_copy uffdio_copy; if (offset >= nr_pages * page_size) fprintf(stderr, "unexpected offset %lu\n", offset), exit(1); uffdio_copy.dst = (unsigned long) area_dst + offset; uffdio_copy.src = (unsigned long) area_src + offset; uffdio_copy.len = page_size; uffdio_copy.mode = 0; uffdio_copy.copy = 0; if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) { /* real retval in ufdio_copy.copy */ if (uffdio_copy.copy != -EEXIST) fprintf(stderr, "UFFDIO_COPY error %Ld\n", uffdio_copy.copy), exit(1); } else if (uffdio_copy.copy != page_size) { fprintf(stderr, "UFFDIO_COPY unexpected copy %Ld\n", uffdio_copy.copy), exit(1); } else return 1; return 0; } static void *uffd_poll_thread(void *arg) { unsigned long cpu = (unsigned long) arg; struct pollfd pollfd[2]; struct uffd_msg msg; struct uffdio_register uffd_reg; int ret; unsigned long offset; char tmp_chr; unsigned long userfaults = 0; pollfd[0].fd = uffd; pollfd[0].events = POLLIN; pollfd[1].fd = pipefd[cpu*2]; pollfd[1].events = POLLIN; for (;;) { ret = poll(pollfd, 2, -1); if (!ret) fprintf(stderr, "poll error %d\n", ret), exit(1); if (ret < 0) perror("poll"), exit(1); if (pollfd[1].revents & POLLIN) { if (read(pollfd[1].fd, &tmp_chr, 1) != 1) fprintf(stderr, "read pipefd error\n"), exit(1); break; } if (!(pollfd[0].revents & POLLIN)) fprintf(stderr, "pollfd[0].revents %d\n", pollfd[0].revents), exit(1); ret = read(uffd, &msg, sizeof(msg)); if (ret < 0) { if (errno == EAGAIN) continue; perror("nonblocking read error"), exit(1); } switch (msg.event) { default: fprintf(stderr, "unexpected msg event %u\n", msg.event), exit(1); break; case UFFD_EVENT_PAGEFAULT: if (msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) fprintf(stderr, "unexpected write fault\n"), exit(1); offset = (char *)(unsigned long)msg.arg.pagefault.address - area_dst; offset &= ~(page_size-1); if (copy_page(uffd, offset)) userfaults++; break; case UFFD_EVENT_FORK: uffd = msg.arg.fork.ufd; pollfd[0].fd = uffd; break; case UFFD_EVENT_REMOVE: uffd_reg.range.start = msg.arg.remove.start; uffd_reg.range.len = msg.arg.remove.end - msg.arg.remove.start; if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range)) fprintf(stderr, "remove failure\n"), exit(1); break; case UFFD_EVENT_REMAP: area_dst = (char *)(unsigned long)msg.arg.remap.to; break; } } return (void *)userfaults; } pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER; static void *uffd_read_thread(void *arg) { unsigned long *this_cpu_userfaults; struct uffd_msg msg; unsigned long offset; int ret; this_cpu_userfaults = (unsigned long *) arg; *this_cpu_userfaults = 0; pthread_mutex_unlock(&uffd_read_mutex); /* from here cancellation is ok */ for (;;) { ret = read(uffd, &msg, sizeof(msg)); if (ret != sizeof(msg)) { if (ret < 0) perror("blocking read error"), exit(1); else fprintf(stderr, "short read\n"), exit(1); } if (msg.event != UFFD_EVENT_PAGEFAULT) fprintf(stderr, "unexpected msg event %u\n", msg.event), exit(1); if (bounces & BOUNCE_VERIFY && msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) fprintf(stderr, "unexpected write fault\n"), exit(1); offset = (char *)(unsigned long)msg.arg.pagefault.address - area_dst; offset &= ~(page_size-1); if (copy_page(uffd, offset)) (*this_cpu_userfaults)++; } return (void *)NULL; } static void *background_thread(void *arg) { unsigned long cpu = (unsigned long) arg; unsigned long page_nr; for (page_nr = cpu * nr_pages_per_cpu; page_nr < (cpu+1) * nr_pages_per_cpu; page_nr++) copy_page(uffd, page_nr * page_size); return NULL; } static int stress(unsigned long *userfaults) { unsigned long cpu; pthread_t locking_threads[nr_cpus]; pthread_t uffd_threads[nr_cpus]; pthread_t background_threads[nr_cpus]; void **_userfaults = (void **) userfaults; finished = 0; for (cpu = 0; cpu < nr_cpus; cpu++) { if (pthread_create(&locking_threads[cpu], &attr, locking_thread, (void *)cpu)) return 1; if (bounces & BOUNCE_POLL) { if (pthread_create(&uffd_threads[cpu], &attr, uffd_poll_thread, (void *)cpu)) return 1; } else { if (pthread_create(&uffd_threads[cpu], &attr, uffd_read_thread, &_userfaults[cpu]))