// SPDX-License-Identifier: GPL-2.0/* * The main purpose of the tests here is to exercise the migration entry code * paths in the kernel. */ #include "kselftest_harness.h"#include "hugepage_settings.h" #include <string.h>#include <pthread.h>#include <numa.h>#include <numaif.h>#include <sys/mman.h>#include <sys/prctl.h>#include <sys/types.h>#include <signal.h>#include <time.h>#include "vm_util.h" #define TWOMEG (2<<20)#define RUNTIME (20)#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) HUGETLB_SETUP_DEFAULT_PAGES(1) FIXTURE(migration){ pthread_t *threads; pid_t *pids; int nthreads; int n1; int n2;}; static void reset_signals(void){ struct sigaction sa = { .sa_handler = SIG_DFL }; sigemptyset(&sa.sa_mask); sigaction(SIGTERM, &sa, NULL); sigaction(SIGHUP, &sa, NULL); sigaction(SIGINT, &sa, NULL); sigaction(SIGQUIT, &sa, NULL);} FIXTURE_SETUP(migration){ int n; reset_signals(); if (numa_available() < 0) SKIP(return, "NUMA not available"); self->nthreads = numa_num_task_cpus() - 2; self->n1 = -1; self->n2 = -1; for (n = 0; n < numa_max_possible_node(); n++) if (numa_bitmask_isbitset(numa_all_nodes_ptr, n)) { if (self->n1 == -1) { self->n1 = n; } else { self->n2 = n; break; } } if (self->nthreads < 1 || self->n1 < 0 || self->n2 < 0) SKIP(return, "Not enough threads or NUMA nodes available"); self->threads = malloc(self->nthreads * sizeof(*self->threads)); ASSERT_NE(self->threads, NULL); self->pids = malloc(self->nthreads * sizeof(*self->pids)); ASSERT_NE(self->pids, NULL);}; FIXTURE_TEARDOWN(migration){ free(self->threads); free(self->pids);} static bool kill_children(FIXTURE_DATA(migration) * self){ bool err = false; pid_t pid; int i; for (i = 0; i < self->nthreads; i++) { int status = 0; pid = self->pids[i]; if (pid < 0) continue; if (kill(pid, SIGTERM)) err = true; if (pid != waitpid(pid, &status, 0)) err = true; if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGTERM) err = true; } return !err;} int migrate(uint64_t *ptr, int n1, int n2){ int ret, tmp; int status = 0; struct timespec ts1, ts2; int success = 0; if (clock_gettime(CLOCK_MONOTONIC, &ts1)) return -1; while (1) { if (clock_gettime(CLOCK_MONOTONIC, &ts2)) return -1; if (ts2.tv_sec - ts1.tv_sec >= RUNTIME) { /* Reaching both targets verifies a cross-node move. */ if (success >= 2) return 0; else return -2; } ret = move_pages(0, 1, (void **) &ptr, &n2, &status, MPOL_MF_MOVE_ALL); if (ret < 0) { perror("Couldn't migrate pages"); return ret; } /* Migration is best effort. Try again */ if (ret > 0 || status < 0) continue; if (status != n2) { printf("Page is on node %d instead of target node %d\n", status, n2); return -2; } success++; tmp = n2; n2 = n1; n1 = tmp; }} void *access_mem(void *ptr){ while (1) { pthread_testcancel(); /* Force a read from the memory pointed to by ptr. This ensures * the memory access actually happens and prevents the compiler * from optimizing away this entire loop. */ FORCE_READ(*(uint64_t *)ptr); } return NULL;} /* * Basic migration entry testing. One thread will move pages back and forth * between nodes whilst other threads try and access them triggering the * migration entry wait paths in the kernel. */TEST_F_TIMEOUT(migration, private_anon, 2*RUNTIME){ uint64_t *ptr; int i; ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, TWOMEG); for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0);} /* * Same as the previous test but with shared memory. */TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME){ pid_t pid; uint64_t *ptr; int i, err; ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, TWOMEG); for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); /* Parent may have died before prctl so check now. */ if (getppid() == 1) kill(getpid(), SIGHUP); access_mem(ptr); } else { self->pids[i] = pid; } } err = migrate(ptr, self->n1, self->n2); ASSERT_EQ(kill_children(self), true); ASSERT_EQ(err, 0);} /* * Tests the pmd migration entry paths. */TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME){ uint64_t pmdsize; uint64_t *ptr; int i; if (!thp_is_enabled()) SKIP(return, "Transparent Hugepages not available"); pmdsize = read_pmd_pagesize(); if (!pmdsize) SKIP(return, "Reading PMD pagesize failed"); ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); memset(ptr, 0xde, pmdsize); for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0);} /* * migration test with shared anon THP page */ TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME){ uint64_t pmdsize; pid_t pid; uint64_t *ptr; int i, err; if (!thp_is_enabled()) SKIP(return, "Transparent Hugepages not available"); pmdsize = read_pmd_pagesize(); if (!pmdsize) SKIP(return, "Reading PMD pagesize failed"); ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); ASSERT_NE(ptr, MAP_FAILED); ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize); ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0); memset(ptr, 0xde, pmdsize); for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); /* Parent may have died before prctl so check now. */ if (getppid() == 1) kill(getpid(), SIGHUP); access_mem(ptr); } else { self->pids[i] = pid; } } err = migrate(ptr, self->n1, self->n2); ASSERT_EQ(kill_children(self), true); ASSERT_EQ(err, 0);} /* * migration test with private anon hugetlb page */TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME){ unsigned long hugepage_size; uint64_t *ptr; int i; hugepage_size = default_huge_page_size(); if (!hugepage_size) SKIP(return, "Reading HugeTLB pagesize failed"); if (hugetlb_free_default_pages() < 1) SKIP(return, "Not enough huge pages"); ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, hugepage_size); for (i = 0; i < self->nthreads; i++) if (pthread_create(&self->threads[i], NULL, access_mem, ptr)) perror("Couldn't create thread"); ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0); for (i = 0; i < self->nthreads; i++) ASSERT_EQ(pthread_cancel(self->threads[i]), 0);} /* * migration test with shared anon hugetlb page */TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME){ unsigned long hugepage_size; pid_t pid; uint64_t *ptr; int i, err; hugepage_size = default_huge_page_size(); if (!hugepage_size) SKIP(return, "Reading HugeTLB pagesize failed"); if (hugetlb_free_default_pages() < 1) SKIP(return, "Not enough huge pages"); ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); ASSERT_NE(ptr, MAP_FAILED); memset(ptr, 0xde, hugepage_size); for (i = 0; i < self->nthreads; i++) { pid = fork(); if (!pid) { prctl(PR_SET_PDEATHSIG, SIGHUP); /* Parent may have died before prctl so check now. */ if (getppid() == 1) kill(getpid(), SIGHUP); access_mem(ptr); } else { self->pids[i] = pid; } } err = migrate(ptr, self->n1, self->n2); ASSERT_EQ(kill_children(self), true); ASSERT_EQ(err, 0);} TEST_HARNESS_MAIN
// SPDX-License-Identifier: GPL-2.0
/*
* The main purpose of the tests here is to exercise the migration entry code
* paths in the kernel.
*/
#include "kselftest_harness.h"
#include "hugepage_settings.h"
#include <string.h>
#include <pthread.h>
#include <numa.h>
#include <numaif.h>
#include <sys/mman.h>
#include <sys/prctl.h>
#include <sys/types.h>
#include <signal.h>
#include <time.h>
#include "vm_util.h"
#define TWOMEG (2<<20)
#define RUNTIME (20)
#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
HUGETLB_SETUP_DEFAULT_PAGES(1)
FIXTURE(migration)
{
pthread_t *threads;
pid_t *pids;
int nthreads;
int n1;
int n2;
};
static void reset_signals(void)
{
struct sigaction sa = { .sa_handler = SIG_DFL };
sigemptyset(&sa.sa_mask);
sigaction(SIGTERM, &sa, NULL);
sigaction(SIGHUP, &sa, NULL);
sigaction(SIGINT, &sa, NULL);
sigaction(SIGQUIT, &sa, NULL);
}
FIXTURE_SETUP(migration)
{
int n;
reset_signals();
if (numa_available() < 0)
SKIP(return, "NUMA not available");
self->nthreads = numa_num_task_cpus() - 2;
self->n1 = -1;
self->n2 = -1;
for (n = 0; n < numa_max_possible_node(); n++)
if (numa_bitmask_isbitset(numa_all_nodes_ptr, n)) {
if (self->n1 == -1) {
self->n1 = n;
} else {
self->n2 = n;
break;
}
}
if (self->nthreads < 1 || self->n1 < 0 || self->n2 < 0)
SKIP(return, "Not enough threads or NUMA nodes available");
self->threads = malloc(self->nthreads * sizeof(*self->threads));
ASSERT_NE(self->threads, NULL);
self->pids = malloc(self->nthreads * sizeof(*self->pids));
ASSERT_NE(self->pids, NULL);
};
FIXTURE_TEARDOWN(migration)
{
free(self->threads);
free(self->pids);
}
static bool kill_children(FIXTURE_DATA(migration) * self)
{
bool err = false;
pid_t pid;
int i;
for (i = 0; i < self->nthreads; i++) {
int status = 0;
pid = self->pids[i];
if (pid < 0)
continue;
if (kill(pid, SIGTERM))
err = true;
if (pid != waitpid(pid, &status, 0))
err = true;
if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGTERM)
err = true;
}
return !err;
}
int migrate(uint64_t *ptr, int n1, int n2)
{
int ret, tmp;
int status = 0;
struct timespec ts1, ts2;
int success = 0;
if (clock_gettime(CLOCK_MONOTONIC, &ts1))
return -1;
while (1) {
if (clock_gettime(CLOCK_MONOTONIC, &ts2))
return -1;
if (ts2.tv_sec - ts1.tv_sec >= RUNTIME) {
/* Reaching both targets verifies a cross-node move. */
if (success >= 2)
return 0;
else
return -2;
}
ret = move_pages(0, 1, (void **) &ptr, &n2, &status,
MPOL_MF_MOVE_ALL);
if (ret < 0) {
perror("Couldn't migrate pages");
return ret;
}
/* Migration is best effort. Try again */
if (ret > 0 || status < 0)
continue;
if (status != n2) {
printf("Page is on node %d instead of target node %d\n",
status, n2);
return -2;
}
success++;
tmp = n2;
n2 = n1;
n1 = tmp;
}
}
void *access_mem(void *ptr)
{
while (1) {
pthread_testcancel();
/* Force a read from the memory pointed to by ptr. This ensures
* the memory access actually happens and prevents the compiler
* from optimizing away this entire loop.
*/
FORCE_READ(*(uint64_t *)ptr);
}
return NULL;
}
/*
* Basic migration entry testing. One thread will move pages back and forth
* between nodes whilst other threads try and access them triggering the
* migration entry wait paths in the kernel.
*/
TEST_F_TIMEOUT(migration, private_anon, 2*RUNTIME)
{
uint64_t *ptr;
int i;
ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
ASSERT_NE(ptr, MAP_FAILED);
memset(ptr, 0xde, TWOMEG);
for (i = 0; i < self->nthreads; i++)
if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
perror("Couldn't create thread");
ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
for (i = 0; i < self->nthreads; i++)
ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
}
/*
* Same as the previous test but with shared memory.
*/
TEST_F_TIMEOUT(migration, shared_anon, 2*RUNTIME)
{
pid_t pid;
uint64_t *ptr;
int i, err;
ptr = mmap(NULL, TWOMEG, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
ASSERT_NE(ptr, MAP_FAILED);
memset(ptr, 0xde, TWOMEG);
for (i = 0; i < self->nthreads; i++) {
pid = fork();
if (!pid) {
prctl(PR_SET_PDEATHSIG, SIGHUP);
/* Parent may have died before prctl so check now. */
if (getppid() == 1)
kill(getpid(), SIGHUP);
access_mem(ptr);
} else {
self->pids[i] = pid;
}
}
err = migrate(ptr, self->n1, self->n2);
ASSERT_EQ(kill_children(self), true);
ASSERT_EQ(err, 0);
}
/*
* Tests the pmd migration entry paths.
*/
TEST_F_TIMEOUT(migration, private_anon_thp, 2*RUNTIME)
{
uint64_t pmdsize;
uint64_t *ptr;
int i;
if (!thp_is_enabled())
SKIP(return, "Transparent Hugepages not available");
pmdsize = read_pmd_pagesize();
if (!pmdsize)
SKIP(return, "Reading PMD pagesize failed");
ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
ASSERT_NE(ptr, MAP_FAILED);
ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize);
ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0);
memset(ptr, 0xde, pmdsize);
for (i = 0; i < self->nthreads; i++)
if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
perror("Couldn't create thread");
ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
for (i = 0; i < self->nthreads; i++)
ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
}
/*
* migration test with shared anon THP page
*/
TEST_F_TIMEOUT(migration, shared_anon_thp, 2*RUNTIME)
{
uint64_t pmdsize;
pid_t pid;
uint64_t *ptr;
int i, err;
if (!thp_is_enabled())
SKIP(return, "Transparent Hugepages not available");
pmdsize = read_pmd_pagesize();
if (!pmdsize)
SKIP(return, "Reading PMD pagesize failed");
ptr = mmap(NULL, 2 * pmdsize, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
ASSERT_NE(ptr, MAP_FAILED);
ptr = (uint64_t *) ALIGN((uintptr_t) ptr, pmdsize);
ASSERT_EQ(madvise(ptr, pmdsize, MADV_HUGEPAGE), 0);
memset(ptr, 0xde, pmdsize);
for (i = 0; i < self->nthreads; i++) {
pid = fork();
if (!pid) {
prctl(PR_SET_PDEATHSIG, SIGHUP);
/* Parent may have died before prctl so check now. */
if (getppid() == 1)
kill(getpid(), SIGHUP);
access_mem(ptr);
} else {
self->pids[i] = pid;
}
}
err = migrate(ptr, self->n1, self->n2);
ASSERT_EQ(kill_children(self), true);
ASSERT_EQ(err, 0);
}
/*
* migration test with private anon hugetlb page
*/
TEST_F_TIMEOUT(migration, private_anon_htlb, 2*RUNTIME)
{
unsigned long hugepage_size;
uint64_t *ptr;
int i;
hugepage_size = default_huge_page_size();
if (!hugepage_size)
SKIP(return, "Reading HugeTLB pagesize failed");
if (hugetlb_free_default_pages() < 1)
SKIP(return, "Not enough huge pages");
ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
ASSERT_NE(ptr, MAP_FAILED);
memset(ptr, 0xde, hugepage_size);
for (i = 0; i < self->nthreads; i++)
if (pthread_create(&self->threads[i], NULL, access_mem, ptr))
perror("Couldn't create thread");
ASSERT_EQ(migrate(ptr, self->n1, self->n2), 0);
for (i = 0; i < self->nthreads; i++)
ASSERT_EQ(pthread_cancel(self->threads[i]), 0);
}
/*
* migration test with shared anon hugetlb page
*/
TEST_F_TIMEOUT(migration, shared_anon_htlb, 2*RUNTIME)
{
unsigned long hugepage_size;
pid_t pid;
uint64_t *ptr;
int i, err;
hugepage_size = default_huge_page_size();
if (!hugepage_size)
SKIP(return, "Reading HugeTLB pagesize failed");
if (hugetlb_free_default_pages() < 1)
SKIP(return, "Not enough huge pages");
ptr = mmap(NULL, hugepage_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
ASSERT_NE(ptr, MAP_FAILED);
memset(ptr, 0xde, hugepage_size);
for (i = 0; i < self->nthreads; i++) {
pid = fork();
if (!pid) {
prctl(PR_SET_PDEATHSIG, SIGHUP);
/* Parent may have died before prctl so check now. */
if (getppid() == 1)
kill(getpid(), SIGHUP);
access_mem(ptr);
} else {
self->pids[i] = pid;
}
}
err = migrate(ptr, self->n1, self->n2);
ASSERT_EQ(kill_children(self), true);
ASSERT_EQ(err, 0);
}
TEST_HARNESS_MAIN