#!/usr/bin/env node --redirect-warnings=/dev/null const fs = require("fs");const path = require("path");const { spawnSync } = require("child_process"); const FAILING_SEED_REGEX = /failing seed: (\d+)/gi;const CARGO_TEST_ARGS = ["--release", "--lib", "--package", "collab"]; if (require.main === module) { if (process.argv.length < 4) { process.stderr.write( "usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n", ); process.exit(1); } minimizeTestPlan( process.argv[2], process.argv[3], parseInt(process.argv[4]) || 0, );} function minimizeTestPlan(inputPlanPath, outputPlanPath, startIndex = 0) { const tempPlanPath = inputPlanPath + ".try"; fs.copyFileSync(inputPlanPath, outputPlanPath); let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8")); process.stderr.write("minimizing failing test plan...\n"); for (let ix = startIndex; ix < testPlan.length; ix++) { // Skip 'MutateClients' entries, since they themselves are not single operations. if (testPlan[ix].MutateClients) { continue; } // Remove a row from the test plan const newTestPlan = testPlan.slice(); newTestPlan.splice(ix, 1); fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), "utf8"); process.stderr.write( `${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`, ); const failingSeed = runTests({ SEED: "0", LOAD_PLAN: tempPlanPath, SAVE_PLAN: tempPlanPath, ITERATIONS: "500", }); // If the test failed, keep the test plan with the removed row. Reload the test // plan from the JSON file, since the test itself will remove any operations // which are no longer valid before saving the test plan. if (failingSeed != null) { process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`); fs.copyFileSync(tempPlanPath, outputPlanPath); testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8")); ix--; } else { process.stderr.write(` - keep.\n`); } } fs.unlinkSync(tempPlanPath); // Re-run the final minimized plan to get the correct failing seed. // This is a workaround for the fact that the execution order can // slightly change when replaying a test plan after it has been // saved and loaded. const failingSeed = runTests({ SEED: "0", ITERATIONS: "5000", LOAD_PLAN: outputPlanPath, }); process.stderr.write(`final test plan: ${outputPlanPath}\n`); process.stderr.write(`final seed: ${failingSeed}\n`); return failingSeed;} function buildTests() { const { status } = spawnSync( "cargo", ["test", "--no-run", ...CARGO_TEST_ARGS], { stdio: "inherit", encoding: "utf8", env: { ...process.env, }, }, ); if (status !== 0) { throw new Error("build failed"); }} function runTests(env) { const { status, stdout } = spawnSync( "cargo", ["test", ...CARGO_TEST_ARGS, "random_project_collaboration"], { stdio: "pipe", encoding: "utf8", env: { ...process.env, ...env, }, }, ); if (status !== 0) { FAILING_SEED_REGEX.lastIndex = 0; const match = FAILING_SEED_REGEX.exec(stdout); if (!match) { process.stderr.write("test failed, but no failing seed found:\n"); process.stderr.write(stdout); process.stderr.write("\n"); process.exit(1); } return match[1]; } else { return null; }} function serializeTestPlan(plan) { return "[\n" + plan.map((row) => JSON.stringify(row)).join(",\n") + "\n]\n";} exports.buildTests = buildTests;exports.runTests = runTests;exports.minimizeTestPlan = minimizeTestPlan;
#!/usr/bin/env node --redirect-warnings=/dev/null
const fs = require("fs");
const path = require("path");
const { spawnSync } = require("child_process");
const FAILING_SEED_REGEX = /failing seed: (\d+)/gi;
const CARGO_TEST_ARGS = ["--release", "--lib", "--package", "collab"];
if (require.main === module) {
if (process.argv.length < 4) {
process.stderr.write(
"usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n",
);
process.exit(1);
}
minimizeTestPlan(
process.argv[2],
process.argv[3],
parseInt(process.argv[4]) || 0,
);
}
function minimizeTestPlan(inputPlanPath, outputPlanPath, startIndex = 0) {
const tempPlanPath = inputPlanPath + ".try";
fs.copyFileSync(inputPlanPath, outputPlanPath);
let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8"));
process.stderr.write("minimizing failing test plan...\n");
for (let ix = startIndex; ix < testPlan.length; ix++) {
// Skip 'MutateClients' entries, since they themselves are not single operations.
if (testPlan[ix].MutateClients) {
continue;
}
// Remove a row from the test plan
const newTestPlan = testPlan.slice();
newTestPlan.splice(ix, 1);
fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), "utf8");
process.stderr.write(
`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`,
);
const failingSeed = runTests({
SEED: "0",
LOAD_PLAN: tempPlanPath,
SAVE_PLAN: tempPlanPath,
ITERATIONS: "500",
});
// If the test failed, keep the test plan with the removed row. Reload the test
// plan from the JSON file, since the test itself will remove any operations
// which are no longer valid before saving the test plan.
if (failingSeed != null) {
process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`);
fs.copyFileSync(tempPlanPath, outputPlanPath);
testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8"));
ix--;
} else {
process.stderr.write(` - keep.\n`);
}
}
fs.unlinkSync(tempPlanPath);
// Re-run the final minimized plan to get the correct failing seed.
// This is a workaround for the fact that the execution order can
// slightly change when replaying a test plan after it has been
// saved and loaded.
const failingSeed = runTests({
SEED: "0",
ITERATIONS: "5000",
LOAD_PLAN: outputPlanPath,
});
process.stderr.write(`final test plan: ${outputPlanPath}\n`);
process.stderr.write(`final seed: ${failingSeed}\n`);
return failingSeed;
}
function buildTests() {
const { status } = spawnSync(
"cargo",
["test", "--no-run", ...CARGO_TEST_ARGS],
{
stdio: "inherit",
encoding: "utf8",
env: {
...process.env,
},
},
);
if (status !== 0) {
throw new Error("build failed");
}
}
function runTests(env) {
const { status, stdout } = spawnSync(
"cargo",
["test", ...CARGO_TEST_ARGS, "random_project_collaboration"],
{
stdio: "pipe",
encoding: "utf8",
env: {
...process.env,
...env,
},
},
);
if (status !== 0) {
FAILING_SEED_REGEX.lastIndex = 0;
const match = FAILING_SEED_REGEX.exec(stdout);
if (!match) {
process.stderr.write("test failed, but no failing seed found:\n");
process.stderr.write(stdout);
process.stderr.write("\n");
process.exit(1);
}
return match[1];
} else {
return null;
}
}
function serializeTestPlan(plan) {
return "[\n" + plan.map((row) => JSON.stringify(row)).join(",\n") + "\n]\n";
}
exports.buildTests = buildTests;
exports.runTests = runTests;
exports.minimizeTestPlan = minimizeTestPlan;