/** * Differential 3-layer process-spawn benchmark (PoC for the fuzz-perf harness). * * Primary op `node_exit` = spawn a process that runs `node -e 'process.exit(0)'` and * wait for it to exit, measured at three layers so the ratios are honest: * *   native floor   real host node via Rust std::process   (native-baseline binary) *   node reference  real host node via Node child_process  (in-process, host) *   guest           node inside a secure-exec VM (a fresh V8 isolate per spawn) * *   emulation_tax = guest.p50 / node.p50   <- what secure-exec's emulation adds *   total_tax     = guest.p50 / native.p50 <- total cost over the raw host floor * * The guest layer hits the V8-isolate-per-spawn path that the surface map flagged as * the dominant process cost (a new isolate + 2 OS threads + snapshot deserialize per * spawn). We compare it against a real host node process doing the same logical op, so * the tax reflects emulation overhead rather than "JS startup is slow". * * For absolute-floor context we also print the native cost of a libc `sh -c 'exit 0'` * (op `spawn_exit`) — ~0.3 ms — to show how far the node-spawn floor already sits above * the cheapest possible process. The WASM-shell guest path (sh/echo) is a future * addition once the registry WASM binaries are built. * * Run: *   pnpm exec tsx scripts/benchmarks/process-spawn.bench.ts *   BENCH_ITERATIONS=5 BENCH_WARMUP=2 pnpm exec tsx scripts/benchmarks/process-spawn.bench.ts  # smoke * * Low-variance discipline: pin a core (taskset -c) and pass --expose-gc; one shared VM * is reused across all guest iters (no per-iter VM boot). Trust p50/p95/p99, not mean. */ import { type ChildProcess, spawn, spawnSync } from "node:child_process";import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { createBenchSidecar, createBenchVm, type BenchVm } from "../lib/vm.js";import type { SidecarProcess } from "@rivet-dev/agentos-runtime-core";import {	forceGC,	getHardware,	printTable,	round,	stats,} from "../lib/perf-utils.js";import { runNativeLayer } from "../lib/native.js";import { runNativePhaseLayer } from "../lib/native.js"; const ITERATIONS = Number(process.env.BENCH_ITERATIONS ?? 200);const WARMUP = Number(process.env.BENCH_WARMUP ?? 20);const PROCESS_LIFECYCLE_TRACE =	process.env.BENCH_PROCESS_LIFECYCLE_TRACE === "1"; const NODE_ARGS = ["-e", "process.exit(0)"];const FANOUT = 8;const FANOUT_LADDER = [1, 8, 32] as const; type PhaseName = "total" | "spawn" | "wait_reap" | "spawn_batch" | "wait_reap_batch";type PhaseSamples = Partial<Record<PhaseName, number[]>>; interface ProcessLifecycleTrace {	version: "BENCH-035" | "BENCH-069";	pid: number;	processId: string;	command: string;	args: string[];	firstWaiterSource?: string;	lastWaiterSource?: string;	waiterSources?: Record<string, number>;	spawnToWaitBeginMs?: number;	waitBlockedOnStartMs?: number;	waitTotalMs?: number;	trailingOutputDrainMs?: number;	finishRoute?: string;	start?: Record<string, number | boolean | undefined>;	wait?: Record<string, number | boolean | string | undefined>;	exit?: Record<string, number | undefined>;		output?: Record<string, number | undefined>;	} interface SidecarExecutePhaseMetric {	stage: string;	calls: number;	totalUs: number;	avgUs: number;	maxUs: number;} interface HostWriteSyncRow {	row: "root_cwd_no_write" | "host_cwd_no_write" | "host_cwd_write_1file";	cwd: string;	writesPerProcess: number;	samples: number;	wallMs: ReturnType<typeof stats>;	sidecarExecutePhases: SidecarExecutePhaseMetric[];	cleanSkipCalls: number;	hostWriteVisible?: boolean;} interface FanoutLadderRow {	fanout: number;	samples: number;	totalMs: ReturnType<typeof stats>;	spawnBatchMs: ReturnType<typeof stats>;	waitReapBatchMs: ReturnType<typeof stats>;	perProcessWaitReapP50Ms: number;	sidecarExecutePhases: SidecarExecutePhaseMetric[];} interface NestedChildProcessRow {	row: "guest_node_child_process_8";	fanout: number;	samples: number;	warmup: number;	command: string;	nodeParentMs: ReturnType<typeof stats>;	guestParentMs: ReturnType<typeof stats>;	guestVsNodeP50: number;	guestMinusNodeP50Ms: number;	parentObserved: NestedChildProcessObservationSummary;	expectedLifecycleProcessesPerIteration: number;	expectedSidecarProcessExecutions: number;	expectedPublicParentExecutions: number;	expectedNestedChildProcessExecutions: number;	sidecarExecutePhases: SidecarExecutePhaseMetric[];} interface NestedChildProcessObservation {	expectedChildren: number;	exitCallbacks: number;	closeCallbacks: number;	errorCallbacks: number;	nonZeroExitCodes: number;	nonZeroCloseCodes: number;} interface NestedChildProcessObservationSummary extends NestedChildProcessObservation {	samples: number;} interface NestedChildProcessLayerResult {	ms: ReturnType<typeof stats>;	observed: NestedChildProcessObservationSummary;} function nowMs(start: bigint): number {	return Number(process.hrtime.bigint() - start) / 1e6;} function waitNodeChild(child: ChildProcess): Promise<void> {	return new Promise((resolve, reject) => {		child.on("error", reject);		child.on("exit", (code) =>			code === 0 ? resolve() : reject(new Error(`node exited ${code}`)),		);	});} /** Node reference layer: spawn a real host node that exits, timed in-process. */function runNodeLayer(): number[] {	const samples: number[] = [];	for (let i = 0; i < WARMUP + ITERATIONS; i++) {		const t = process.hrtime.bigint();		const r = spawnSync("node", NODE_ARGS, { stdio: "ignore" });		const ms = nowMs(t);		if (r.status !== 0) {			throw new Error(`node layer: node exited ${r.status}`);		}		if (i >= WARMUP) samples.push(ms);	}	return samples;} /** Guest layer: spawn node inside one reused VM (a fresh V8 isolate per spawn). */async function runGuestLayer(sidecar: SidecarProcess): Promise<number[]> {	const vm = await createBenchVm({ sidecar });	try {		for (let i = 0; i < WARMUP; i++) {			const p = vm.spawn("node", NODE_ARGS);			const code = await vm.waitProcess(p.pid);			if (code !== 0) throw new Error(`guest warmup: exit ${code}`);		}		const samples: number[] = [];		for (let i = 0; i < ITERATIONS; i++) {			const t = process.hrtime.bigint();			const p = vm.spawn("node", NODE_ARGS);			await vm.waitProcess(p.pid);			samples.push(nowMs(t));		}		return samples;	} finally {		await vm.dispose();	}} async function runNodeExitPhases(): Promise<PhaseSamples> {	const samples: PhaseSamples = { total: [], spawn: [], wait_reap: [] };	for (let i = 0; i < WARMUP + ITERATIONS; i++) {		const totalStart = process.hrtime.bigint();		const spawnStart = process.hrtime.bigint();		const child = spawn("node", NODE_ARGS, { stdio: "ignore" });		const spawnMs = nowMs(spawnStart);		const waitStart = process.hrtime.bigint();		await waitNodeChild(child);		const waitMs = nowMs(waitStart);		const totalMs = nowMs(totalStart);		if (i >= WARMUP) {			samples.total?.push(totalMs);			samples.spawn?.push(spawnMs);			samples.wait_reap?.push(waitMs);		}	}	return samples;} async function runNodeFanoutPhases(): Promise<PhaseSamples> {	const samples: PhaseSamples = {		total: [],		spawn_batch: [],		wait_reap_batch: [],	};	for (let i = 0; i < WARMUP + ITERATIONS; i++) {		const totalStart = process.hrtime.bigint();		const spawnStart = process.hrtime.bigint();		const children = Array.from({ length: FANOUT }, () =>			spawn("node", NODE_ARGS, { stdio: "ignore" }),		);		const spawnMs = nowMs(spawnStart);		const waitStart = process.hrtime.bigint();		await Promise.all(children.map(waitNodeChild));		const waitMs = nowMs(waitStart);		const totalMs = nowMs(totalStart);		if (i >= WARMUP) {			samples.total?.push(totalMs);			samples.spawn_batch?.push(spawnMs);			samples.wait_reap_batch?.push(waitMs);		}	}	return samples;} async function runGuestNodeExitPhases(sidecar: SidecarProcess): Promise<PhaseSamples> {	const samples: PhaseSamples = { total: [], spawn: [], wait_reap: [] };	const vm = await createBenchVm({ sidecar });	try {		for (let i = 0; i < WARMUP + ITERATIONS; i++) {			const totalStart = process.hrtime.bigint();			const spawnStart = process.hrtime.bigint();			const child = vm.spawn("node", NODE_ARGS);			const spawnMs = nowMs(spawnStart);			const waitStart = process.hrtime.bigint();			const code = await vm.waitProcess(child.pid);			const waitMs = nowMs(waitStart);			if (code !== 0) throw new Error(`guest phase node_exit: exit ${code}`);			const totalMs = nowMs(totalStart);			if (i >= WARMUP) {				samples.total?.push(totalMs);				samples.spawn?.push(spawnMs);				samples.wait_reap?.push(waitMs);			}		}		return samples;	} finally {		await vm.dispose();	}} async function runGuestFanoutPhases(sidecar: SidecarProcess): Promise<PhaseSamples> {	const samples: PhaseSamples = {		total: [],		spawn_batch: [],		wait_reap_batch: [],	};	const vm = await createBenchVm({ sidecar });	try {		for (let i = 0; i < WARMUP + ITERATIONS; i++) {			const totalStart = process.hrtime.bigint();			const spawnStart = process.hrtime.bigint();			const children = Array.from({ length: FANOUT }, () =>				vm.spawn("node", NODE_ARGS),			);			const spawnMs = nowMs(spawnStart);			const waitStart = process.hrtime.bigint();			const codes = await Promise.all(				children.map((child) => vm.waitProcess(child.pid)),			);			const waitMs = nowMs(waitStart);			const badCode = codes.find((code) => code !== 0);			if (badCode !== undefined) {				throw new Error(`guest phase fanout: exit ${badCode}`);			}			const totalMs = nowMs(totalStart);			if (i >= WARMUP) {				samples.total?.push(totalMs);				samples.spawn_batch?.push(spawnMs);				samples.wait_reap_batch?.push(waitMs);			}		}		return samples;	} finally {		await vm.dispose();	}} async function runGuestFanoutLadderCase(	sidecar: SidecarProcess,	executePhasesFile: string,	fanout: number,): Promise<FanoutLadderRow> {	const beforePhases = readSidecarExecutePhases(executePhasesFile);	const vm = await createBenchVm({ sidecar });	try {		const totalSamples: number[] = [];		const spawnBatchSamples: number[] = [];		const waitReapBatchSamples: number[] = [];		for (let i = 0; i < WARMUP + ITERATIONS; i++) {			const totalStart = process.hrtime.bigint();			const spawnStart = process.hrtime.bigint();			const children = Array.from({ length: fanout }, () =>				vm.spawn("node", NODE_ARGS),			);			const spawnMs = nowMs(spawnStart);			const waitStart = process.hrtime.bigint();			const codes = await Promise.all(				children.map((child) => vm.waitProcess(child.pid)),			);			const waitMs = nowMs(waitStart);			const badCode = codes.find((code) => code !== 0);			if (badCode !== undefined) {				throw new Error(`guest fanout ladder ${fanout}: exit ${badCode}`);			}			const totalMs = nowMs(totalStart);			if (i >= WARMUP) {				totalSamples.push(totalMs);				spawnBatchSamples.push(spawnMs);				waitReapBatchSamples.push(waitMs);			}		}		const waitReapBatch = stats(waitReapBatchSamples);		return {			fanout,			samples: totalSamples.length,			totalMs: stats(totalSamples),			spawnBatchMs: stats(spawnBatchSamples),			waitReapBatchMs: waitReapBatch,			perProcessWaitReapP50Ms: round(waitReapBatch.p50 / fanout),			sidecarExecutePhases: diffSidecarExecutePhases(				beforePhases,				readSidecarExecutePhases(executePhasesFile),			),		};	} finally {		await vm.dispose();	}} async function runGuestFanoutLadderRows(	sidecar: SidecarProcess,	executePhasesFile: string,): Promise<FanoutLadderRow[]> {	const rows: FanoutLadderRow[] = [];	for (const fanout of FANOUT_LADDER) {		rows.push(await runGuestFanoutLadderCase(sidecar, executePhasesFile, fanout));	}	return rows;} function nestedChildProcessScript(fanout: number): string {	return `const { spawn } = require("node:child_process");const fanout = ${fanout};let exits = 0;let closes = 0;let errors = 0;let nonZeroExitCodes = 0;let nonZeroCloseCodes = 0;let failed = false;let done = false;const timeout = setTimeout(() => fail("nested child_process fanout timed out"), 5000);function fail(message) {  if (failed) return;  failed = true;  clearTimeout(timeout);  console.error(message);  process.exit(1);}function maybeDone() {  if (!done && closes === fanout) {    done = true;    clearTimeout(timeout);    console.log("__BENCH098__" + JSON.stringify({      expectedChildren: fanout,      exitCallbacks: exits,      closeCallbacks: closes,      errorCallbacks: errors,      nonZeroExitCodes,      nonZeroCloseCodes,    }));  }}for (let i = 0; i < fanout; i++) {  const child = spawn("node", ["-e", "process.exit(0)"], { stdio: "ignore" });  child.on("error", (error) => {    errors++;    fail(error && error.message ? error.message : String(error));  });  child.on("exit", (code, signal) => {    if (code !== 0) nonZeroExitCodes++;    exits++;    maybeDone();  });  child.on("close", (code, signal) => {    if (code !== 0) nonZeroCloseCodes++;    closes++;    maybeDone();  });}`;} function parseNestedChildProcessObservations(	stdout: string,	fanout: number,	context: string,): NestedChildProcessObservation[] {	const observations = stdout		.split(/\r?\n/)		.filter((line) => line.startsWith("__BENCH098__"))		.map((line) => JSON.parse(line.slice("__BENCH098__".length)) as NestedChildProcessObservation);	if (observations.length !== 1) {		throw new Error(`${context}: expected one BENCH-098 marker, got ${observations.length}`);	}	assertNestedChildProcessObservation(observations[0], fanout, context);	return observations;} function assertNestedChildProcessObservation(	observation: NestedChildProcessObservation,	fanout: number,	context: string,): void {	if (		observation.expectedChildren !== fanout ||		observation.closeCallbacks !== fanout ||		observation.errorCallbacks !== 0 ||		observation.nonZeroExitCodes !== 0 ||		observation.nonZeroCloseCodes !== 0 ||		observation.exitCallbacks < 0 ||		observation.exitCallbacks > fanout	) {		throw new Error(			`${context}: invalid nested child_process marker ${JSON.stringify(observation)}`,		);	}} function summarizeNestedChildProcessObservations(	observations: NestedChildProcessObservation[],): NestedChildProcessObservationSummary {	return {		samples: observations.length,		expectedChildren: observations[0]?.expectedChildren ?? 0,		exitCallbacks: observations.reduce(			(sum, observation) => sum + observation.exitCallbacks,			0,		),		closeCallbacks: observations.reduce(			(sum, observation) => sum + observation.closeCallbacks,			0,		),		errorCallbacks: observations.reduce((sum, observation) => sum + observation.errorCallbacks, 0),		nonZeroExitCodes: observations.reduce(			(sum, observation) => sum + observation.nonZeroExitCodes,			0,		),		nonZeroCloseCodes: observations.reduce(			(sum, observation) => sum + observation.nonZeroCloseCodes,			0,		),	};} function runNodeNestedChildProcessLayer(	script: string,	fanout: number,): NestedChildProcessLayerResult {	const samples: number[] = [];	const observations: NestedChildProcessObservation[] = [];	for (let i = 0; i < WARMUP + ITERATIONS; i++) {		const t = process.hrtime.bigint();		const r = spawnSync("node", ["-e", script], {			encoding: "utf8",			stdio: ["ignore", "pipe", "pipe"],		});		const ms = nowMs(t);		if (r.status !== 0) {			throw new Error(				`node nested child_process: node exited ${r.status}: ${r.stderr}`,			);		}		const marker = parseNestedChildProcessObservations(

Lines 1–500 of 1480. Load more