// ec2 provider: ephemeral guests on AWS EC2 for the cross-OS supervised-daemon // e2e where tart can't run (Windows; optionally Linux). Mirrors tart.ts — launch // a fresh instance, drive over SSH (key-based; PowerShell on Windows), REBOOT for // real, tear down. // // Credentials are NEVER embedded here: the `aws` CLI uses the ambient sign-in // (`aws configure` / env). Every instance is tagged `executor-e2e` and always // terminated on discard; the security group is scoped to this host's egress IP. // // Reboot is gated on a real boot-time change (Windows `LastBootUpTime`), not mere // SSH reachability — an orderly shutdown keeps the daemon serving for several // seconds, so "SSH answered" alone can false-pass a reboot that never happened. import { execFile } from "node:child_process"; import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; import net from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; import { type SshResult, sleep, type VmArch, type VmHandle, type VmOs, type VmProvider, } from "./types"; const execFileP = promisify(execFile); const REGION = process.env.E2E_EC2_REGION ?? "us-west-2"; const INSTANCE_TYPE = process.env.E2E_EC2_INSTANCE_TYPE ?? "t3.medium"; const TAG = "executor-e2e"; const SSH_OPTS = [ "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=10", "-o", "LogLevel=ERROR", ]; const guestUser = (os: VmOs): string => os === "windows" ? "Administrator" : (process.env.E2E_EC2_LINUX_USER ?? "ubuntu"); /** * Reboot an EC2 guest by address, statelessly (no live handle) — the mirror of * tart's sshRebootGuest, for the worker-side `restart()`. The connection drops * mid-call, so errors are swallowed; the caller's down-gate + up-poll confirm * the real reboot. */ export const ec2RebootGuest = async ( host: string, keyPath: string, os: VmOs = "windows", ): Promise => { const cmd = os === "windows" ? "Restart-Computer -Force" : "sudo reboot"; await execFileP("ssh", ["-i", keyPath, ...SSH_OPTS, `${guestUser(os)}@${host}`, cmd]).catch( () => undefined, ); }; const aws = async (args: ReadonlyArray): Promise => { const { stdout } = await execFileP("aws", ["--region", REGION, "--output", "text", ...args], { maxBuffer: 64 * 1024 * 1024, }); return stdout.trim(); }; /** This host's public egress IP, for the inbound-SSH security-group rule. */ const egressIp = async (): Promise => { const { stdout } = await execFileP("curl", [ "-s", "--max-time", "10", "https://checkip.amazonaws.com", ]); return stdout.trim(); }; /** Latest AWS-published base AMI for the guest OS (resolve dynamically — ids rotate). */ const latestAmi = async (os: VmOs): Promise => { if (os === "windows") { const viaSsm = await aws([ "ssm", "get-parameters", "--names", "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base", "--query", "Parameters[0].Value", ]).catch(() => ""); if (viaSsm && viaSsm !== "None") return viaSsm; return aws([ "ec2", "describe-images", "--owners", "amazon", "--filters", "Name=name,Values=Windows_Server-2022-English-Full-Base-*", "Name=state,Values=available", "--query", "reverse(sort_by(Images,&CreationDate))[0].ImageId", ]); } return aws([ "ssm", "get-parameters", "--names", "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id", "--query", "Parameters[0].Value", ]); }; const defaultSubnet = async (): Promise => { const vpc = await aws([ "ec2", "describe-vpcs", "--filters", "Name=isDefault,Values=true", "--query", "Vpcs[0].VpcId", ]); const subnet = await aws([ "ec2", "describe-subnets", "--filters", `Name=vpc-id,Values=${vpc}`, "Name=default-for-az,Values=true", "--query", "Subnets[0].SubnetId", ]); return subnet && subnet !== "None" ? subnet : aws([ "ec2", "describe-subnets", "--filters", `Name=vpc-id,Values=${vpc}`, "--query", "Subnets[0].SubnetId", ]); }; /** Create (idempotently) a security group allowing inbound SSH from this host. */ const ensureSecurityGroup = async (myIp: string): Promise => { const name = `${TAG}-sg`; let sg = await aws([ "ec2", "describe-security-groups", "--filters", `Name=group-name,Values=${name}`, "--query", "SecurityGroups[0].GroupId", ]).catch(() => ""); if (!sg || sg === "None") { sg = await aws([ "ec2", "create-security-group", "--group-name", name, "--description", "executor e2e ephemeral guests (SSH from CI host)", "--query", "GroupId", ]); } // Authorize this host's IP for SSH; ignore "already exists". await aws([ "ec2", "authorize-security-group-ingress", "--group-id", sg, "--protocol", "tcp", "--port", "22", "--cidr", `${myIp}/32`, ]).catch(() => undefined); return sg; }; /** PowerShell user-data: enable OpenSSH, default the shell to PowerShell, and * authorize our public key for the Administrator account. */ const windowsUserData = (publicKey: string): string => [ "", "Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0", "Set-Service -Name sshd -StartupType Automatic", "Start-Service sshd", "New-ItemProperty -Path 'HKLM:\\SOFTWARE\\OpenSSH' -Name DefaultShell -Value 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' -PropertyType String -Force", "New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -ErrorAction SilentlyContinue", "$akf = 'C:\\ProgramData\\ssh\\administrators_authorized_keys'", `Set-Content -Path $akf -Value '${publicKey}'`, "icacls $akf /inheritance:r /grant 'Administrators:F' /grant 'SYSTEM:F'", "", ].join("\n"); const linuxUserData = (publicKey: string): string => ["#cloud-config", "ssh_authorized_keys:", ` - ${publicKey}`].join("\n"); const freePort = (): Promise => new Promise((resolve, reject) => { const srv = net.createServer(); srv.on("error", reject); srv.listen(0, "127.0.0.1", () => { const port = (srv.address() as net.AddressInfo).port; srv.close(() => resolve(port)); }); }); const waitLocalPort = async (port: number, attempts = 40): Promise => { for (let i = 0; i < attempts; i++) { const ok = await new Promise((resolve) => { const sock = net.connect({ host: "127.0.0.1", port }, () => { sock.destroy(); resolve(true); }); sock.on("error", () => resolve(false)); sock.setTimeout(1000, () => { sock.destroy(); resolve(false); }); }); if (ok) return; await sleep(500); } throw new Error(`tunnel local port ${port} never came up`); }; export const ec2Vm = (os: VmOs, arch: VmArch = "x64"): VmProvider => ({ os, provision: async () => { const user = guestUser(os); // A throwaway SSH keypair, authorized via user-data (no EC2 key pair needed — // we drive over OpenSSH key auth, not the Windows password). const keyDir = mkdtempSync(join(tmpdir(), "executor-ec2-")); const keyPath = join(keyDir, "id"); await execFileP("ssh-keygen", ["-t", "ed25519", "-N", "", "-q", "-f", keyPath]); chmodSync(keyPath, 0o600); const publicKey = (await execFileP("ssh-keygen", ["-y", "-f", keyPath])).stdout.trim(); const [myIp, ami, subnet] = await Promise.all([egressIp(), latestAmi(os), defaultSubnet()]); const sg = await ensureSecurityGroup(myIp); const userData = os === "windows" ? windowsUserData(publicKey) : linuxUserData(publicKey); const userDataFile = join(keyDir, "user-data.txt"); writeFileSync(userDataFile, userData); const instanceId = await aws([ "ec2", "run-instances", "--image-id", ami, "--instance-type", INSTANCE_TYPE, "--count", "1", "--security-group-ids", sg, "--subnet-id", subnet, "--associate-public-ip-address", "--instance-initiated-shutdown-behavior", "terminate", "--user-data", `file://${userDataFile}`, "--tag-specifications", `ResourceType=instance,Tags=[{Key=Name,Value=${TAG}-${os}},{Key=purpose,Value=e2e}]`, "--query", "Instances[0].InstanceId", ]); let ip = ""; const tunnelClosers: Array<() => void> = []; const ssh = async (command: string): Promise => { try { const { stdout, stderr } = await execFileP( "ssh", ["-i", keyPath, ...SSH_OPTS, `${user}@${ip}`, command], { maxBuffer: 64 * 1024 * 1024 }, ); return { stdout, stderr, code: 0 }; } catch (err) { const e = err as { stdout?: string; stderr?: string; code?: number }; return { stdout: e.stdout ?? "", stderr: e.stderr ?? "", code: typeof e.code === "number" ? e.code : 1, }; } }; const waitSshUp = async (attempts: number): Promise => { for (let i = 0; i < attempts; i++) { if ((await ssh(os === "windows" ? "echo ok" : "true")).code === 0) return true; await sleep(5000); } return false; }; const waitSshDown = async (attempts = 40): Promise => { for (let i = 0; i < attempts; i++) { if ((await ssh("echo up").catch(() => ({ code: 1 }) as SshResult)).code !== 0) return; await sleep(3000); } // never observed down — caller's boot-time check is the backstop. }; const bootTime = async (): Promise => os === "windows" ? ( await ssh("(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToString('o')") ).stdout.trim() : (await ssh("cat /proc/sys/kernel/random/boot_id")).stdout.trim(); const handle: VmHandle = { os, arch, sshKeyPath: keyPath, get host() { return ip; }, ssh, push: async (localPath, remotePath) => { await execFileP("scp", [ "-i", keyPath, "-r", ...SSH_OPTS, localPath, `${user}@${ip}:${remotePath}`, ]); }, reboot: async () => { const before = await bootTime(); await ssh(os === "windows" ? "Restart-Computer -Force" : "sudo reboot").catch( () => undefined, ); await waitSshDown(); if (!(await waitSshUp(60))) throw new Error(`ec2 ${os}: SSH did not return after reboot`); const after = await bootTime(); if (before && after && before === after) { throw new Error( `ec2 ${os}: boot time unchanged after reboot — the guest never actually rebooted`, ); } }, tunnel: async (guestPort) => { const localPort = await freePort(); let closed = false; let child: ReturnType | undefined; const { spawn } = await import("node:child_process"); const spawnOnce = (): void => { child = spawn( "ssh", [ "-i", keyPath, ...SSH_OPTS, "-N", "-L", `${localPort}:127.0.0.1:${guestPort}`, `${user}@${ip}`, ], { stdio: "ignore" }, ); child.on("exit", () => { if (!closed) setTimeout(spawnOnce, 2000); }); }; spawnOnce(); const close = (): void => { closed = true; child?.kill(); }; tunnelClosers.push(close); await waitLocalPort(localPort); return { localPort, close }; }, discard: async () => { for (const close of tunnelClosers) close(); await aws(["ec2", "terminate-instances", "--instance-ids", instanceId]).catch( () => undefined, ); }, }; // Wait for a public IP, then for OpenSSH (Windows boot + FoD install ≈ 2-4 min). for (let i = 0; i < 60; i++) { const got = await aws([ "ec2", "describe-instances", "--instance-ids", instanceId, "--query", "Reservations[0].Instances[0].PublicIpAddress", ]).catch(() => ""); if (got && got !== "None") { ip = got; break; } await sleep(5000); } if (!ip) { await handle.discard(); throw new Error(`ec2 ${os}: no public IP within 300s`); } if (!(await waitSshUp(60))) { await handle.discard(); throw new Error(`ec2 ${os}: SSH never came up`); } return handle; }, });