- cli target authenticates with the daemon's auth.json bearer (was Basic from service.key); globalsetup reads the token from the guest and publishes it. - restart() gates on the daemon going DOWN before polling it back up, so an orderly shutdown / reconnecting tunnel can't false-pass a reboot that never happened (waitForHttpDown). - EC2 VmProvider (e2e/src/vm/ec2.ts) + cli-windows globalsetup unblock Windows: launch, key-based SSH/PowerShell drive, real boot-time-gated reboot, terminate.
Unified · Split
| @@ -100,3 +100,27 @@ export const waitForHttp = async ( · 99 unmodified lines | |||
| 100 | } | 100 | } |
| 101 | throw new Error(`timed out waiting for ${url}: ${String(lastError)}`); | 101 | throw new Error(`timed out waiting for ${url}: ${String(lastError)}`); |
| 102 | }; | 102 | }; |
| 103 | |||
| 104 | /** | ||
| 105 | * Reboot DOWN-GATE: wait until `url` stops answering (fetch rejects — connection | ||
| 106 | * refused/reset). An orderly OS shutdown keeps the daemon serving for several | ||
| 107 | * seconds, and a reconnecting tunnel re-establishes the forward, so polling for | ||
| 108 | * "up" right after a reboot command can false-pass a reboot that never happened. | ||
| 109 | * Gating on the server actually going DOWN first makes restart-persistence prove | ||
| 110 | * a real reboot. Throws if it never goes down within the deadline. | ||
| 111 | */ | ||
| 112 | export const waitForHttpDown = async ( | ||
| 113 | url: string, | ||
| 114 | options: { readonly timeoutMs?: number } = {}, | ||
| 115 | ): Promise<void> => { | ||
| 116 | const deadline = Date.now() + (options.timeoutMs ?? 120_000); | ||
| 117 | while (Date.now() < deadline) { | ||
| 118 | try { | ||
| 119 | await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(2000) }); | ||
| 120 | } catch { | ||
| 121 | return; // the server (or its tunnel) is gone — the reboot took | ||
| 122 | } | ||
| 123 | await new Promise((resolve) => setTimeout(resolve, 500)); | ||
| 124 | } | ||
| 125 | throw new Error(`${url} never became unreachable — the reboot may not have taken`); | ||
| 126 | }; | ||
| @@ -0,0 +1,3 @@ | |||
| 1 | import { setupCliTarget } from "./cli.globalsetup"; | ||
| 2 | |||
| 3 | export default (): Promise<(() => Promise<void>) | void> => setupCliTarget("windows"); | ||
| @@ -3,42 +3,73 @@ · 2 unmodified lines | |||
| 3 | // the host over a reconnecting SSH tunnel. Publishes connection + reboot info | 3 | // the host over a reconnecting SSH tunnel. Publishes connection + reboot info |
| 4 | // via env (inherited by the test workers spawned afterward). Per-OS entrypoints | 4 | // via env (inherited by the test workers spawned afterward). Per-OS entrypoints |
| 5 | // (cli-macos.globalsetup.ts, …) call setupCliTarget with their OS. | 5 | // (cli-macos.globalsetup.ts, …) call setupCliTarget with their OS. |
| 6 | // | ||
| 7 | // macOS + Linux run on local tart guests (Apple-Silicon host); Windows runs on | ||
| 8 | // an ephemeral EC2 instance. The supervised daemon is bearer-gated: it mints/ | ||
| 9 | // loads its token into auth.json, which we read from the guest and publish so | ||
| 10 | // the api surface authenticates with `Authorization: Bearer`. | ||
| 6 | import { buildGuestBinary } from "../src/vm/build-binary"; | 11 | import { buildGuestBinary } from "../src/vm/build-binary"; |
| 12 | import { ec2Vm } from "../src/vm/ec2"; | ||
| 7 | import { tartVm } from "../src/vm/tart"; | 13 | import { tartVm } from "../src/vm/tart"; |
| 8 | import type { VmArch, VmOs } from "../src/vm/types"; | 14 | import type { VmArch, VmHandle, VmOs } from "../src/vm/types"; |
| 9 | import { waitForHttp } from "./boot"; | 15 | import { waitForHttp } from "./boot"; |
| 10 | 16 | ||
| 11 | const PORT = 4789; | 17 | const PORT = 4789; |
| 12 | const GUEST_DIR = "~/ed"; | ||
| 13 | 18 | ||
| 14 | export async function setupCliTarget(os: VmOs): Promise<(() => Promise<void>) | void> { | 19 | // Per-OS guest specifics: working dir, binary name, and the shell idioms for |
| 15 | process.env.E2E_VM_OS = os; // so the worker-side target resolves the same OS | 20 | // prep/cleanup — Unix `sh` on tart (macOS/Linux), PowerShell on EC2 Windows. |
| 21 | const guestPlan = (os: VmOs) => { | ||
| 16 | if (os === "windows") { | 22 | if (os === "windows") { |
| 17 | throw new Error("cli-windows is pending the ec2 provider; run cli-macos / cli-linux for now"); | 23 | const dir = "C:/ed"; |
| 24 | const exe = `${dir}/executor.exe`; | ||
| 25 | return { | ||
| 26 | dir, | ||
| 27 | prep: `Remove-Item -Recurse -Force '${dir}' -ErrorAction SilentlyContinue; New-Item -ItemType Directory -Force -Path '${dir}' | Out-Null`, | ||
| 28 | postPush: "Write-Output ok", // no chmod/quarantine on Windows | ||
| 29 | install: `& '${exe}' service install --port ${PORT}`, | ||
| 30 | readToken: 'Get-Content "$env:USERPROFILE\\.executor\\server-control\\auth.json" -Raw', | ||
| 31 | uninstall: `& '${exe}' service uninstall`, | ||
| 32 | }; | ||
| 18 | } | 33 | } |
| 34 | const dir = "~/ed"; | ||
| 35 | const exe = `${dir}/executor`; | ||
| 36 | return { | ||
| 37 | dir, | ||
| 38 | prep: `rm -rf ${dir} && mkdir -p ${dir}`, | ||
| 39 | // macOS quarantines scp'd executables; clear it so the binary can run. | ||
| 40 | postPush: | ||
| 41 | os === "macos" | ||
| 42 | ? `chmod +x ${exe}; xattr -dr com.apple.quarantine ${dir} 2>/dev/null || true` | ||
| 43 | : `chmod +x ${exe}`, | ||
| 44 | install: `${exe} service install --port ${PORT}`, | ||
| 45 | readToken: "cat ~/.executor/server-control/auth.json", | ||
| 46 | uninstall: `${exe} service uninstall`, | ||
| 47 | }; | ||
| 48 | }; | ||
| 19 | 49 | ||
| 20 | const arch: VmArch = "arm64"; // tart guests on an Apple-Silicon host | 50 | export async function setupCliTarget(os: VmOs): Promise<(() => Promise<void>) | void> { |
| 51 | process.env.E2E_VM_OS = os; // so the worker-side target resolves the same OS | ||
| 52 | |||
| 53 | const arch: VmArch = os === "windows" ? "x64" : "arm64"; | ||
| 21 | const binDir = await buildGuestBinary(os, arch); | 54 | const binDir = await buildGuestBinary(os, arch); |
| 22 | const vm = await tartVm(os, arch).provision(); | 55 | const vm: VmHandle = |
| 56 | os === "windows" ? await ec2Vm(os, arch).provision() : await tartVm(os, arch).provision(); | ||
| 57 | const plan = guestPlan(os); | ||
| 23 | 58 | ||
| 24 | let tunnelClose: (() => void) | undefined; | 59 | let tunnelClose: (() => void) | undefined; |
| 25 | try { | 60 | try { |
| 26 | await vm.ssh(`rm -rf ${GUEST_DIR} && mkdir -p ${GUEST_DIR}`); | 61 | await vm.ssh(plan.prep); |
| 27 | await vm.push(`${binDir}/.`, `${GUEST_DIR}/`); | 62 | await vm.push(`${binDir}/.`, os === "windows" ? plan.dir : `${plan.dir}/`); |
| 28 | // macOS quarantines scp'd executables; clear it so the binary can run. | 63 | await vm.ssh(plan.postPush); |
| 29 | await vm.ssh( | ||
| 30 | os === "macos" | ||
| 31 | ? `chmod +x ${GUEST_DIR}/executor; xattr -dr com.apple.quarantine ${GUEST_DIR} 2>/dev/null || true` | ||
| 32 | : `chmod +x ${GUEST_DIR}/executor`, | ||
| 33 | ); | ||
| 34 | 64 | ||
| 35 | const install = await vm.ssh(`${GUEST_DIR}/executor service install --port ${PORT}`); | 65 | const install = await vm.ssh(plan.install); |
| 36 | if (install.code !== 0) { | 66 | if (install.code !== 0) { |
| 37 | throw new Error(`service install failed: ${install.stderr.trim() || install.stdout.trim()}`); | 67 | throw new Error(`service install failed: ${install.stderr.trim() || install.stdout.trim()}`); |
| 38 | } | 68 | } |
| 39 | 69 | ||
| 40 | const keyRaw = (await vm.ssh("cat ~/.executor/server-control/service.key")).stdout.trim(); | 70 | // The supervised daemon mints/loads its bearer into auth.json on first boot. |
| 41 | const password = (JSON.parse(keyRaw) as { password: string }).password; | 71 | const tokenRaw = (await vm.ssh(plan.readToken)).stdout.trim(); |
| 72 | const token = (JSON.parse(tokenRaw) as { token: string }).token; | ||
| 42 | 73 | ||
| 43 | const tunnel = await vm.tunnel(PORT); | 74 | const tunnel = await vm.tunnel(PORT); |
| 44 | tunnelClose = tunnel.close; | 75 | tunnelClose = tunnel.close; |
| @@ -46,20 +77,21 @@ export async function setupCliTarget(os: VmOs): Promise<(() => Promise<void>) | · 1 unmodified line | |||
| 46 | await waitForHttp(`${baseUrl}/`, { timeoutMs: 60_000 }); | 77 | await waitForHttp(`${baseUrl}/`, { timeoutMs: 60_000 }); |
| 47 | 78 | ||
| 48 | process.env.E2E_CLI_BASE_URL = baseUrl; | 79 | process.env.E2E_CLI_BASE_URL = baseUrl; |
| 49 | process.env.E2E_CLI_AUTH_PASSWORD = password; | 80 | process.env.E2E_CLI_AUTH_TOKEN = token; |
| 50 | process.env.E2E_CLI_VM_HOST = vm.host; | 81 | process.env.E2E_CLI_VM_HOST = vm.host; |
| 82 | if (vm.sshKeyPath) process.env.E2E_CLI_SSH_KEY = vm.sshKeyPath; | ||
| 51 | process.env.E2E_CLI_TUNNEL_PORT = String(tunnel.localPort); | 83 | process.env.E2E_CLI_TUNNEL_PORT = String(tunnel.localPort); |
| 52 | process.env.E2E_CLI_BIN_DIR = GUEST_DIR; | 84 | process.env.E2E_CLI_BIN_DIR = plan.dir; |
| 53 | } catch (error) { | 85 | } catch (error) { |
| 54 | tunnelClose?.(); | 86 | tunnelClose?.(); |
| 55 | await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined); | 87 | await vm.ssh(plan.uninstall).catch(() => undefined); |
| 56 | await vm.discard(); | 88 | await vm.discard(); |
| 57 | throw error; | 89 | throw error; |
| 58 | } | 90 | } |
| 59 | 91 | ||
| 60 | return async () => { | 92 | return async () => { |
| 61 | tunnelClose?.(); | 93 | tunnelClose?.(); |
| 62 | await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined); | 94 | await vm.ssh(plan.uninstall).catch(() => undefined); |
| 63 | await vm.discard(); | 95 | await vm.discard(); |
| 64 | }; | 96 | }; |
| 65 | } | 97 | } |
| @@ -0,0 +1,422 @@ | |||
| 1 | // ec2 provider: ephemeral guests on AWS EC2 for the cross-OS supervised-daemon | ||
| 2 | // e2e where tart can't run (Windows; optionally Linux). Mirrors tart.ts — launch | ||
| 3 | // a fresh instance, drive over SSH (key-based; PowerShell on Windows), REBOOT for | ||
| 4 | // real, tear down. | ||
| 5 | // | ||
| 6 | // Credentials are NEVER embedded here: the `aws` CLI uses the ambient sign-in | ||
| 7 | // (`aws configure` / env). Every instance is tagged `executor-e2e` and always | ||
| 8 | // terminated on discard; the security group is scoped to this host's egress IP. | ||
| 9 | // | ||
| 10 | // Reboot is gated on a real boot-time change (Windows `LastBootUpTime`), not mere | ||
| 11 | // SSH reachability — an orderly shutdown keeps the daemon serving for several | ||
| 12 | // seconds, so "SSH answered" alone can false-pass a reboot that never happened. | ||
| 13 | |||
| 14 | import { execFile } from "node:child_process"; | ||
| 15 | import { chmodSync, mkdtempSync, writeFileSync } from "node:fs"; | ||
| 16 | import net from "node:net"; | ||
| 17 | import { tmpdir } from "node:os"; | ||
| 18 | import { join } from "node:path"; | ||
| 19 | import { promisify } from "node:util"; | ||
| 20 | |||
| 21 | import { | ||
| 22 | type SshResult, | ||
| 23 | sleep, | ||
| 24 | type VmArch, | ||
| 25 | type VmHandle, | ||
| 26 | type VmOs, | ||
| 27 | type VmProvider, | ||
| 28 | } from "./types"; | ||
| 29 | |||
| 30 | const execFileP = promisify(execFile); | ||
| 31 | |||
| 32 | const REGION = process.env.E2E_EC2_REGION ?? "us-west-2"; | ||
| 33 | const INSTANCE_TYPE = process.env.E2E_EC2_INSTANCE_TYPE ?? "t3.medium"; | ||
| 34 | const TAG = "executor-e2e"; | ||
| 35 | |||
| 36 | const SSH_OPTS = [ | ||
| 37 | "-o", | ||
| 38 | "StrictHostKeyChecking=no", | ||
| 39 | "-o", | ||
| 40 | "UserKnownHostsFile=/dev/null", | ||
| 41 | "-o", | ||
| 42 | "ConnectTimeout=10", | ||
| 43 | "-o", | ||
| 44 | "ServerAliveInterval=10", | ||
| 45 | "-o", | ||
| 46 | "LogLevel=ERROR", | ||
| 47 | ]; | ||
| 48 | |||
| 49 | const guestUser = (os: VmOs): string => | ||
| 50 | os === "windows" ? "Administrator" : (process.env.E2E_EC2_LINUX_USER ?? "ubuntu"); | ||
| 51 | |||
| 52 | /** | ||
| 53 | * Reboot an EC2 guest by address, statelessly (no live handle) — the mirror of | ||
| 54 | * tart's sshRebootGuest, for the worker-side `restart()`. The connection drops | ||
| 55 | * mid-call, so errors are swallowed; the caller's down-gate + up-poll confirm | ||
| 56 | * the real reboot. | ||
| 57 | */ | ||
| 58 | export const ec2RebootGuest = async ( | ||
| 59 | host: string, | ||
| 60 | keyPath: string, | ||
| 61 | os: VmOs = "windows", | ||
| 62 | ): Promise<void> => { | ||
| 63 | const cmd = os === "windows" ? "Restart-Computer -Force" : "sudo reboot"; | ||
| 64 | await execFileP("ssh", ["-i", keyPath, ...SSH_OPTS, `${guestUser(os)}@${host}`, cmd]).catch( | ||
| 65 | () => undefined, | ||
| 66 | ); | ||
| 67 | }; | ||
| 68 | |||
| 69 | const aws = async (args: ReadonlyArray<string>): Promise<string> => { | ||
| 70 | const { stdout } = await execFileP("aws", ["--region", REGION, "--output", "text", ...args], { | ||
| 71 | maxBuffer: 64 * 1024 * 1024, | ||
| 72 | }); | ||
| 73 | return stdout.trim(); | ||
| 74 | }; | ||
| 75 | |||
| 76 | /** This host's public egress IP, for the inbound-SSH security-group rule. */ | ||
| 77 | const egressIp = async (): Promise<string> => { | ||
| 78 | const { stdout } = await execFileP("curl", [ | ||
| 79 | "-s", | ||
| 80 | "--max-time", | ||
| 81 | "10", | ||
| 82 | "https://checkip.amazonaws.com", | ||
| 83 | ]); | ||
| 84 | return stdout.trim(); | ||
| 85 | }; | ||
| 86 | |||
| 87 | /** Latest AWS-published base AMI for the guest OS (resolve dynamically — ids rotate). */ | ||
| 88 | const latestAmi = async (os: VmOs): Promise<string> => { | ||
| 89 | if (os === "windows") { | ||
| 90 | const viaSsm = await aws([ | ||
| 91 | "ssm", | ||
| 92 | "get-parameters", | ||
| 93 | "--names", | ||
| 94 | "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base", | ||
| 95 | "--query", | ||
| 96 | "Parameters[0].Value", | ||
| 97 | ]).catch(() => ""); | ||
| 98 | if (viaSsm && viaSsm !== "None") return viaSsm; | ||
| 99 | return aws([ | ||
| 100 | "ec2", | ||
| 101 | "describe-images", | ||
| 102 | "--owners", | ||
| 103 | "amazon", | ||
| 104 | "--filters", | ||
| 105 | "Name=name,Values=Windows_Server-2022-English-Full-Base-*", | ||
| 106 | "Name=state,Values=available", | ||
| 107 | "--query", | ||
| 108 | "reverse(sort_by(Images,&CreationDate))[0].ImageId", | ||
| 109 | ]); | ||
| 110 | } | ||
| 111 | return aws([ | ||
| 112 | "ssm", | ||
| 113 | "get-parameters", | ||
| 114 | "--names", | ||
| 115 | "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id", | ||
| 116 | "--query", | ||
| 117 | "Parameters[0].Value", | ||
| 118 | ]); | ||
| 119 | }; | ||
| 120 | |||
| 121 | const defaultSubnet = async (): Promise<string> => { | ||
| 122 | const vpc = await aws([ | ||
| 123 | "ec2", | ||
| 124 | "describe-vpcs", | ||
| 125 | "--filters", | ||
| 126 | "Name=isDefault,Values=true", | ||
| 127 | "--query", | ||
| 128 | "Vpcs[0].VpcId", | ||
| 129 | ]); | ||
| 130 | const subnet = await aws([ | ||
| 131 | "ec2", | ||
| 132 | "describe-subnets", | ||
| 133 | "--filters", | ||
| 134 | `Name=vpc-id,Values=${vpc}`, | ||
| 135 | "Name=default-for-az,Values=true", | ||
| 136 | "--query", | ||
| 137 | "Subnets[0].SubnetId", | ||
| 138 | ]); | ||
| 139 | return subnet && subnet !== "None" | ||
| 140 | ? subnet | ||
| 141 | : aws([ | ||
| 142 | "ec2", | ||
| 143 | "describe-subnets", | ||
| 144 | "--filters", | ||
| 145 | `Name=vpc-id,Values=${vpc}`, | ||
| 146 | "--query", | ||
| 147 | "Subnets[0].SubnetId", | ||
| 148 | ]); | ||
| 149 | }; | ||
| 150 | |||
| 151 | /** Create (idempotently) a security group allowing inbound SSH from this host. */ | ||
| 152 | const ensureSecurityGroup = async (myIp: string): Promise<string> => { | ||
| 153 | const name = `${TAG}-sg`; | ||
| 154 | let sg = await aws([ | ||
| 155 | "ec2", | ||
| 156 | "describe-security-groups", | ||
| 157 | "--filters", | ||
| 158 | `Name=group-name,Values=${name}`, | ||
| 159 | "--query", | ||
| 160 | "SecurityGroups[0].GroupId", | ||
| 161 | ]).catch(() => ""); | ||
| 162 | if (!sg || sg === "None") { | ||
| 163 | sg = await aws([ | ||
| 164 | "ec2", | ||
| 165 | "create-security-group", | ||
| 166 | "--group-name", | ||
| 167 | name, | ||
| 168 | "--description", | ||
| 169 | "executor e2e ephemeral guests (SSH from CI host)", | ||
| 170 | "--query", | ||
| 171 | "GroupId", | ||
| 172 | ]); | ||
| 173 | } | ||
| 174 | // Authorize this host's IP for SSH; ignore "already exists". | ||
| 175 | await aws([ | ||
| 176 | "ec2", | ||
| 177 | "authorize-security-group-ingress", | ||
| 178 | "--group-id", | ||
| 179 | sg, | ||
| 180 | "--protocol", | ||
| 181 | "tcp", | ||
| 182 | "--port", | ||
| 183 | "22", | ||
| 184 | "--cidr", | ||
| 185 | `${myIp}/32`, | ||
| 186 | ]).catch(() => undefined); | ||
| 187 | return sg; | ||
| 188 | }; | ||
| 189 | |||
| 190 | /** PowerShell user-data: enable OpenSSH, default the shell to PowerShell, and | ||
| 191 | * authorize our public key for the Administrator account. */ | ||
| 192 | const windowsUserData = (publicKey: string): string => | ||
| 193 | [ | ||
| 194 | "<powershell>", | ||
| 195 | "Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0", | ||
| 196 | "Set-Service -Name sshd -StartupType Automatic", | ||
| 197 | "Start-Service sshd", | ||
| 198 | "New-ItemProperty -Path 'HKLM:\\SOFTWARE\\OpenSSH' -Name DefaultShell -Value 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' -PropertyType String -Force", | ||
| 199 | "New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -ErrorAction SilentlyContinue", | ||
| 200 | "$akf = 'C:\\ProgramData\\ssh\\administrators_authorized_keys'", | ||
| 201 | `Set-Content -Path $akf -Value '${publicKey}'`, | ||
| 202 | "icacls $akf /inheritance:r /grant 'Administrators:F' /grant 'SYSTEM:F'", | ||
| 203 | "</powershell>", | ||
| 204 | ].join("\n"); | ||
| 205 | |||
| 206 | const linuxUserData = (publicKey: string): string => | ||
| 207 | ["#cloud-config", "ssh_authorized_keys:", ` - ${publicKey}`].join("\n"); | ||
| 208 | |||
| 209 | const freePort = (): Promise<number> => | ||
| 210 | new Promise((resolve, reject) => { | ||
| 211 | const srv = net.createServer(); | ||
| 212 | srv.on("error", reject); | ||
| 213 | srv.listen(0, "127.0.0.1", () => { | ||
| 214 | const port = (srv.address() as net.AddressInfo).port; | ||
| 215 | srv.close(() => resolve(port)); | ||
| 216 | }); | ||
| 217 | }); | ||
| 218 | |||
| 219 | const waitLocalPort = async (port: number, attempts = 40): Promise<void> => { | ||
| 220 | for (let i = 0; i < attempts; i++) { | ||
| 221 | const ok = await new Promise<boolean>((resolve) => { | ||
| 222 | const sock = net.connect({ host: "127.0.0.1", port }, () => { | ||
| 223 | sock.destroy(); | ||
| 224 | resolve(true); | ||
| 225 | }); | ||
| 226 | sock.on("error", () => resolve(false)); | ||
| 227 | sock.setTimeout(1000, () => { | ||
| 228 | sock.destroy(); | ||
| 229 | resolve(false); | ||
| 230 | }); | ||
| 231 | }); | ||
| 232 | if (ok) return; | ||
| 233 | await sleep(500); | ||
| 234 | } | ||
| 235 | throw new Error(`tunnel local port ${port} never came up`); | ||
| 236 | }; | ||
| 237 | |||
| 238 | export const ec2Vm = (os: VmOs, arch: VmArch = "x64"): VmProvider => ({ | ||
| 239 | os, | ||
| 240 | provision: async () => { | ||
| 241 | const user = guestUser(os); | ||
| 242 | // A throwaway SSH keypair, authorized via user-data (no EC2 key pair needed — | ||
| 243 | // we drive over OpenSSH key auth, not the Windows password). | ||
| 244 | const keyDir = mkdtempSync(join(tmpdir(), "executor-ec2-")); | ||
| 245 | const keyPath = join(keyDir, "id"); | ||
| 246 | await execFileP("ssh-keygen", ["-t", "ed25519", "-N", "", "-q", "-f", keyPath]); | ||
| 247 | chmodSync(keyPath, 0o600); | ||
| 248 | const publicKey = (await execFileP("ssh-keygen", ["-y", "-f", keyPath])).stdout.trim(); | ||
| 249 | |||
| 250 | const [myIp, ami, subnet] = await Promise.all([egressIp(), latestAmi(os), defaultSubnet()]); | ||
| 251 | const sg = await ensureSecurityGroup(myIp); | ||
| 252 | const userData = os === "windows" ? windowsUserData(publicKey) : linuxUserData(publicKey); | ||
| 253 | const userDataFile = join(keyDir, "user-data.txt"); | ||
| 254 | writeFileSync(userDataFile, userData); | ||
| 255 | |||
| 256 | const instanceId = await aws([ | ||
| 257 | "ec2", | ||
| 258 | "run-instances", | ||
| 259 | "--image-id", | ||
| 260 | ami, | ||
| 261 | "--instance-type", | ||
| 262 | INSTANCE_TYPE, | ||
| 263 | "--count", | ||
| 264 | "1", | ||
| 265 | "--security-group-ids", | ||
| 266 | sg, | ||
| 267 | "--subnet-id", | ||
| 268 | subnet, | ||
| 269 | "--associate-public-ip-address", | ||
| 270 | "--instance-initiated-shutdown-behavior", | ||
| 271 | "terminate", | ||
| 272 | "--user-data", | ||
| 273 | `file://${userDataFile}`, | ||
| 274 | "--tag-specifications", | ||
| 275 | `ResourceType=instance,Tags=[{Key=Name,Value=${TAG}-${os}},{Key=purpose,Value=e2e}]`, | ||
| 276 | "--query", | ||
| 277 | "Instances[0].InstanceId", | ||
| 278 | ]); | ||
| 279 | |||
| 280 | let ip = ""; | ||
| 281 | const tunnelClosers: Array<() => void> = []; | ||
| 282 | |||
| 283 | const ssh = async (command: string): Promise<SshResult> => { | ||
| 284 | try { | ||
| 285 | const { stdout, stderr } = await execFileP( | ||
| 286 | "ssh", | ||
| 287 | ["-i", keyPath, ...SSH_OPTS, `${user}@${ip}`, command], | ||
| 288 | { maxBuffer: 64 * 1024 * 1024 }, | ||
| 289 | ); | ||
| 290 | return { stdout, stderr, code: 0 }; | ||
| 291 | } catch (err) { | ||
| 292 | const e = err as { stdout?: string; stderr?: string; code?: number }; | ||
| 293 | return { | ||
| 294 | stdout: e.stdout ?? "", | ||
| 295 | stderr: e.stderr ?? "", | ||
| 296 | code: typeof e.code === "number" ? e.code : 1, | ||
| 297 | }; | ||
| 298 | } | ||
| 299 | }; | ||
| 300 | |||
| 301 | const waitSshUp = async (attempts: number): Promise<boolean> => { | ||
| 302 | for (let i = 0; i < attempts; i++) { | ||
| 303 | if ((await ssh(os === "windows" ? "echo ok" : "true")).code === 0) return true; | ||
| 304 | await sleep(5000); | ||
| 305 | } | ||
| 306 | return false; | ||
| 307 | }; | ||
| 308 | |||
| 309 | const waitSshDown = async (attempts = 40): Promise<void> => { | ||
| 310 | for (let i = 0; i < attempts; i++) { | ||
| 311 | if ((await ssh("echo up").catch(() => ({ code: 1 }) as SshResult)).code !== 0) return; | ||
| 312 | await sleep(3000); | ||
| 313 | } | ||
| 314 | // never observed down — caller's boot-time check is the backstop. | ||
| 315 | }; | ||
| 316 | |||
| 317 | const bootTime = async (): Promise<string> => | ||
| 318 | os === "windows" | ||
| 319 | ? ( | ||
| 320 | await ssh("(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToString('o')") | ||
| 321 | ).stdout.trim() | ||
| 322 | : (await ssh("cat /proc/sys/kernel/random/boot_id")).stdout.trim(); | ||
| 323 | |||
| 324 | const handle: VmHandle = { | ||
| 325 | os, | ||
| 326 | arch, | ||
| 327 | sshKeyPath: keyPath, | ||
| 328 | get host() { | ||
| 329 | return ip; | ||
| 330 | }, | ||
| 331 | ssh, | ||
| 332 | push: async (localPath, remotePath) => { | ||
| 333 | await execFileP("scp", [ | ||
| 334 | "-i", | ||
| 335 | keyPath, | ||
| 336 | "-r", | ||
| 337 | ...SSH_OPTS, | ||
| 338 | localPath, | ||
| 339 | `${user}@${ip}:${remotePath}`, | ||
| 340 | ]); | ||
| 341 | }, | ||
| 342 | reboot: async () => { | ||
| 343 | const before = await bootTime(); | ||
| 344 | await ssh(os === "windows" ? "Restart-Computer -Force" : "sudo reboot").catch( | ||
| 345 | () => undefined, | ||
| 346 | ); | ||
| 347 | await waitSshDown(); | ||
| 348 | if (!(await waitSshUp(60))) throw new Error(`ec2 ${os}: SSH did not return after reboot`); | ||
| 349 | const after = await bootTime(); | ||
| 350 | if (before && after && before === after) { | ||
| 351 | throw new Error( | ||
| 352 | `ec2 ${os}: boot time unchanged after reboot — the guest never actually rebooted`, | ||
| 353 | ); | ||
| 354 | } | ||
| 355 | }, | ||
| 356 | tunnel: async (guestPort) => { | ||
| 357 | const localPort = await freePort(); | ||
| 358 | let closed = false; | ||
| 359 | let child: ReturnType<typeof import("node:child_process").spawn> | undefined; | ||
| 360 | const { spawn } = await import("node:child_process"); | ||
| 361 | const spawnOnce = (): void => { | ||
| 362 | child = spawn( | ||
| 363 | "ssh", | ||
| 364 | [ | ||
| 365 | "-i", | ||
| 366 | keyPath, | ||
| 367 | ...SSH_OPTS, | ||
| 368 | "-N", | ||
| 369 | "-L", | ||
| 370 | `${localPort}:127.0.0.1:${guestPort}`, | ||
| 371 | `${user}@${ip}`, | ||
| 372 | ], | ||
| 373 | { stdio: "ignore" }, | ||
| 374 | ); | ||
| 375 | child.on("exit", () => { | ||
| 376 | if (!closed) setTimeout(spawnOnce, 2000); | ||
| 377 | }); | ||
| 378 | }; | ||
| 379 | spawnOnce(); | ||
| 380 | const close = (): void => { | ||
| 381 | closed = true; | ||
| 382 | child?.kill(); | ||
| 383 | }; | ||
| 384 | tunnelClosers.push(close); | ||
| 385 | await waitLocalPort(localPort); | ||
| 386 | return { localPort, close }; | ||
| 387 | }, | ||
| 388 | discard: async () => { | ||
| 389 | for (const close of tunnelClosers) close(); | ||
| 390 | await aws(["ec2", "terminate-instances", "--instance-ids", instanceId]).catch( | ||
| 391 | () => undefined, | ||
| 392 | ); | ||
| 393 | }, | ||
| 394 | }; | ||
| 395 | |||
| 396 | // Wait for a public IP, then for OpenSSH (Windows boot + FoD install ≈ 2-4 min). | ||
| 397 | for (let i = 0; i < 60; i++) { | ||
| 398 | const got = await aws([ | ||
| 399 | "ec2", | ||
| 400 | "describe-instances", | ||
| 401 | "--instance-ids", | ||
| 402 | instanceId, | ||
| 403 | "--query", | ||
| 404 | "Reservations[0].Instances[0].PublicIpAddress", | ||
| 405 | ]).catch(() => ""); | ||
| 406 | if (got && got !== "None") { | ||
| 407 | ip = got; | ||
| 408 | break; | ||
| 409 | } | ||
| 410 | await sleep(5000); | ||
| 411 | } | ||
| 412 | if (!ip) { | ||
| 413 | await handle.discard(); | ||
| 414 | throw new Error(`ec2 ${os}: no public IP within 300s`); | ||
| 415 | } | ||
| 416 | if (!(await waitSshUp(60))) { | ||
| 417 | await handle.discard(); | ||
| 418 | throw new Error(`ec2 ${os}: SSH never came up`); | ||
| 419 | } | ||
| 420 | return handle; | ||
| 421 | }, | ||
| 422 | }); | ||
| @@ -24,6 +24,12 @@ export interface VmHandle { · 23 unmodified lines | |||
| 24 | readonly arch: VmArch; | 24 | readonly arch: VmArch; |
| 25 | /** Current reachable address of the guest (re-resolved across reboots). */ | 25 | /** Current reachable address of the guest (re-resolved across reboots). */ |
| 26 | readonly host: string; | 26 | readonly host: string; |
| 27 | /** | ||
| 28 | * Path to the SSH private key for key-based providers (EC2). Undefined for | ||
| 29 | * password-based providers (tart/sshpass). Published by globalsetup so the | ||
| 30 | * stateless worker-side `restart()` can reboot the guest. | ||
| 31 | */ | ||
| 32 | readonly sshKeyPath?: string; | ||
| 27 | /** Run a command in the guest over SSH (shell on Unix, PowerShell on Windows). */ | 33 | /** Run a command in the guest over SSH (shell on Unix, PowerShell on Windows). */ |
| 28 | ssh(command: string): Promise<SshResult>; | 34 | ssh(command: string): Promise<SshResult>; |
| 29 | /** Copy a local file or directory into the guest (recursive for directories). */ | 35 | /** Copy a local file or directory into the guest (recursive for directories). */ |
| @@ -8,7 +8,8 @@ · 7 unmodified lines | |||
| 8 | // this target only reads what that published via env. | 8 | // this target only reads what that published via env. |
| 9 | import { Effect } from "effect"; | 9 | import { Effect } from "effect"; |
| 10 | 10 | ||
| 11 | import { waitForHttp } from "../setup/boot"; | 11 | import { waitForHttp, waitForHttpDown } from "../setup/boot"; |
| 12 | import { ec2RebootGuest } from "../src/vm/ec2"; | ||
| 12 | import { sshRebootGuest } from "../src/vm/tart"; | 13 | import { sshRebootGuest } from "../src/vm/tart"; |
| 13 | import type { VmOs } from "../src/vm/types"; | 14 | import type { VmOs } from "../src/vm/types"; |
| 14 | import type { Capability, Identity, Target } from "../src/target"; | 15 | import type { Capability, Identity, Target } from "../src/target"; |
| @@ -22,25 +23,36 @@ const env = (key: string): string => { · 7 unmodified lines | |||
| 22 | export const cliTarget = (): Target => { | 23 | export const cliTarget = (): Target => { |
| 23 | const baseUrl = env("E2E_CLI_BASE_URL"); | 24 | const baseUrl = env("E2E_CLI_BASE_URL"); |
| 24 | const os = (process.env.E2E_VM_OS ?? "macos") as VmOs; | 25 | const os = (process.env.E2E_VM_OS ?? "macos") as VmOs; |
| 25 | const username = process.env.E2E_CLI_AUTH_USER ?? "executor"; | ||
| 26 | 26 | ||
| 27 | return { | 27 | return { |
| 28 | name: process.env.E2E_TARGET ?? "cli", | 28 | name: process.env.E2E_TARGET ?? "cli", |
| 29 | baseUrl, | 29 | baseUrl, |
| 30 | mcpUrl: `${baseUrl}/mcp`, | 30 | mcpUrl: `${baseUrl}/mcp`, |
| 31 | capabilities: new Set<Capability>(["api"]), | 31 | capabilities: new Set<Capability>(["api"]), |
| 32 | // The supervised daemon is bearer-gated (auth.json); globalsetup reads the | ||
| 33 | // token from the guest and publishes it. A wrong/absent token still gets a | ||
| 34 | // clean 401, which the api surface treats as "up". | ||
| 32 | newIdentity: () => | 35 | newIdentity: () => |
| 33 | Effect.sync((): Identity => { | 36 | Effect.sync( |
| 34 | const basic = Buffer.from(`${username}:${env("E2E_CLI_AUTH_PASSWORD")}`).toString("base64"); | 37 | (): Identity => ({ |
| 35 | return { label: "cli-daemon", headers: { Authorization: `Basic ${basic}` } }; | 38 | label: "cli-daemon", |
| 36 | }), | 39 | headers: { Authorization: `Bearer ${env("E2E_CLI_AUTH_TOKEN")}` }, |
| 37 | // A genuine machine reboot, not a service kick: reboot the guest OS and wait | 40 | }), |
| 38 | // for the supervised daemon to auto-start and serve again (401 = up). The | 41 | ), |
| 39 | // reconnecting tunnel re-establishes the forward, so the same baseUrl works. | 42 | // A genuine machine reboot, not a service kick: reboot the guest OS, GATE on |
| 43 | // the daemon actually going down (an orderly shutdown serves for several | ||
| 44 | // seconds + the reconnecting tunnel re-forwards, so "reachable" right after | ||
| 45 | // the reboot command would false-pass), then wait for the supervised daemon | ||
| 46 | // to auto-start and serve again — proving the boot-time auto-start path. | ||
| 40 | restart: () => | 47 | restart: () => |
| 41 | Effect.promise(async () => { | 48 | Effect.promise(async () => { |
| 42 | if (os === "windows") throw new Error("cli-windows restart pending the ec2 provider"); | 49 | const host = env("E2E_CLI_VM_HOST"); |
| 43 | await sshRebootGuest(env("E2E_CLI_VM_HOST")); | 50 | if (os === "windows") { |
| 51 | await ec2RebootGuest(host, env("E2E_CLI_SSH_KEY"), os); | ||
| 52 | } else { | ||
| 53 | await sshRebootGuest(host); | ||
| 54 | } | ||
| 55 | await waitForHttpDown(`${baseUrl}/`, { timeoutMs: 120_000 }); | ||
| 44 | await waitForHttp(`${baseUrl}/`, { timeoutMs: 240_000 }); | 56 | await waitForHttp(`${baseUrl}/`, { timeoutMs: 240_000 }); |
| 45 | }), | 57 | }), |
| 46 | }; | 58 | }; |