History

e2e/cli: bearer-gated daemon target, reboot down-gate, EC2 provider for Windows

- cli target authenticates with the daemon's auth.json bearer (was Basic from service.key); globalsetup reads the token from the guest and publishes it. - restart() gates on the daemon going DOWN before polling it back up, so an orderly shutdown / reconnecting tunnel can't false-pass a reboot that never happened (waitForHttpDown). - EC2 VmProvider (e2e/src/vm/ec2.ts) + cli-windows globalsetup unblock Windows: launch, key-based SSH/PowerShell drive, real boot-time-gated reboot, terminate.

Rhys Sullivan committed 5 months ago

Commit d5ae77d1 parent cfd22846 files · +532 −33Browse files

Unified · Split

6 files
  1. e2e/setup/boot.ts +24 −0
  2. e2e/setup/cli-windows.globalsetup.ts +3 −0
  3. e2e/setup/cli.globalsetup.ts +54 −22
  4. e2e/src/vm/ec2.ts +422 −0
  5. e2e/src/vm/types.ts +6 −0
  6. e2e/targets/cli.ts +23 −11
e2e/setup/boot.ts+24−0
@@ -100,3 +100,27 @@ export const waitForHttp = async ( · 99 unmodified lines
100100 }
101101 throw new Error(`timed out waiting for ${url}: ${String(lastError)}`);
102102 };
103+
104+/**
105+ * Reboot DOWN-GATE: wait until `url` stops answering (fetch rejects — connection
106+ * refused/reset). An orderly OS shutdown keeps the daemon serving for several
107+ * seconds, and a reconnecting tunnel re-establishes the forward, so polling for
108+ * "up" right after a reboot command can false-pass a reboot that never happened.
109+ * Gating on the server actually going DOWN first makes restart-persistence prove
110+ * a real reboot. Throws if it never goes down within the deadline.
111+ */
112+export const waitForHttpDown = async (
113+ url: string,
114+ options: { readonly timeoutMs?: number } = {},
115+): Promise<void> => {
116+ const deadline = Date.now() + (options.timeoutMs ?? 120_000);
117+ while (Date.now() < deadline) {
118+ try {
119+ await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(2000) });
120+ } catch {
121+ return; // the server (or its tunnel) is gone — the reboot took
122+ }
123+ await new Promise((resolve) => setTimeout(resolve, 500));
124+ }
125+ throw new Error(`${url} never became unreachable — the reboot may not have taken`);
126+};
e2e/setup/cli-windows.globalsetup.ts+3−0
@@ -0,0 +1,3 @@
1+import { setupCliTarget } from "./cli.globalsetup";
2+
3+export default (): Promise<(() => Promise<void>) | void> => setupCliTarget("windows");
e2e/setup/cli.globalsetup.ts+54−22
@@ -3,42 +3,73 @@ · 2 unmodified lines
33 // the host over a reconnecting SSH tunnel. Publishes connection + reboot info
44 // via env (inherited by the test workers spawned afterward). Per-OS entrypoints
55 // (cli-macos.globalsetup.ts, …) call setupCliTarget with their OS.
6+//
7+// macOS + Linux run on local tart guests (Apple-Silicon host); Windows runs on
8+// an ephemeral EC2 instance. The supervised daemon is bearer-gated: it mints/
9+// loads its token into auth.json, which we read from the guest and publish so
10+// the api surface authenticates with `Authorization: Bearer`.
611 import { buildGuestBinary } from "../src/vm/build-binary";
12+import { ec2Vm } from "../src/vm/ec2";
713 import { tartVm } from "../src/vm/tart";
8−import type { VmArch, VmOs } from "../src/vm/types";
14+import type { VmArch, VmHandle, VmOs } from "../src/vm/types";
915 import { waitForHttp } from "./boot";
1016
1117 const PORT = 4789;
12−const GUEST_DIR = "~/ed";
1318
14−export async function setupCliTarget(os: VmOs): Promise<(() => Promise<void>) | void> {
15− process.env.E2E_VM_OS = os; // so the worker-side target resolves the same OS
19+// Per-OS guest specifics: working dir, binary name, and the shell idioms for
20+// prep/cleanup — Unix `sh` on tart (macOS/Linux), PowerShell on EC2 Windows.
21+const guestPlan = (os: VmOs) => {
1622 if (os === "windows") {
17− throw new Error("cli-windows is pending the ec2 provider; run cli-macos / cli-linux for now");
23+ const dir = "C:/ed";
24+ const exe = `${dir}/executor.exe`;
25+ return {
26+ dir,
27+ prep: `Remove-Item -Recurse -Force '${dir}' -ErrorAction SilentlyContinue; New-Item -ItemType Directory -Force -Path '${dir}' | Out-Null`,
28+ postPush: "Write-Output ok", // no chmod/quarantine on Windows
29+ install: `& '${exe}' service install --port ${PORT}`,
30+ readToken: 'Get-Content "$env:USERPROFILE\\.executor\\server-control\\auth.json" -Raw',
31+ uninstall: `& '${exe}' service uninstall`,
32+ };
1833 }
34+ const dir = "~/ed";
35+ const exe = `${dir}/executor`;
36+ return {
37+ dir,
38+ prep: `rm -rf ${dir} && mkdir -p ${dir}`,
39+ // macOS quarantines scp'd executables; clear it so the binary can run.
40+ postPush:
41+ os === "macos"
42+ ? `chmod +x ${exe}; xattr -dr com.apple.quarantine ${dir} 2>/dev/null || true`
43+ : `chmod +x ${exe}`,
44+ install: `${exe} service install --port ${PORT}`,
45+ readToken: "cat ~/.executor/server-control/auth.json",
46+ uninstall: `${exe} service uninstall`,
47+ };
48+};
1949
20− const arch: VmArch = "arm64"; // tart guests on an Apple-Silicon host
50+export async function setupCliTarget(os: VmOs): Promise<(() => Promise<void>) | void> {
51+ process.env.E2E_VM_OS = os; // so the worker-side target resolves the same OS
52+
53+ const arch: VmArch = os === "windows" ? "x64" : "arm64";
2154 const binDir = await buildGuestBinary(os, arch);
22− const vm = await tartVm(os, arch).provision();
55+ const vm: VmHandle =
56+ os === "windows" ? await ec2Vm(os, arch).provision() : await tartVm(os, arch).provision();
57+ const plan = guestPlan(os);
2358
2459 let tunnelClose: (() => void) | undefined;
2560 try {
26− await vm.ssh(`rm -rf ${GUEST_DIR} && mkdir -p ${GUEST_DIR}`);
27− await vm.push(`${binDir}/.`, `${GUEST_DIR}/`);
28− // macOS quarantines scp'd executables; clear it so the binary can run.
29− await vm.ssh(
30− os === "macos"
31− ? `chmod +x ${GUEST_DIR}/executor; xattr -dr com.apple.quarantine ${GUEST_DIR} 2>/dev/null || true`
32− : `chmod +x ${GUEST_DIR}/executor`,
33− );
61+ await vm.ssh(plan.prep);
62+ await vm.push(`${binDir}/.`, os === "windows" ? plan.dir : `${plan.dir}/`);
63+ await vm.ssh(plan.postPush);
3464
35− const install = await vm.ssh(`${GUEST_DIR}/executor service install --port ${PORT}`);
65+ const install = await vm.ssh(plan.install);
3666 if (install.code !== 0) {
3767 throw new Error(`service install failed: ${install.stderr.trim() || install.stdout.trim()}`);
3868 }
3969
40− const keyRaw = (await vm.ssh("cat ~/.executor/server-control/service.key")).stdout.trim();
41− const password = (JSON.parse(keyRaw) as { password: string }).password;
70+ // The supervised daemon mints/loads its bearer into auth.json on first boot.
71+ const tokenRaw = (await vm.ssh(plan.readToken)).stdout.trim();
72+ const token = (JSON.parse(tokenRaw) as { token: string }).token;
4273
4374 const tunnel = await vm.tunnel(PORT);
4475 tunnelClose = tunnel.close;
@@ -46,20 +77,21 @@ export async function setupCliTarget(os: VmOs): Promise<(() => Promise<void>) | · 1 unmodified line
4677 await waitForHttp(`${baseUrl}/`, { timeoutMs: 60_000 });
4778
4879 process.env.E2E_CLI_BASE_URL = baseUrl;
49− process.env.E2E_CLI_AUTH_PASSWORD = password;
80+ process.env.E2E_CLI_AUTH_TOKEN = token;
5081 process.env.E2E_CLI_VM_HOST = vm.host;
82+ if (vm.sshKeyPath) process.env.E2E_CLI_SSH_KEY = vm.sshKeyPath;
5183 process.env.E2E_CLI_TUNNEL_PORT = String(tunnel.localPort);
52− process.env.E2E_CLI_BIN_DIR = GUEST_DIR;
84+ process.env.E2E_CLI_BIN_DIR = plan.dir;
5385 } catch (error) {
5486 tunnelClose?.();
55− await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined);
87+ await vm.ssh(plan.uninstall).catch(() => undefined);
5688 await vm.discard();
5789 throw error;
5890 }
5991
6092 return async () => {
6193 tunnelClose?.();
62− await vm.ssh(`${GUEST_DIR}/executor service uninstall`).catch(() => undefined);
94+ await vm.ssh(plan.uninstall).catch(() => undefined);
6395 await vm.discard();
6496 };
6597 }
e2e/src/vm/ec2.ts+422−0
@@ -0,0 +1,422 @@
1+// ec2 provider: ephemeral guests on AWS EC2 for the cross-OS supervised-daemon
2+// e2e where tart can't run (Windows; optionally Linux). Mirrors tart.ts — launch
3+// a fresh instance, drive over SSH (key-based; PowerShell on Windows), REBOOT for
4+// real, tear down.
5+//
6+// Credentials are NEVER embedded here: the `aws` CLI uses the ambient sign-in
7+// (`aws configure` / env). Every instance is tagged `executor-e2e` and always
8+// terminated on discard; the security group is scoped to this host's egress IP.
9+//
10+// Reboot is gated on a real boot-time change (Windows `LastBootUpTime`), not mere
11+// SSH reachability — an orderly shutdown keeps the daemon serving for several
12+// seconds, so "SSH answered" alone can false-pass a reboot that never happened.
13+
14+import { execFile } from "node:child_process";
15+import { chmodSync, mkdtempSync, writeFileSync } from "node:fs";
16+import net from "node:net";
17+import { tmpdir } from "node:os";
18+import { join } from "node:path";
19+import { promisify } from "node:util";
20+
21+import {
22+ type SshResult,
23+ sleep,
24+ type VmArch,
25+ type VmHandle,
26+ type VmOs,
27+ type VmProvider,
28+} from "./types";
29+
30+const execFileP = promisify(execFile);
31+
32+const REGION = process.env.E2E_EC2_REGION ?? "us-west-2";
33+const INSTANCE_TYPE = process.env.E2E_EC2_INSTANCE_TYPE ?? "t3.medium";
34+const TAG = "executor-e2e";
35+
36+const SSH_OPTS = [
37+ "-o",
38+ "StrictHostKeyChecking=no",
39+ "-o",
40+ "UserKnownHostsFile=/dev/null",
41+ "-o",
42+ "ConnectTimeout=10",
43+ "-o",
44+ "ServerAliveInterval=10",
45+ "-o",
46+ "LogLevel=ERROR",
47+];
48+
49+const guestUser = (os: VmOs): string =>
50+ os === "windows" ? "Administrator" : (process.env.E2E_EC2_LINUX_USER ?? "ubuntu");
51+
52+/**
53+ * Reboot an EC2 guest by address, statelessly (no live handle) — the mirror of
54+ * tart's sshRebootGuest, for the worker-side `restart()`. The connection drops
55+ * mid-call, so errors are swallowed; the caller's down-gate + up-poll confirm
56+ * the real reboot.
57+ */
58+export const ec2RebootGuest = async (
59+ host: string,
60+ keyPath: string,
61+ os: VmOs = "windows",
62+): Promise<void> => {
63+ const cmd = os === "windows" ? "Restart-Computer -Force" : "sudo reboot";
64+ await execFileP("ssh", ["-i", keyPath, ...SSH_OPTS, `${guestUser(os)}@${host}`, cmd]).catch(
65+ () => undefined,
66+ );
67+};
68+
69+const aws = async (args: ReadonlyArray<string>): Promise<string> => {
70+ const { stdout } = await execFileP("aws", ["--region", REGION, "--output", "text", ...args], {
71+ maxBuffer: 64 * 1024 * 1024,
72+ });
73+ return stdout.trim();
74+};
75+
76+/** This host's public egress IP, for the inbound-SSH security-group rule. */
77+const egressIp = async (): Promise<string> => {
78+ const { stdout } = await execFileP("curl", [
79+ "-s",
80+ "--max-time",
81+ "10",
82+ "https://checkip.amazonaws.com",
83+ ]);
84+ return stdout.trim();
85+};
86+
87+/** Latest AWS-published base AMI for the guest OS (resolve dynamically — ids rotate). */
88+const latestAmi = async (os: VmOs): Promise<string> => {
89+ if (os === "windows") {
90+ const viaSsm = await aws([
91+ "ssm",
92+ "get-parameters",
93+ "--names",
94+ "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base",
95+ "--query",
96+ "Parameters[0].Value",
97+ ]).catch(() => "");
98+ if (viaSsm && viaSsm !== "None") return viaSsm;
99+ return aws([
100+ "ec2",
101+ "describe-images",
102+ "--owners",
103+ "amazon",
104+ "--filters",
105+ "Name=name,Values=Windows_Server-2022-English-Full-Base-*",
106+ "Name=state,Values=available",
107+ "--query",
108+ "reverse(sort_by(Images,&CreationDate))[0].ImageId",
109+ ]);
110+ }
111+ return aws([
112+ "ssm",
113+ "get-parameters",
114+ "--names",
115+ "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id",
116+ "--query",
117+ "Parameters[0].Value",
118+ ]);
119+};
120+
121+const defaultSubnet = async (): Promise<string> => {
122+ const vpc = await aws([
123+ "ec2",
124+ "describe-vpcs",
125+ "--filters",
126+ "Name=isDefault,Values=true",
127+ "--query",
128+ "Vpcs[0].VpcId",
129+ ]);
130+ const subnet = await aws([
131+ "ec2",
132+ "describe-subnets",
133+ "--filters",
134+ `Name=vpc-id,Values=${vpc}`,
135+ "Name=default-for-az,Values=true",
136+ "--query",
137+ "Subnets[0].SubnetId",
138+ ]);
139+ return subnet && subnet !== "None"
140+ ? subnet
141+ : aws([
142+ "ec2",
143+ "describe-subnets",
144+ "--filters",
145+ `Name=vpc-id,Values=${vpc}`,
146+ "--query",
147+ "Subnets[0].SubnetId",
148+ ]);
149+};
150+
151+/** Create (idempotently) a security group allowing inbound SSH from this host. */
152+const ensureSecurityGroup = async (myIp: string): Promise<string> => {
153+ const name = `${TAG}-sg`;
154+ let sg = await aws([
155+ "ec2",
156+ "describe-security-groups",
157+ "--filters",
158+ `Name=group-name,Values=${name}`,
159+ "--query",
160+ "SecurityGroups[0].GroupId",
161+ ]).catch(() => "");
162+ if (!sg || sg === "None") {
163+ sg = await aws([
164+ "ec2",
165+ "create-security-group",
166+ "--group-name",
167+ name,
168+ "--description",
169+ "executor e2e ephemeral guests (SSH from CI host)",
170+ "--query",
171+ "GroupId",
172+ ]);
173+ }
174+ // Authorize this host's IP for SSH; ignore "already exists".
175+ await aws([
176+ "ec2",
177+ "authorize-security-group-ingress",
178+ "--group-id",
179+ sg,
180+ "--protocol",
181+ "tcp",
182+ "--port",
183+ "22",
184+ "--cidr",
185+ `${myIp}/32`,
186+ ]).catch(() => undefined);
187+ return sg;
188+};
189+
190+/** PowerShell user-data: enable OpenSSH, default the shell to PowerShell, and
191+ * authorize our public key for the Administrator account. */
192+const windowsUserData = (publicKey: string): string =>
193+ [
194+ "<powershell>",
195+ "Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0",
196+ "Set-Service -Name sshd -StartupType Automatic",
197+ "Start-Service sshd",
198+ "New-ItemProperty -Path 'HKLM:\\SOFTWARE\\OpenSSH' -Name DefaultShell -Value 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' -PropertyType String -Force",
199+ "New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -ErrorAction SilentlyContinue",
200+ "$akf = 'C:\\ProgramData\\ssh\\administrators_authorized_keys'",
201+ `Set-Content -Path $akf -Value '${publicKey}'`,
202+ "icacls $akf /inheritance:r /grant 'Administrators:F' /grant 'SYSTEM:F'",
203+ "</powershell>",
204+ ].join("\n");
205+
206+const linuxUserData = (publicKey: string): string =>
207+ ["#cloud-config", "ssh_authorized_keys:", ` - ${publicKey}`].join("\n");
208+
209+const freePort = (): Promise<number> =>
210+ new Promise((resolve, reject) => {
211+ const srv = net.createServer();
212+ srv.on("error", reject);
213+ srv.listen(0, "127.0.0.1", () => {
214+ const port = (srv.address() as net.AddressInfo).port;
215+ srv.close(() => resolve(port));
216+ });
217+ });
218+
219+const waitLocalPort = async (port: number, attempts = 40): Promise<void> => {
220+ for (let i = 0; i < attempts; i++) {
221+ const ok = await new Promise<boolean>((resolve) => {
222+ const sock = net.connect({ host: "127.0.0.1", port }, () => {
223+ sock.destroy();
224+ resolve(true);
225+ });
226+ sock.on("error", () => resolve(false));
227+ sock.setTimeout(1000, () => {
228+ sock.destroy();
229+ resolve(false);
230+ });
231+ });
232+ if (ok) return;
233+ await sleep(500);
234+ }
235+ throw new Error(`tunnel local port ${port} never came up`);
236+};
237+
238+export const ec2Vm = (os: VmOs, arch: VmArch = "x64"): VmProvider => ({
239+ os,
240+ provision: async () => {
241+ const user = guestUser(os);
242+ // A throwaway SSH keypair, authorized via user-data (no EC2 key pair needed —
243+ // we drive over OpenSSH key auth, not the Windows password).
244+ const keyDir = mkdtempSync(join(tmpdir(), "executor-ec2-"));
245+ const keyPath = join(keyDir, "id");
246+ await execFileP("ssh-keygen", ["-t", "ed25519", "-N", "", "-q", "-f", keyPath]);
247+ chmodSync(keyPath, 0o600);
248+ const publicKey = (await execFileP("ssh-keygen", ["-y", "-f", keyPath])).stdout.trim();
249+
250+ const [myIp, ami, subnet] = await Promise.all([egressIp(), latestAmi(os), defaultSubnet()]);
251+ const sg = await ensureSecurityGroup(myIp);
252+ const userData = os === "windows" ? windowsUserData(publicKey) : linuxUserData(publicKey);
253+ const userDataFile = join(keyDir, "user-data.txt");
254+ writeFileSync(userDataFile, userData);
255+
256+ const instanceId = await aws([
257+ "ec2",
258+ "run-instances",
259+ "--image-id",
260+ ami,
261+ "--instance-type",
262+ INSTANCE_TYPE,
263+ "--count",
264+ "1",
265+ "--security-group-ids",
266+ sg,
267+ "--subnet-id",
268+ subnet,
269+ "--associate-public-ip-address",
270+ "--instance-initiated-shutdown-behavior",
271+ "terminate",
272+ "--user-data",
273+ `file://${userDataFile}`,
274+ "--tag-specifications",
275+ `ResourceType=instance,Tags=[{Key=Name,Value=${TAG}-${os}},{Key=purpose,Value=e2e}]`,
276+ "--query",
277+ "Instances[0].InstanceId",
278+ ]);
279+
280+ let ip = "";
281+ const tunnelClosers: Array<() => void> = [];
282+
283+ const ssh = async (command: string): Promise<SshResult> => {
284+ try {
285+ const { stdout, stderr } = await execFileP(
286+ "ssh",
287+ ["-i", keyPath, ...SSH_OPTS, `${user}@${ip}`, command],
288+ { maxBuffer: 64 * 1024 * 1024 },
289+ );
290+ return { stdout, stderr, code: 0 };
291+ } catch (err) {
292+ const e = err as { stdout?: string; stderr?: string; code?: number };
293+ return {
294+ stdout: e.stdout ?? "",
295+ stderr: e.stderr ?? "",
296+ code: typeof e.code === "number" ? e.code : 1,
297+ };
298+ }
299+ };
300+
301+ const waitSshUp = async (attempts: number): Promise<boolean> => {
302+ for (let i = 0; i < attempts; i++) {
303+ if ((await ssh(os === "windows" ? "echo ok" : "true")).code === 0) return true;
304+ await sleep(5000);
305+ }
306+ return false;
307+ };
308+
309+ const waitSshDown = async (attempts = 40): Promise<void> => {
310+ for (let i = 0; i < attempts; i++) {
311+ if ((await ssh("echo up").catch(() => ({ code: 1 }) as SshResult)).code !== 0) return;
312+ await sleep(3000);
313+ }
314+ // never observed down — caller's boot-time check is the backstop.
315+ };
316+
317+ const bootTime = async (): Promise<string> =>
318+ os === "windows"
319+ ? (
320+ await ssh("(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToString('o')")
321+ ).stdout.trim()
322+ : (await ssh("cat /proc/sys/kernel/random/boot_id")).stdout.trim();
323+
324+ const handle: VmHandle = {
325+ os,
326+ arch,
327+ sshKeyPath: keyPath,
328+ get host() {
329+ return ip;
330+ },
331+ ssh,
332+ push: async (localPath, remotePath) => {
333+ await execFileP("scp", [
334+ "-i",
335+ keyPath,
336+ "-r",
337+ ...SSH_OPTS,
338+ localPath,
339+ `${user}@${ip}:${remotePath}`,
340+ ]);
341+ },
342+ reboot: async () => {
343+ const before = await bootTime();
344+ await ssh(os === "windows" ? "Restart-Computer -Force" : "sudo reboot").catch(
345+ () => undefined,
346+ );
347+ await waitSshDown();
348+ if (!(await waitSshUp(60))) throw new Error(`ec2 ${os}: SSH did not return after reboot`);
349+ const after = await bootTime();
350+ if (before && after && before === after) {
351+ throw new Error(
352+ `ec2 ${os}: boot time unchanged after reboot — the guest never actually rebooted`,
353+ );
354+ }
355+ },
356+ tunnel: async (guestPort) => {
357+ const localPort = await freePort();
358+ let closed = false;
359+ let child: ReturnType<typeof import("node:child_process").spawn> | undefined;
360+ const { spawn } = await import("node:child_process");
361+ const spawnOnce = (): void => {
362+ child = spawn(
363+ "ssh",
364+ [
365+ "-i",
366+ keyPath,
367+ ...SSH_OPTS,
368+ "-N",
369+ "-L",
370+ `${localPort}:127.0.0.1:${guestPort}`,
371+ `${user}@${ip}`,
372+ ],
373+ { stdio: "ignore" },
374+ );
375+ child.on("exit", () => {
376+ if (!closed) setTimeout(spawnOnce, 2000);
377+ });
378+ };
379+ spawnOnce();
380+ const close = (): void => {
381+ closed = true;
382+ child?.kill();
383+ };
384+ tunnelClosers.push(close);
385+ await waitLocalPort(localPort);
386+ return { localPort, close };
387+ },
388+ discard: async () => {
389+ for (const close of tunnelClosers) close();
390+ await aws(["ec2", "terminate-instances", "--instance-ids", instanceId]).catch(
391+ () => undefined,
392+ );
393+ },
394+ };
395+
396+ // Wait for a public IP, then for OpenSSH (Windows boot + FoD install ≈ 2-4 min).
397+ for (let i = 0; i < 60; i++) {
398+ const got = await aws([
399+ "ec2",
400+ "describe-instances",
401+ "--instance-ids",
402+ instanceId,
403+ "--query",
404+ "Reservations[0].Instances[0].PublicIpAddress",
405+ ]).catch(() => "");
406+ if (got && got !== "None") {
407+ ip = got;
408+ break;
409+ }
410+ await sleep(5000);
411+ }
412+ if (!ip) {
413+ await handle.discard();
414+ throw new Error(`ec2 ${os}: no public IP within 300s`);
415+ }
416+ if (!(await waitSshUp(60))) {
417+ await handle.discard();
418+ throw new Error(`ec2 ${os}: SSH never came up`);
419+ }
420+ return handle;
421+ },
422+});
e2e/src/vm/types.ts+6−0
@@ -24,6 +24,12 @@ export interface VmHandle { · 23 unmodified lines
2424 readonly arch: VmArch;
2525 /** Current reachable address of the guest (re-resolved across reboots). */
2626 readonly host: string;
27+ /**
28+ * Path to the SSH private key for key-based providers (EC2). Undefined for
29+ * password-based providers (tart/sshpass). Published by globalsetup so the
30+ * stateless worker-side `restart()` can reboot the guest.
31+ */
32+ readonly sshKeyPath?: string;
2733 /** Run a command in the guest over SSH (shell on Unix, PowerShell on Windows). */
2834 ssh(command: string): Promise<SshResult>;
2935 /** Copy a local file or directory into the guest (recursive for directories). */
e2e/targets/cli.ts+23−11
@@ -8,7 +8,8 @@ · 7 unmodified lines
88 // this target only reads what that published via env.
99 import { Effect } from "effect";
1010
11−import { waitForHttp } from "../setup/boot";
11+import { waitForHttp, waitForHttpDown } from "../setup/boot";
12+import { ec2RebootGuest } from "../src/vm/ec2";
1213 import { sshRebootGuest } from "../src/vm/tart";
1314 import type { VmOs } from "../src/vm/types";
1415 import type { Capability, Identity, Target } from "../src/target";
@@ -22,25 +23,36 @@ const env = (key: string): string => { · 7 unmodified lines
2223 export const cliTarget = (): Target => {
2324 const baseUrl = env("E2E_CLI_BASE_URL");
2425 const os = (process.env.E2E_VM_OS ?? "macos") as VmOs;
25− const username = process.env.E2E_CLI_AUTH_USER ?? "executor";
2626
2727 return {
2828 name: process.env.E2E_TARGET ?? "cli",
2929 baseUrl,
3030 mcpUrl: `${baseUrl}/mcp`,
3131 capabilities: new Set<Capability>(["api"]),
32+ // The supervised daemon is bearer-gated (auth.json); globalsetup reads the
33+ // token from the guest and publishes it. A wrong/absent token still gets a
34+ // clean 401, which the api surface treats as "up".
3235 newIdentity: () =>
33− Effect.sync((): Identity => {
34− const basic = Buffer.from(`${username}:${env("E2E_CLI_AUTH_PASSWORD")}`).toString("base64");
35− return { label: "cli-daemon", headers: { Authorization: `Basic ${basic}` } };
36− }),
37− // A genuine machine reboot, not a service kick: reboot the guest OS and wait
38− // for the supervised daemon to auto-start and serve again (401 = up). The
39− // reconnecting tunnel re-establishes the forward, so the same baseUrl works.
36+ Effect.sync(
37+ (): Identity => ({
38+ label: "cli-daemon",
39+ headers: { Authorization: `Bearer ${env("E2E_CLI_AUTH_TOKEN")}` },
40+ }),
41+ ),
42+ // A genuine machine reboot, not a service kick: reboot the guest OS, GATE on
43+ // the daemon actually going down (an orderly shutdown serves for several
44+ // seconds + the reconnecting tunnel re-forwards, so "reachable" right after
45+ // the reboot command would false-pass), then wait for the supervised daemon
46+ // to auto-start and serve again — proving the boot-time auto-start path.
4047 restart: () =>
4148 Effect.promise(async () => {
42− if (os === "windows") throw new Error("cli-windows restart pending the ec2 provider");
43− await sshRebootGuest(env("E2E_CLI_VM_HOST"));
49+ const host = env("E2E_CLI_VM_HOST");
50+ if (os === "windows") {
51+ await ec2RebootGuest(host, env("E2E_CLI_SSH_KEY"), os);
52+ } else {
53+ await sshRebootGuest(host);
54+ }
55+ await waitForHttpDown(`${baseUrl}/`, { timeoutMs: 120_000 });
4456 await waitForHttp(`${baseUrl}/`, { timeoutMs: 240_000 });
4557 }),
4658 };

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t