Make fresh-worktree e2e turnkey: per-checkout ports, port-squat fail-fast, bootstrap script

- e2e ports are now derived from a hash of the checkout root (42000-45999 blocks) so concurrent worktrees never collide; E2E_*_PORT/URL still override. - globalsetups fail fast with the squatting PID when a port is taken instead of silently attaching to a foreign checkout's dev server (the cause of 31-scenario auth-error cascades). - scripts/bootstrap.ts: one idempotent command for submodules + install + playwright; scripts/reap-dev-servers.ts kills orphaned dev stacks. - e2e: bun run ports prints this checkout's derived ports. - sources-api scenario: align with payload-first OpenAPI results (PR #956 changed the product, the cloud-only scenario was not updated). - AGENTS.md: fresh-worktree setup + environment gotchas (fish shell, /tmp scripts, bun.lock conflicts, per-checkout ports).

Rhys Sullivan committed 5 months ago

Commit a2bb24e1 parent cbf936515 files · +242 −16Browse files

Unified · Split

15 files

.gitignore +3 −0
AGENTS.md +23 −0
e2e/AGENTS.md +11 −4
e2e/cloud/sources-api.test.ts +3 −4
e2e/package.json +1 −0
e2e/scripts/ports.ts +12 −0
e2e/setup/boot.ts +39 −1
e2e/setup/cloud.globalsetup.ts +8 −1
e2e/setup/selfhost.globalsetup.ts +3 −1
e2e/src/ports.ts +33 −0
e2e/targets/cloud.ts +5 −4
e2e/targets/selfhost.ts +2 −1
package.json +2 −0
scripts/bootstrap.ts +32 −0
scripts/reap-dev-servers.ts +65 −0

.gitignore+3−0

@@ -102,6 +102,9 @@ executor.jsonc · 101 unmodified lines
102	102	MISTAKES.md
103	103	DESIRES.md
104	104	LEARNINGS.md
	105	+
	106	+# Agent scratch scripts (workspace-resolvable alternative to /tmp)
	107	+scratch/
105	108	.alchemy/
106	109
107	110	# Pi coding agent

AGENTS.md+23−0

@@ -1,5 +1,28 @@
1	1	# AGENTS.md
2	2
	3	+## Fresh Checkout / Worktree Setup
	4	+
	5	+Run `bun run bootstrap` first in any fresh checkout or worktree. It is
	6	+idempotent: inits the vendor submodules (`vendor/emulate`, `vendor/mcporter` —
	7	+the e2e suite needs both), runs `bun install` (whose prepare hook builds the
	8	+internal packages dev servers fail without), and installs Playwright chromium.
	9	+Skipping it is why fresh worktrees die with "Failed to resolve entry for
	10	+package '@executor-js/vite-plugin'".
	11	+
	12	+## Environment Gotchas (learned the hard way)
	13	+
	14	+- The shell is fish, and the working directory resets between Bash calls. Use
	15	+ absolute paths rooted at THIS worktree (check `pwd`), never
	16	+ `/Users/rhys/src/executor` from memory, and don't rely on a prior `cd`.
	17	+- Don't write probe scripts to `/tmp` — they can't resolve workspace packages
	18	+ (`effect`, `playwright`, ...). Put scratch scripts under the repo root
	19	+ (`scratch/` is gitignored) so bun resolves the workspace.
	20	+- `bun.lock` conflicts on rebase/merge: take either side, then re-run
	21	+ `bun install` to regenerate it — never hand-merge the lockfile.
	22	+- e2e dev-server ports are derived per checkout (`cd e2e && bun run ports`).
	23	+ If a boot reports a squatted port, an old dev server leaked — kill it by
	24	+ PID from the error message; don't move your own ports to dodge it.
	25	+
3	26	## Task Completion Requirements
4	27
5	28	- Use Effect Vitest for tests.

e2e/AGENTS.md+11−4

@@ -83,18 +83,25 @@ const r = yield * session.call("execute", { code: "return 1 + 1;" }); · 82 unmodified lines
83	83	cd e2e
84	84	bun run test # boots both dev servers, runs everything
85	85	bun run test:cloud # one target
86		−# attach to an already-running server while iterating:
87		−E2E_CLOUD_URL=http://127.0.0.1:4798 ../node_modules/.bin/vitest run --project cloud <file>
88		−E2E_SELFHOST_URL=http://localhost:4799 ../node_modules/.bin/vitest run --project selfhost <file>
	86	+bun run ports # print THIS checkout's derived ports
	87	+# attach to an already-running server while iterating (use `bun run ports` URLs):
	88	+E2E_CLOUD_URL=http://127.0.0.1:<port> ../node_modules/.bin/vitest run --project cloud <file>
	89	+E2E_SELFHOST_URL=http://localhost:<port> ../node_modules/.bin/vitest run --project selfhost <file>
89	90	```
90	91
	92	+Ports are derived per checkout (hash of the repo root — see `src/ports.ts`),
	93	+so suites in different worktrees never fight. If a port is somehow taken, the
	94	+boot fails fast naming the squatting process; kill it or override with
	95	+`E2E_CLOUD_PORT`-style env vars.
	96	+
91	97	Each run writes `runs/<target>/<slug>/result.json` plus any browser artifacts
92	98	(trace.zip / session.mp4 / screenshots). `bun run serve` hosts the scenario ×
93	99	target matrix; a run page links the trace into Playwright's trace viewer.
94	100
95	101	## Discovering endpoints
96	102
97		−- The full OpenAPI spec: `curl http://127.0.0.1:4798/api/openapi.json` (cloud).
	103	+- The full OpenAPI spec: `curl http://127.0.0.1:<cloud port>/api/openapi.json`
	104	+ (cloud; port from `bun run ports`).
98	105	- The typed client mirrors it: `client.<group>.<endpoint>(...)` with groups
99	106	tools/integrations/connections/providers/executions/oauth/policies.
100	107	- To see payload shapes, read the API definitions under

e2e/cloud/sources-api.test.ts+3−4

@@ -184,13 +184,12 @@ scenario( · 183 unmodified lines
184	184	}),
185	185	);
186	186	expect(execution.isError, "the execution succeeded").toBe(false);
	187	+ // Payload-first: `data` IS the upstream body; transport facts (status,
	188	+ // headers) ride in the optional `http` side channel.
187	189	expect(execution.structured, "the tool returned the upstream's echo").toMatchObject({
188	190	result: {
189	191	ok: true,
190		− data: {
191		− status: 200,
192		− data: { message: "hello", suffix: "world", path: "/echo/hello" },
193		− },
	192	+ data: { message: "hello", suffix: "world", path: "/echo/hello" },
194	193	},
195	194	});
196	195

e2e/package.json+1−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	"test:cloud": "vitest run --project cloud",
9	9	"test:selfhost": "vitest run --project selfhost",
10	10	"test:watch": "vitest",
	11	+ "ports": "bun scripts/ports.ts",
11	12	"viewer:build": "bun scripts/rebuild-viewer.ts",
12	13	"serve": "bun scripts/rebuild-viewer.ts && bun scripts/serve.ts",
13	14	"typecheck": "tsc --noEmit",

e2e/scripts/ports.ts+12−0

@@ -0,0 +1,12 @@
	1	+// Print this checkout's derived e2e ports (see src/ports.ts) so an agent or
	2	+// human can curl the booted servers or attach with E2E_<TARGET>_URL.
	3	+import { AUTUMN_EMULATOR_PORT, CLOUD_DB_PORT, CLOUD_PORT, WORKOS_EMULATOR_PORT } from "../targets/cloud";
	4	+import { SELFHOST_PORT } from "../targets/selfhost";
	5	+import { repoRoot } from "../src/ports";
	6	+
	7	+console.log(`e2e ports for ${repoRoot}`);
	8	+console.log(` cloud http://127.0.0.1:${CLOUD_PORT}`);
	9	+console.log(` cloud dev-db ${CLOUD_DB_PORT}`);
	10	+console.log(` workos emulator ${WORKOS_EMULATOR_PORT}`);
	11	+console.log(` autumn emulator ${AUTUMN_EMULATOR_PORT}`);
	12	+console.log(` selfhost http://localhost:${SELFHOST_PORT}`);

e2e/setup/boot.ts+39−1

@@ -2,7 +2,8 @@ · 1 unmodified line
2	2	// server, wait until it answers HTTP, and hand vitest a teardown. The apps own
3	3	// what runs (their dev stack, their stub flags); this file only owns process
4	4	// lifecycle, so it stays target-agnostic.
5		−import { spawn, type ChildProcess } from "node:child_process";
	5	+import { execFileSync, spawn, type ChildProcess } from "node:child_process";
	6	+import { connect } from "node:net";
6	7
7	8	export interface BootedProcesses {
8	9	readonly teardown: () => Promise<void>;
@@ -72,6 +73,43 @@ export const bootProcesses = ( · 63 unmodified lines
72	73	};
73	74	};
74	75
	76	+// A port already in LISTEN before we boot means waitForHttp would silently
	77	+// attach to a FOREIGN server (a leaked dev server from another checkout or a
	78	+// crashed prior run) — every scenario then fails with baffling auth errors
	79	+// instead of one clear message. Fail fast and name the squatter.
	80	+export const ensurePortsFree = async (
	81	+ ports: ReadonlyArray<{ readonly port: number; readonly label: string }>,
	82	+): Promise<void> => {
	83	+ for (const { port, label } of ports) {
	84	+ const inUse = await new Promise<boolean>((resolve) => {
	85	+ const socket = connect({ port, host: "127.0.0.1" });
	86	+ socket.once("connect", () => {
	87	+ socket.destroy();
	88	+ resolve(true);
	89	+ });
	90	+ socket.once("error", () => resolve(false));
	91	+ socket.setTimeout(1_000, () => {
	92	+ socket.destroy();
	93	+ resolve(false);
	94	+ });
	95	+ });
	96	+ if (!inUse) continue;
	97	+ let owner = "(lsof unavailable)";
	98	+ try {
	99	+ owner = execFileSync("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN"], {
	100	+ encoding: "utf8",
	101	+ }).trim();
	102	+ } catch {
	103	+ // lsof failing is fine — the error below carries the essential fact.
	104	+ }
	105	+ throw new Error(
	106	+ `e2e: port ${port} (${label}) is already in use — likely a leaked dev server ` +
	107	+ `from another checkout or a previous run. Kill it or set the E2E_*_PORT/URL ` +
	108	+ `env vars to relocate.\n${owner}`,
	109	+ );
	110	+ }
	111	+};
	112	+
75	113	export const waitForHttp = async (
76	114	url: string,
77	115	options: { readonly timeoutMs?: number; readonly expectRedirect?: boolean } = {},

e2e/setup/cloud.globalsetup.ts+8−1

@@ -10,7 +10,7 @@ import { fileURLToPath } from "node:url"; · 9 unmodified lines
10	10	// Vendored fork import (same pattern as mcporter).
11	11	import { createEmulator } from "@executor-js/emulate";
12	12
13		−import { bootProcesses, waitForHttp } from "./boot";
	13	+import { bootProcesses, ensurePortsFree, waitForHttp } from "./boot";
14	14	import {
15	15	CLOUD_BASE_URL,
16	16	CLOUD_DB_PORT,
@@ -29,6 +29,13 @@ export default async function setup(): Promise<(() => Promise<void>) \| void> { · 12 unmodified lines
29	29	return;
30	30	}
31	31
	32	+ await ensurePortsFree([
	33	+ { port: CLOUD_PORT, label: "cloud vite dev" },
	34	+ { port: CLOUD_DB_PORT, label: "cloud dev-db (PGlite)" },
	35	+ { port: WORKOS_EMULATOR_PORT, label: "WorkOS emulator" },
	36	+ { port: AUTUMN_EMULATOR_PORT, label: "Autumn emulator" },
	37	+ ]);
	38	+
32	39	// Fresh dev DB per suite run — hermetic, like the selfhost data dir. The
33	40	// WorkOS emulator mints org ids from a per-process counter, so a persisted
34	41	// DB from a previous invocation collides with the new boot's ids (identities

e2e/setup/selfhost.globalsetup.ts+3−1

@@ -6,7 +6,7 @@ import { rmSync } from "node:fs"; · 5 unmodified lines
6	6	import { resolve } from "node:path";
7	7	import { fileURLToPath } from "node:url";
8	8
9		−import { bootProcesses, waitForHttp } from "./boot";
	9	+import { bootProcesses, ensurePortsFree, waitForHttp } from "./boot";
10	10	import { SELFHOST_ADMIN, SELFHOST_BASE_URL, SELFHOST_PORT } from "../targets/selfhost";
11	11
12	12	const selfhostDir = fileURLToPath(new URL("../../apps/host-selfhost/", import.meta.url));
@@ -17,6 +17,8 @@ export default async function setup(): Promise<(() => Promise<void>) \| void> { · 4 unmodified lines
17	17	return;
18	18	}
19	19
	20	+ await ensurePortsFree([{ port: SELFHOST_PORT, label: "selfhost vite dev" }]);
	21	+
20	22	// Fresh data dir per suite run — hermetic; in-suite isolation comes from
21	23	// fresh identities, not resets.
22	24	const dataDir = resolve(selfhostDir, ".e2e-data");

e2e/src/ports.ts+33−0

@@ -0,0 +1,33 @@
	1	+// Per-checkout port derivation: every checkout (main repo, agent worktree,
	2	+// /tmp rig) gets its own deterministic block of e2e ports, so concurrent
	3	+// suites never fight over a shared default. The collision failure mode is
	4	+// brutal: vite's --strictPort exit is swallowed by the boot glue and
	5	+// waitForHttp happily attaches to the OTHER checkout's server, failing dozens
	6	+// of scenarios with baffling auth errors instead of one clear bind error.
	7	+// Individual E2E_*_PORT env vars still override, and E2E_<TARGET>_URL still
	8	+// attaches to a running instance.
	9	+import { resolve } from "node:path";
	10	+import { fileURLToPath } from "node:url";
	11	+
	12	+/** The repo root identifies the checkout (stable regardless of process cwd). */
	13	+export const repoRoot = resolve(fileURLToPath(new URL("../..", import.meta.url)));
	14	+
	15	+// FNV-1a — tiny, deterministic, and the same value in every process of this
	16	+// checkout (globalsetup and test workers must agree on the ports).
	17	+const hash = (text: string): number => {
	18	+ let h = 2166136261;
	19	+ for (let i = 0; i < text.length; i++) {
	20	+ h ^= text.charCodeAt(i);
	21	+ h = Math.imul(h, 16777619);
	22	+ }
	23	+ return h >>> 0;
	24	+};
	25	+
	26	+// 400 blocks of 10 ports in 42000..45999: unprivileged, clear of common dev
	27	+// servers, and below macOS's ephemeral range (49152+).
	28	+export const portBlock = 42000 + (hash(repoRoot) % 400) * 10;
	29	+
	30	+export const e2ePort = (envVar: string, offset: number): number => {
	31	+ const fromEnv = process.env[envVar];
	32	+ return fromEnv ? Number(fromEnv) : portBlock + offset;
	33	+};

e2e/targets/cloud.ts+5−4

@@ -8,13 +8,14 @@ import { randomUUID } from "node:crypto"; · 7 unmodified lines
8	8
9	9	import { Effect } from "effect";
10	10
	11	+import { e2ePort } from "../src/ports";
11	12	import type { Identity, Target } from "../src/target";
12	13
13		−export const CLOUD_PORT = Number(process.env.E2E_CLOUD_PORT ?? 4798);
14		−export const CLOUD_DB_PORT = Number(process.env.E2E_CLOUD_DB_PORT ?? 5436);
	14	+export const CLOUD_PORT = e2ePort("E2E_CLOUD_PORT", 0);
	15	+export const CLOUD_DB_PORT = e2ePort("E2E_CLOUD_DB_PORT", 1);
15	16	export const CLOUD_BASE_URL = process.env.E2E_CLOUD_URL ?? `http://127.0.0.1:${CLOUD_PORT}`;
16		−export const WORKOS_EMULATOR_PORT = Number(process.env.E2E_WORKOS_EMULATOR_PORT ?? 4914);
17		−export const AUTUMN_EMULATOR_PORT = Number(process.env.E2E_AUTUMN_EMULATOR_PORT ?? 4915);
	17	+export const WORKOS_EMULATOR_PORT = e2ePort("E2E_WORKOS_EMULATOR_PORT", 2);
	18	+export const AUTUMN_EMULATOR_PORT = e2ePort("E2E_AUTUMN_EMULATOR_PORT", 3);
18	19	export const E2E_WORKOS_CLIENT_ID = "client_e2e_emulate";
19	20	export const E2E_COOKIE_PASSWORD = "e2e_cookie_password_0123456789abcdef0123456789abcdef";
20	21

e2e/targets/selfhost.ts+2−1

@@ -6,9 +6,10 @@ import { Effect } from "effect"; · 5 unmodified lines
6	6
7	7	import { cookieConsentStrategy } from "@executor-js/mcporter";
8	8
	9	+import { e2ePort } from "../src/ports";
9	10	import type { Identity, Target } from "../src/target";
10	11
11		−export const SELFHOST_PORT = Number(process.env.E2E_SELFHOST_PORT ?? 4799);
	12	+export const SELFHOST_PORT = e2ePort("E2E_SELFHOST_PORT", 4);
12	13	export const SELFHOST_BASE_URL =
13	14	process.env.E2E_SELFHOST_URL ?? `http://localhost:${SELFHOST_PORT}`;
14	15

package.json+2−0

@@ -30,6 +30,8 @@ · 29 unmodified lines
30	30	],
31	31	"type": "module",
32	32	"scripts": {
	33	+ "bootstrap": "bun run scripts/bootstrap.ts",
	34	+ "reap": "bun run scripts/reap-dev-servers.ts",
33	35	"dev": "turbo run dev --filter='!@executor-js/desktop' --filter='!@executor-js/cloud'",
34	36	"dev:desktop": "turbo run dev",
35	37	"dev:cli": "EXECUTOR_DEV=1 EXECUTOR_DATA_DIR=${EXECUTOR_DATA_DIR:-apps/local/.executor-dev} bun run apps/cli/src/main.ts",

scripts/bootstrap.ts+32−0

@@ -0,0 +1,32 @@
	1	+// One-command setup for a fresh checkout or agent worktree: submodules,
	2	+// dependencies (whose prepare hook builds the internal packages dev servers
	3	+// need), and the Playwright browser the e2e suite drives. Idempotent and
	4	+// safe to re-run; each step prints what it is doing.
	5	+import { execFileSync } from "node:child_process";
	6	+import { existsSync } from "node:fs";
	7	+import { resolve } from "node:path";
	8	+import { fileURLToPath } from "node:url";
	9	+
	10	+const repoRoot = resolve(fileURLToPath(new URL("..", import.meta.url)));
	11	+
	12	+const run = (label: string, cmd: string, args: ReadonlyArray<string>) => {
	13	+ console.log(`\n[bootstrap] ${label}: ${cmd} ${args.join(" ")}`);
	14	+ execFileSync(cmd, [...args], { cwd: repoRoot, stdio: "inherit" });
	15	+};
	16	+
	17	+run("vendor submodules", "git", ["submodule", "update", "--init", "--recursive"]);
	18	+
	19	+// `bun install` runs the workspace prepare hook, which builds
	20	+// @executor-js/vite-plugin and @executor-js/react — the two artifacts the
	21	+// apps' vite dev servers fail without in a fresh worktree.
	22	+run("dependencies (+ prepare builds)", "bun", ["install"]);
	23	+
	24	+// e2e browser scenarios need Playwright's chromium; the cache is shared
	25	+// per-machine so this is a fast no-op when already present.
	26	+run("playwright chromium", "bunx", ["playwright", "install", "chromium"]);
	27	+
	28	+if (!existsSync(resolve(repoRoot, "node_modules/.bin/vitest"))) {
	29	+ throw new Error("bootstrap: vitest missing after install — bun install likely failed");
	30	+}
	31	+
	32	+console.log("\n[bootstrap] done — `cd e2e && bun run test` runs the full suite.");

scripts/reap-dev-servers.ts+65−0

@@ -0,0 +1,65 @@
	1	+// Find (and optionally kill) leaked dev-stack processes — vite dev servers,
	2	+// PGlite dev-dbs, e2e emulators — left behind when a session dies without
	3	+// running its teardown. Default: kill only ORPHANS (processes whose checkout
	4	+// path no longer exists, e.g. a removed worktree) and list the rest.
	5	+// `--all` also kills live-checkout servers (do this only when you know no
	6	+// other agent session is using them). `--dry-run` lists without killing.
	7	+import { execFileSync } from "node:child_process";
	8	+import { existsSync } from "node:fs";
	9	+
	10	+const args = new Set(process.argv.slice(2));
	11	+const dryRun = args.has("--dry-run");
	12	+const killAll = args.has("--all");
	13	+
	14	+const DEV_PATTERN = /vite dev\|dev-db\.ts\|scripts\/dev\.ts/;
	15	+
	16	+const ps = execFileSync("ps", ["-axo", "pid=,command="], { encoding: "utf8" });
	17	+
	18	+interface Candidate {
	19	+ readonly pid: number;
	20	+ readonly command: string;
	21	+ readonly checkout: string \| undefined;
	22	+ readonly orphan: boolean;
	23	+}
	24	+
	25	+const candidates: Candidate[] = [];
	26	+for (const line of ps.split("\n")) {
	27	+ const match = line.match(/^\s(\d+)\s+(.)$/);
	28	+ if (!match) continue;
	29	+ const [, pidText, command] = match;
	30	+ if (!DEV_PATTERN.test(command!)) continue;
	31	+ if (Number(pidText) === process.pid) continue;
	32	+ // The checkout root is whatever absolute path prefixes node_modules/ or a
	33	+ // workspace dir in the command line.
	34	+ const pathMatch = command!.match(/(\/[^ ]*?)\/(?:node_modules\|apps\|packages\|e2e)\//);
	35	+ const checkout = pathMatch?.[1];
	36	+ const orphan = checkout !== undefined && !existsSync(checkout);
	37	+ candidates.push({ pid: Number(pidText), command: command!.slice(0, 160), checkout, orphan });
	38	+}
	39	+
	40	+if (candidates.length === 0) {
	41	+ console.log("reap: no dev-stack processes found.");
	42	+ process.exit(0);
	43	+}
	44	+
	45	+for (const c of candidates) {
	46	+ const shouldKill = !dryRun && (c.orphan \|\| killAll);
	47	+ const tag = c.orphan ? "ORPHAN" : "live ";
	48	+ console.log(`${shouldKill ? "KILL " : "keep "} ${tag} pid=${c.pid} ${c.command}`);
	49	+ if (shouldKill) {
	50	+ try {
	51	+ // Group kill (detached boots) with a direct-pid fallback.
	52	+ try {
	53	+ process.kill(-c.pid, "SIGTERM");
	54	+ } catch {
	55	+ process.kill(c.pid, "SIGTERM");
	56	+ }
	57	+ } catch (error) {
	58	+ console.error(` failed to kill ${c.pid}: ${String(error)}`);
	59	+ }
	60	+ }
	61	+}
	62	+
	63	+if (!killAll && candidates.some((c) => !c.orphan)) {
	64	+ console.log("\nLive-checkout servers were kept (another session may own them); --all kills those too.");
	65	+}