Unified · Split
| @@ -0,0 +1,885 @@ | ||
| 1 | +/* oxlint-disable executor/no-try-catch-or-throw, executor/no-error-constructor, executor/no-promise-reject -- e2e boundary: drives launchd, a packaged Electron process, CDP, and optional ffmpeg recording */ | |
| 2 | +import { type ChildProcess, execFile, execFileSync, spawn } from "node:child_process"; | |
| 3 | +import { | |
| 4 | + chmodSync, | |
| 5 | + existsSync, | |
| 6 | + mkdirSync, | |
| 7 | + mkdtempSync, | |
| 8 | + readFileSync, | |
| 9 | + rmSync, | |
| 10 | + statSync, | |
| 11 | + writeFileSync, | |
| 12 | +} from "node:fs"; | |
| 13 | +import { homedir, tmpdir } from "node:os"; | |
| 14 | +import { dirname, join, resolve } from "node:path"; | |
| 15 | +import { promisify } from "node:util"; | |
| 16 | + | |
| 17 | +import { expect, it } from "@effect/vitest"; | |
| 18 | +import { Effect } from "effect"; | |
| 19 | + | |
| 20 | +import { scenario } from "../src/scenario"; | |
| 21 | +import { RunDir } from "../src/services"; | |
| 22 | + | |
| 23 | +const execFileAsync = promisify(execFile); | |
| 24 | +const SERVICE_LABEL = "sh.executor.daemon"; | |
| 25 | +const SCENARIO_NAME = | |
| 26 | + "Desktop packaged supervised attach: stale launchd registration pins the 15s dead boot wait"; | |
| 27 | +const RECORDING_FILE = resolve("recordings/stale-launchd-slow-boot.mp4"); | |
| 28 | +const RECORDING_WIDTH = 1280; | |
| 29 | +const RECORDING_HEIGHT = 720; | |
| 30 | +const STALE_PLIST_PORT = 58_251; | |
| 31 | + | |
| 32 | +interface PackagedExecutorBridge { | |
| 33 | + readonly getServerConnection: () => Promise<{ | |
| 34 | + readonly kind: string; | |
| 35 | + readonly origin: string; | |
| 36 | + } | null>; | |
| 37 | +} | |
| 38 | + | |
| 39 | +interface PackagedApp { | |
| 40 | + readonly child: ChildProcess; | |
| 41 | + cdp: CdpPage; | |
| 42 | + readonly debugPort: string; | |
| 43 | + readonly output: () => string; | |
| 44 | +} | |
| 45 | + | |
| 46 | +interface CdpResponse<T> { | |
| 47 | + readonly id: number; | |
| 48 | + readonly result?: T; | |
| 49 | + readonly error?: { readonly message?: string; readonly data?: string }; | |
| 50 | +} | |
| 51 | + | |
| 52 | +interface CdpEvaluateResult { | |
| 53 | + readonly result: { readonly value?: unknown }; | |
| 54 | + readonly exceptionDetails?: unknown; | |
| 55 | +} | |
| 56 | + | |
| 57 | +interface CdpTarget { | |
| 58 | + readonly type: string; | |
| 59 | + readonly url: string; | |
| 60 | + readonly webSocketDebuggerUrl?: string; | |
| 61 | +} | |
| 62 | + | |
| 63 | +interface ServerManifest { | |
| 64 | + readonly kind: string; | |
| 65 | + readonly pid: number; | |
| 66 | +} | |
| 67 | + | |
| 68 | +interface LaunchTiming { | |
| 69 | + readonly app: PackagedApp; | |
| 70 | + readonly pageCdpMs: number; | |
| 71 | +} | |
| 72 | + | |
| 73 | +interface BootResult { | |
| 74 | + readonly pageCdpMs: number; | |
| 75 | + readonly sidecarReadyMs: number; | |
| 76 | + readonly connectionKind: string; | |
| 77 | + readonly origin: string; | |
| 78 | + readonly manifestKind: string; | |
| 79 | + readonly manifestPid: number; | |
| 80 | + readonly matchedLogLine: string | null; | |
| 81 | +} | |
| 82 | + | |
| 83 | +declare global { | |
| 84 | + interface Window { | |
| 85 | + readonly executor: PackagedExecutorBridge; | |
| 86 | + } | |
| 87 | +} | |
| 88 | + | |
| 89 | +class CdpPage { | |
| 90 | + private nextId = 1; | |
| 91 | + private readonly pending = new Map< | |
| 92 | + number, | |
| 93 | + { | |
| 94 | + readonly resolve: (value: unknown) => void; | |
| 95 | + readonly reject: (error: Error) => void; | |
| 96 | + } | |
| 97 | + >(); | |
| 98 | + | |
| 99 | + private constructor(private readonly socket: WebSocket) { | |
| 100 | + socket.addEventListener("message", (event) => { | |
| 101 | + const data = event.data; | |
| 102 | + if (typeof data !== "string") return; | |
| 103 | + const message = JSON.parse(data) as CdpResponse<unknown>; | |
| 104 | + if (!message.id) return; | |
| 105 | + const pending = this.pending.get(message.id); | |
| 106 | + if (!pending) return; | |
| 107 | + this.pending.delete(message.id); | |
| 108 | + if (message.error) { | |
| 109 | + pending.reject(new Error(message.error.message ?? "CDP command failed")); | |
| 110 | + return; | |
| 111 | + } | |
| 112 | + pending.resolve(message.result); | |
| 113 | + }); | |
| 114 | + socket.addEventListener("close", () => { | |
| 115 | + for (const [, pending] of this.pending) { | |
| 116 | + pending.reject(new Error("CDP socket closed")); | |
| 117 | + } | |
| 118 | + this.pending.clear(); | |
| 119 | + }); | |
| 120 | + } | |
| 121 | + | |
| 122 | + static connect = (url: string): Promise<CdpPage> => | |
| 123 | + new Promise((resolveConnect, rejectConnect) => { | |
| 124 | + const socket = new WebSocket(url); | |
| 125 | + const timer = setTimeout(() => { | |
| 126 | + socket.close(); | |
| 127 | + rejectConnect(new Error(`Timed out connecting to page CDP target ${url}`)); | |
| 128 | + }, 30_000); | |
| 129 | + socket.addEventListener( | |
| 130 | + "open", | |
| 131 | + () => { | |
| 132 | + clearTimeout(timer); | |
| 133 | + resolveConnect(new CdpPage(socket)); | |
| 134 | + }, | |
| 135 | + { once: true }, | |
| 136 | + ); | |
| 137 | + socket.addEventListener( | |
| 138 | + "error", | |
| 139 | + () => { | |
| 140 | + clearTimeout(timer); | |
| 141 | + rejectConnect(new Error(`Failed to connect to page CDP target ${url}`)); | |
| 142 | + }, | |
| 143 | + { once: true }, | |
| 144 | + ); | |
| 145 | + }); | |
| 146 | + | |
| 147 | + command = async <T>(method: string, params: Record<string, unknown> = {}): Promise<T> => { | |
| 148 | + const id = this.nextId; | |
| 149 | + this.nextId += 1; | |
| 150 | + const result = new Promise<T>((resolveCommand, rejectCommand) => { | |
| 151 | + this.pending.set(id, { | |
| 152 | + resolve: (value) => resolveCommand(value as T), | |
| 153 | + reject: rejectCommand, | |
| 154 | + }); | |
| 155 | + }); | |
| 156 | + this.socket.send(JSON.stringify({ id, method, params })); | |
| 157 | + return result; | |
| 158 | + }; | |
| 159 | + | |
| 160 | + evaluate = async <T>(expression: string): Promise<T> => { | |
| 161 | + const result = await this.command<CdpEvaluateResult>("Runtime.evaluate", { | |
| 162 | + expression, | |
| 163 | + awaitPromise: true, | |
| 164 | + returnByValue: true, | |
| 165 | + }); | |
| 166 | + if (result.exceptionDetails) { | |
| 167 | + throw new Error(`CDP evaluation failed: ${JSON.stringify(result.exceptionDetails)}`); | |
| 168 | + } | |
| 169 | + return result.result.value as T; | |
| 170 | + }; | |
| 171 | + | |
| 172 | + waitForText = async (text: string, timeoutMs: number): Promise<void> => { | |
| 173 | + const deadline = Date.now() + timeoutMs; | |
| 174 | + const expression = `document.body?.innerText.includes(${JSON.stringify(text)}) ?? false`; | |
| 175 | + for (;;) { | |
| 176 | + if (await this.evaluate<boolean>(expression).catch(() => false)) return; | |
| 177 | + if (Date.now() >= deadline) throw new Error(`Timed out waiting for text: ${text}`); | |
| 178 | + await new Promise((resolveWait) => setTimeout(resolveWait, 250)); | |
| 179 | + } | |
| 180 | + }; | |
| 181 | + | |
| 182 | + setViewport = async (width: number, height: number): Promise<void> => { | |
| 183 | + await this.command("Emulation.setDeviceMetricsOverride", { | |
| 184 | + width, | |
| 185 | + height, | |
| 186 | + deviceScaleFactor: 1, | |
| 187 | + mobile: false, | |
| 188 | + }); | |
| 189 | + }; | |
| 190 | + | |
| 191 | + screenshot = async (path: string): Promise<void> => { | |
| 192 | + const result = await this.command<{ readonly data: string }>("Page.captureScreenshot", { | |
| 193 | + format: "png", | |
| 194 | + fromSurface: true, | |
| 195 | + }); | |
| 196 | + writeFileSync(path, Buffer.from(result.data, "base64")); | |
| 197 | + }; | |
| 198 | + | |
| 199 | + close = (): void => { | |
| 200 | + this.socket.close(); | |
| 201 | + }; | |
| 202 | +} | |
| 203 | + | |
| 204 | +const appExe = process.env.E2E_DESKTOP_APP_EXE; | |
| 205 | +const executorBin = process.env.E2E_DESKTOP_EXECUTOR_BIN; | |
| 206 | + | |
| 207 | +const macAquaAvailable = (): boolean => { | |
| 208 | + if (process.platform !== "darwin") return false; | |
| 209 | + try { | |
| 210 | + return execFileSync("launchctl", ["managername"], { encoding: "utf8" }).trim() === "Aqua"; | |
| 211 | + } catch { | |
| 212 | + return false; | |
| 213 | + } | |
| 214 | +}; | |
| 215 | + | |
| 216 | +const packagedSingleInstanceAvailable = (): boolean => { | |
| 217 | + if (process.platform !== "darwin" || !appExe) return true; | |
| 218 | + try { | |
| 219 | + const lines = execFileSync("pgrep", ["-fl", "Executor.app/Contents/MacOS/Executor"], { | |
| 220 | + encoding: "utf8", | |
| 221 | + }) | |
| 222 | + .split("\n") | |
| 223 | + .filter(Boolean); | |
| 224 | + return !lines.some((line) => !line.includes(appExe)); | |
| 225 | + } catch { | |
| 226 | + return true; | |
| 227 | + } | |
| 228 | +}; | |
| 229 | + | |
| 230 | +const requireBundle = (): { readonly app: string; readonly executor: string } => { | |
| 231 | + if (!appExe || !executorBin) { | |
| 232 | + throw new Error( | |
| 233 | + "E2E_DESKTOP_APP_EXE / E2E_DESKTOP_EXECUTOR_BIN not set, did desktop-packaged.globalsetup run?", | |
| 234 | + ); | |
| 235 | + } | |
| 236 | + return { app: appExe, executor: executorBin }; | |
| 237 | +}; | |
| 238 | + | |
| 239 | +const currentUid = (): number => { | |
| 240 | + const getuid = (process as { readonly getuid?: () => number }).getuid; | |
| 241 | + return typeof getuid === "function" ? getuid.call(process) : 0; | |
| 242 | +}; | |
| 243 | + | |
| 244 | +const serviceTarget = (): string => `gui/${currentUid()}/${SERVICE_LABEL}`; | |
| 245 | +const launchAgentPath = (): string => | |
| 246 | + join(homedir(), "Library", "LaunchAgents", `${SERVICE_LABEL}.plist`); | |
| 247 | +const isolatedDesktopSettingsDir = (home: string): string => | |
| 248 | + join(home, ".executor-desktop-settings"); | |
| 249 | + | |
| 250 | +const packagedAppEnv = (home: string): NodeJS.ProcessEnv => ({ | |
| 251 | + ...process.env, | |
| 252 | + HOME: home, | |
| 253 | + EXECUTOR_DESKTOP_SETTINGS_DIR: isolatedDesktopSettingsDir(home), | |
| 254 | +}); | |
| 255 | + | |
| 256 | +const cleanBootEnv = (home: string): NodeJS.ProcessEnv => ({ | |
| 257 | + ...packagedAppEnv(home), | |
| 258 | + EXECUTOR_TEST_SKIP_BACKGROUND_SERVICE: "1", | |
| 259 | + EXECUTOR_TEST_AUTO_CONFIRM_BACKGROUND_SERVICE: "0", | |
| 260 | +}); | |
| 261 | + | |
| 262 | +interface LaunchdServiceSnapshot { | |
| 263 | + readonly plist: string | null; | |
| 264 | + readonly wasLoaded: boolean; | |
| 265 | +} | |
| 266 | + | |
| 267 | +const launchctl = async (args: ReadonlyArray<string>): Promise<boolean> => { | |
| 268 | + try { | |
| 269 | + await execFileAsync("launchctl", [...args]); | |
| 270 | + return true; | |
| 271 | + } catch { | |
| 272 | + return false; | |
| 273 | + } | |
| 274 | +}; | |
| 275 | + | |
| 276 | +const captureLaunchdService = (): LaunchdServiceSnapshot | null => { | |
| 277 | + if (process.platform !== "darwin") return null; | |
| 278 | + const path = launchAgentPath(); | |
| 279 | + const plist = existsSync(path) ? readFileSync(path, "utf8") : null; | |
| 280 | + let wasLoaded = false; | |
| 281 | + try { | |
| 282 | + execFileSync("launchctl", ["print", serviceTarget()], { stdio: "ignore" }); | |
| 283 | + wasLoaded = true; | |
| 284 | + } catch { | |
| 285 | + wasLoaded = false; | |
| 286 | + } | |
| 287 | + return { plist, wasLoaded }; | |
| 288 | +}; | |
| 289 | + | |
| 290 | +const restoreLaunchdService = async (snapshot: LaunchdServiceSnapshot | null): Promise<void> => { | |
| 291 | + if (!snapshot) return; | |
| 292 | + const target = serviceTarget(); | |
| 293 | + await launchctl(["bootout", target]); | |
| 294 | + const path = launchAgentPath(); | |
| 295 | + if (snapshot.plist === null) { | |
| 296 | + rmSync(path, { force: true }); | |
| 297 | + return; | |
| 298 | + } | |
| 299 | + mkdirSync(dirname(path), { recursive: true }); | |
| 300 | + writeFileSync(path, snapshot.plist, { mode: 0o600 }); | |
| 301 | + chmodSync(path, 0o600); | |
| 302 | + await launchctl(["enable", target]); | |
| 303 | + if (snapshot.wasLoaded) { | |
| 304 | + const bootstrapped = await launchctl(["bootstrap", `gui/${currentUid()}`, path]); | |
| 305 | + if (bootstrapped) await launchctl(["kickstart", "-k", target]); | |
| 306 | + } | |
| 307 | +}; | |
| 308 | + | |
| 309 | +const waitForPageWebSocket = async (debugPort: string): Promise<string> => { | |
| 310 | + const deadline = Date.now() + 120_000; | |
| 311 | + for (;;) { | |
| 312 | + const targets = (await fetch(`http://127.0.0.1:${debugPort}/json/list`) | |
| 313 | + .then((response) => (response.ok ? response.json() : [])) | |
| 314 | + .catch(() => [])) as ReadonlyArray<CdpTarget>; | |
| 315 | + const page = targets.find( | |
| 316 | + (target) => | |
| 317 | + target.type === "page" && | |
| 318 | + target.webSocketDebuggerUrl && | |
| 319 | + !target.url.startsWith("devtools://"), | |
| 320 | + ); | |
| 321 | + if (page?.webSocketDebuggerUrl) return page.webSocketDebuggerUrl; | |
| 322 | + if (Date.now() >= deadline) { | |
| 323 | + throw new Error("Timed out waiting for packaged app page CDP target"); | |
| 324 | + } | |
| 325 | + await new Promise((resolveWait) => setTimeout(resolveWait, 100)); | |
| 326 | + } | |
| 327 | +}; | |
| 328 | + | |
| 329 | +const launchPackaged = async ( | |
| 330 | + env: NodeJS.ProcessEnv, | |
| 331 | + recorder: BootRecorder | null, | |
| 332 | +): Promise<LaunchTiming> => { | |
| 333 | + const { app } = requireBundle(); | |
| 334 | + const startedAt = Date.now(); | |
| 335 | + let output = ""; | |
| 336 | + let settled = false; | |
| 337 | + const child = spawn(app, ["--remote-debugging-port=0"], { | |
| 338 | + env, | |
| 339 | + stdio: ["ignore", "pipe", "pipe"], | |
| 340 | + }); | |
| 341 | + recorder?.start(startedAt); | |
| 342 | + | |
| 343 | + try { | |
| 344 | + const browserCdpUrl = await new Promise<string>((resolveLaunch, rejectLaunch) => { | |
| 345 | + const timer = setTimeout(() => { | |
| 346 | + rejectLaunch(new Error(`Timed out waiting for packaged app CDP URL\n${output}`)); | |
| 347 | + }, 120_000); | |
| 348 | + const settle = (fn: () => void) => { | |
| 349 | + if (settled) return; | |
| 350 | + settled = true; | |
| 351 | + clearTimeout(timer); | |
| 352 | + fn(); | |
| 353 | + }; | |
| 354 | + const collectOutput = (chunk: Buffer) => { | |
| 355 | + const text = chunk.toString(); | |
| 356 | + output = (output + text).slice(-16_384); | |
| 357 | + const match = output.match(/DevTools listening on (ws:\/\/[^\s]+)/); | |
| 358 | + if (match) settle(() => resolveLaunch(match[1])); | |
| 359 | + }; | |
| 360 | + child.stdout?.on("data", collectOutput); | |
| 361 | + child.stderr?.on("data", collectOutput); | |
| 362 | + child.once("error", (error) => settle(() => rejectLaunch(error))); | |
| 363 | + child.once("exit", (code, signal) => | |
| 364 | + settle(() => | |
| 365 | + rejectLaunch( | |
| 366 | + new Error(`Packaged app exited before CDP (code=${code} signal=${signal})\n${output}`), | |
| 367 | + ), | |
| 368 | + ), | |
| 369 | + ); | |
| 370 | + }); | |
| 371 | + | |
| 372 | + const debugPort = new URL(browserCdpUrl).port; | |
| 373 | + const pageCdpUrl = await waitForPageWebSocket(debugPort); | |
| 374 | + const cdp = await CdpPage.connect(pageCdpUrl); | |
| 375 | + await cdp.command("Runtime.enable"); | |
| 376 | + await cdp.command("Page.enable"); | |
| 377 | + await cdp.setViewport(RECORDING_WIDTH, RECORDING_HEIGHT); | |
| 378 | + recorder?.attachPage(cdp); | |
| 379 | + return { | |
| 380 | + app: { child, cdp, debugPort, output: () => output }, | |
| 381 | + pageCdpMs: Date.now() - startedAt, | |
| 382 | + }; | |
| 383 | + } catch (error) { | |
| 384 | + await stopProcess(child); | |
| 385 | + throw error; | |
| 386 | + } | |
| 387 | +}; | |
| 388 | + | |
| 389 | +const stopProcess = async (child: ChildProcess | undefined): Promise<void> => { | |
| 390 | + if (!child || child.exitCode !== null || child.signalCode !== null) return; | |
| 391 | + await new Promise<void>((resolveStop) => { | |
| 392 | + const timeout = setTimeout(() => { | |
| 393 | + child.kill("SIGKILL"); | |
| 394 | + resolveStop(); | |
| 395 | + }, 5_000); | |
| 396 | + child.once("exit", () => { | |
| 397 | + clearTimeout(timeout); | |
| 398 | + resolveStop(); | |
| 399 | + }); | |
| 400 | + child.kill("SIGTERM"); | |
| 401 | + }); | |
| 402 | +}; | |
| 403 | + | |
| 404 | +const closePackaged = async (app: PackagedApp | undefined): Promise<void> => { | |
| 405 | + app?.cdp.close(); | |
| 406 | + await stopProcess(app?.child); | |
| 407 | +}; | |
| 408 | + | |
| 409 | +const xmlEscape = (value: string): string => | |
| 410 | + value | |
| 411 | + .replaceAll("&", "&") | |
| 412 | + .replaceAll('"', """) | |
| 413 | + .replaceAll(">", ">") | |
| 414 | + .replaceAll("<", "<"); | |
| 415 | + | |
| 416 | +const writeStaleLaunchAgentPlist = (home: string): void => { | |
| 417 | + const { executor } = requireBundle(); | |
| 418 | + const dataDir = join(home, ".executor"); | |
| 419 | + const logs = join(dataDir, "logs"); | |
| 420 | + const plistDir = join(home, "Library", "LaunchAgents"); | |
| 421 | + mkdirSync(plistDir, { recursive: true }); | |
| 422 | + mkdirSync(logs, { recursive: true }); | |
| 423 | + mkdirSync(dataDir, { recursive: true }); | |
| 424 | + | |
| 425 | + const programArguments = [ | |
| 426 | + executor, | |
| 427 | + "daemon", | |
| 428 | + "run", | |
| 429 | + "--foreground", | |
| 430 | + "--port", | |
| 431 | + String(STALE_PLIST_PORT), | |
| 432 | + "--hostname", | |
| 433 | + "127.0.0.1", | |
| 434 | + ]; | |
| 435 | + const programArgs = programArguments | |
| 436 | + .map((arg) => ` <string>${xmlEscape(arg)}</string>`) | |
| 437 | + .join("\n"); | |
| 438 | + const plist = `<?xml version="1.0" encoding="UTF-8"?> | |
| 439 | +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
| 440 | +<plist version="1.0"> | |
| 441 | +<dict> | |
| 442 | + <key>Label</key> | |
| 443 | + <string>${SERVICE_LABEL}</string> | |
| 444 | + <key>ProgramArguments</key> | |
| 445 | + <array> | |
| 446 | +${programArgs} | |
| 447 | + </array> | |
| 448 | + <key>EnvironmentVariables</key> | |
| 449 | + <dict> | |
| 450 | + <key>EXECUTOR_SUPERVISED</key> | |
| 451 | + <string>1</string> | |
| 452 | + <key>EXECUTOR_DATA_DIR</key> | |
| 453 | + <string>${xmlEscape(dataDir)}</string> | |
| 454 | + <key>EXECUTOR_SCOPE_DIR</key> | |
| 455 | + <string>${xmlEscape(dataDir)}</string> | |
| 456 | + <key>EXECUTOR_CLIENT</key> | |
| 457 | + <string>desktop</string> | |
| 458 | + </dict> | |
| 459 | + <key>RunAtLoad</key> | |
| 460 | + <true/> | |
| 461 | + <key>KeepAlive</key> | |
| 462 | + <dict> | |
| 463 | + <key>SuccessfulExit</key> | |
| 464 | + <false/> | |
| 465 | + </dict> | |
| 466 | + <key>ProcessType</key> | |
| 467 | + <string>Background</string> | |
| 468 | + <key>WorkingDirectory</key> | |
| 469 | + <string>${xmlEscape(dataDir)}</string> | |
| 470 | + <key>StandardOutPath</key> | |
| 471 | + <string>${xmlEscape(join(logs, "daemon.log"))}</string> | |
| 472 | + <key>StandardErrorPath</key> | |
| 473 | + <string>${xmlEscape(join(logs, "daemon.error.log"))}</string> | |
| 474 | +</dict> | |
| 475 | +</plist> | |
| 476 | +`; | |
| 477 | + writeFileSync(join(plistDir, `${SERVICE_LABEL}.plist`), plist, { mode: 0o600 }); | |
| 478 | +}; | |
| 479 | + | |
| 480 | +const readServerManifest = (home: string): ServerManifest => { | |
| 481 | + const manifestPath = join(home, ".executor", "server-control", "server.json"); | |
| 482 | + return JSON.parse(readFileSync(manifestPath, "utf8")) as ServerManifest; | |
| 483 | +}; | |
| 484 | + | |
| 485 | +const readKickstartFailureLine = (home: string): string | null => { | |
| 486 | + const logPath = join(home, "Library", "Logs", "Executor", "main.log"); | |
| 487 | + if (!existsSync(logPath)) return null; | |
| 488 | + return ( | |
| 489 | + readFileSync(logPath, "utf8") | |
| 490 | + .split("\n") | |
| 491 | + .find( | |
| 492 | + (line) => | |
| 493 | + line.includes("launchctl kickstart failed (exit 113)") && | |
| 494 | + line.includes('Could not find service "sh.executor.daemon"'), | |
| 495 | + ) ?? null | |
| 496 | + ); | |
| 497 | +}; | |
| 498 | + | |
| 499 | +const waitForKickstartFailureLine = async (home: string): Promise<string> => { | |
| 500 | + const deadline = Date.now() + 30_000; | |
| 501 | + for (;;) { | |
| 502 | + const line = readKickstartFailureLine(home); | |
| 503 | + if (line) return line; | |
| 504 | + if (Date.now() >= deadline) { | |
| 505 | + const logPath = join(home, "Library", "Logs", "Executor", "main.log"); | |
| 506 | + const tail = existsSync(logPath) | |
| 507 | + ? readFileSync(logPath, "utf8").split("\n").slice(-20).join("\n") | |
| 508 | + : "<main.log missing>"; | |
| 509 | + throw new Error(`Timed out waiting for kickstart failure log line\n${tail}`); | |
| 510 | + } | |
| 511 | + await new Promise((resolveWait) => setTimeout(resolveWait, 250)); | |
| 512 | + } | |
| 513 | +}; | |
| 514 | + | |
| 515 | +const bootPackagedApp = async (input: { | |
| 516 | + readonly home: string; | |
| 517 | + readonly env: NodeJS.ProcessEnv; | |
| 518 | + readonly recorder: BootRecorder | null; | |
| 519 | + readonly expectKickstartFailure: boolean; | |
| 520 | +}): Promise<BootResult> => { | |
| 521 | + let app: PackagedApp | undefined; | |
| 522 | + const startedAt = Date.now(); | |
| 523 | + try { | |
| 524 | + const launched = await launchPackaged(input.env, input.recorder); | |
| 525 | + app = launched.app; | |
| 526 | + const page = app.cdp; | |
| 527 | + await page.waitForText("Settings", 120_000); | |
| 528 | + const sidecarReadyMs = Date.now() - startedAt; | |
| 529 | + await input.recorder?.captureReadyFrame(); | |
| 530 | + await input.recorder?.captureReadyFrame(); | |
| 531 | + const connection = await page.evaluate<{ | |
| 532 | + readonly kind: string; | |
| 533 | + readonly origin: string; | |
| 534 | + } | null>("window.executor.getServerConnection()"); | |
| 535 | + expect(connection, "the app eventually exposes a server connection").not.toBeNull(); | |
| 536 | + const manifest = readServerManifest(input.home); | |
| 537 | + const matchedLogLine = input.expectKickstartFailure | |
| 538 | + ? await waitForKickstartFailureLine(input.home) | |
| 539 | + : readKickstartFailureLine(input.home); | |
| 540 | + return { | |
| 541 | + pageCdpMs: launched.pageCdpMs, | |
| 542 | + sidecarReadyMs, | |
| 543 | + connectionKind: connection!.kind, | |
| 544 | + origin: connection!.origin, | |
| 545 | + manifestKind: manifest.kind, | |
| 546 | + manifestPid: manifest.pid, | |
| 547 | + matchedLogLine, | |
| 548 | + }; | |
| 549 | + } finally { | |
| 550 | + await input.recorder?.stop(); | |
| 551 | + await closePackaged(app); | |
| 552 | + } | |
| 553 | +}; | |
| 554 | + | |
| 555 | +const commandAvailable = (command: string): boolean => { | |
| 556 | + try { | |
| 557 | + execFileSync("which", [command], { stdio: "ignore" }); | |
| 558 | + return true; | |
| 559 | + } catch { | |
| 560 | + return false; | |
| 561 | + } | |
| 562 | +}; | |
| 563 | + | |
| 564 | +const emitEvidence = (line: string): void => { | |
| 565 | + execFileSync("printf", ["%s\n", line], { stdio: "inherit" }); | |
| 566 | +}; | |
| 567 | + | |
| 568 | +const FONT: Record<string, readonly string[]> = { | |
| 569 | + " ": ["00000", "00000", "00000", "00000", "00000", "00000", "00000"], | |
| 570 | + ".": ["00000", "00000", "00000", "00000", "00000", "01100", "01100"], | |
| 571 | + "0": ["11110", "10010", "10010", "10010", "10010", "10010", "11110"], | |
| 572 | + "1": ["00100", "01100", "00100", "00100", "00100", "00100", "01110"], | |
| 573 | + "2": ["11110", "00010", "00010", "11110", "10000", "10000", "11110"], | |
| 574 | + "3": ["11110", "00010", "00010", "01110", "00010", "00010", "11110"], | |
| 575 | + "4": ["10010", "10010", "10010", "11110", "00010", "00010", "00010"], | |
| 576 | + "5": ["11110", "10000", "10000", "11110", "00010", "00010", "11110"], | |
| 577 | + "6": ["11110", "10000", "10000", "11110", "10010", "10010", "11110"], | |
| 578 | + "7": ["11110", "00010", "00010", "00100", "00100", "01000", "01000"], | |
| 579 | + "8": ["11110", "10010", "10010", "11110", "10010", "10010", "11110"], | |
| 580 | + "9": ["11110", "10010", "10010", "11110", "00010", "00010", "11110"], | |
| 581 | + A: ["01110", "10001", "10001", "11111", "10001", "10001", "10001"], | |
| 582 | + B: ["11110", "10001", "10001", "11110", "10001", "10001", "11110"], | |
| 583 | + C: ["01111", "10000", "10000", "10000", "10000", "10000", "01111"], | |
| 584 | + D: ["11110", "10001", "10001", "10001", "10001", "10001", "11110"], | |
| 585 | + E: ["11111", "10000", "10000", "11110", "10000", "10000", "11111"], | |
| 586 | + F: ["11111", "10000", "10000", "11110", "10000", "10000", "10000"], | |
| 587 | + G: ["01111", "10000", "10000", "10011", "10001", "10001", "01111"], | |
| 588 | + H: ["10001", "10001", "10001", "11111", "10001", "10001", "10001"], | |
| 589 | + I: ["11111", "00100", "00100", "00100", "00100", "00100", "11111"], | |
| 590 | + J: ["00111", "00010", "00010", "00010", "10010", "10010", "01100"], | |
| 591 | + K: ["10001", "10010", "10100", "11000", "10100", "10010", "10001"], | |
| 592 | + L: ["10000", "10000", "10000", "10000", "10000", "10000", "11111"], | |
| 593 | + M: ["10001", "11011", "10101", "10101", "10001", "10001", "10001"], | |
| 594 | + N: ["10001", "11001", "10101", "10011", "10001", "10001", "10001"], | |
| 595 | + O: ["01110", "10001", "10001", "10001", "10001", "10001", "01110"], | |
| 596 | + P: ["11110", "10001", "10001", "11110", "10000", "10000", "10000"], | |
| 597 | + Q: ["01110", "10001", "10001", "10001", "10101", "10010", "01101"], | |
| 598 | + R: ["11110", "10001", "10001", "11110", "10100", "10010", "10001"], | |
| 599 | + S: ["01111", "10000", "10000", "01110", "00001", "00001", "11110"], | |
| 600 | + T: ["11111", "00100", "00100", "00100", "00100", "00100", "00100"], | |
| 601 | + U: ["10001", "10001", "10001", "10001", "10001", "10001", "01110"], | |
| 602 | + V: ["10001", "10001", "10001", "10001", "10001", "01010", "00100"], | |
| 603 | + W: ["10001", "10001", "10001", "10101", "10101", "10101", "01010"], | |
| 604 | + X: ["10001", "10001", "01010", "00100", "01010", "10001", "10001"], | |
| 605 | + Y: ["10001", "10001", "01010", "00100", "00100", "00100", "00100"], | |
| 606 | + Z: ["11111", "00001", "00010", "00100", "01000", "10000", "11111"], | |
| 607 | +}; | |
| 608 | + | |
| 609 | +const putPixel = (pixels: Buffer, width: number, x: number, y: number, rgb: readonly number[]) => { | |
| 610 | + const index = (y * width + x) * 3; | |
| 611 | + pixels[index] = rgb[0]; | |
| 612 | + pixels[index + 1] = rgb[1]; | |
| 613 | + pixels[index + 2] = rgb[2]; | |
| 614 | +}; | |
| 615 | + | |
| 616 | +const fillRect = ( | |
| 617 | + pixels: Buffer, | |
| 618 | + width: number, | |
| 619 | + x: number, | |
| 620 | + y: number, | |
| 621 | + rectWidth: number, | |
| 622 | + rectHeight: number, | |
| 623 | + rgb: readonly number[], | |
| 624 | +): void => { | |
| 625 | + for (let yy = y; yy < y + rectHeight; yy += 1) { | |
| 626 | + for (let xx = x; xx < x + rectWidth; xx += 1) { | |
| 627 | + putPixel(pixels, width, xx, yy, rgb); | |
| 628 | + } | |
| 629 | + } | |
| 630 | +}; | |
| 631 | + | |
| 632 | +const writePpm = (path: string, width: number, height: number, pixels: Buffer): void => { | |
| 633 | + writeFileSync(path, Buffer.concat([Buffer.from(`P6\n${width} ${height}\n255\n`), pixels])); | |
| 634 | +}; | |
| 635 | + | |
| 636 | +const makeLabelPpm = (path: string, label: string, height: number): void => { | |
| 637 | + const pixels = Buffer.alloc(RECORDING_WIDTH * height * 3); | |
| 638 | + fillRect(pixels, RECORDING_WIDTH, 0, 0, RECORDING_WIDTH, height, [17, 18, 23]); | |
| 639 | + fillRect(pixels, RECORDING_WIDTH, 0, 0, RECORDING_WIDTH, 110, [0, 0, 0]); | |
| 640 | + const scale = 6; | |
| 641 | + let x = 34; | |
| 642 | + const y = 34; | |
| 643 | + for (const char of label.toUpperCase()) { | |
| 644 | + const glyph = FONT[char] ?? FONT[" "]; | |
| 645 | + for (let row = 0; row < glyph.length; row += 1) { | |
| 646 | + for (let col = 0; col < glyph[row].length; col += 1) { | |
| 647 | + if (glyph[row][col] === "1") { | |
| 648 | + fillRect( | |
| 649 | + pixels, | |
| 650 | + RECORDING_WIDTH, | |
| 651 | + x + col * scale, | |
| 652 | + y + row * scale, | |
| 653 | + scale, | |
| 654 | + scale, | |
| 655 | + [255, 255, 255], | |
| 656 | + ); | |
| 657 | + } | |
| 658 | + } | |
| 659 | + } | |
| 660 | + x += 6 * scale; | |
| 661 | + if (x > RECORDING_WIDTH - 40) break; | |
| 662 | + } | |
| 663 | + writePpm(path, RECORDING_WIDTH, height, pixels); | |
| 664 | +}; | |
| 665 | + | |
| 666 | +class BootRecorder { | |
| 667 | + private frame = 0; | |
| 668 | + private page: CdpPage | null = null; | |
| 669 | + private startedAt = 0; | |
| 670 | + private timer: ReturnType<typeof setInterval> | null = null; | |
| 671 | + private queue: Promise<void> = Promise.resolve(); | |
| 672 | + private enabled: boolean; | |
| 673 | + | |
| 674 | + constructor( | |
| 675 | + private readonly frameDir: string, | |
| 676 | + private readonly outputPath: string, | |
| 677 | + ) { | |
| 678 | + this.enabled = process.env.E2E_RECORD === "1" && commandAvailable("ffmpeg"); | |
| 679 | + } | |
| 680 | + | |
| 681 | + static create = (runDir: string): BootRecorder | null => { | |
| 682 | + if (process.env.E2E_RECORD !== "1") return null; | |
| 683 | + const frameDir = join(runDir, "stale-launchd-slow-boot-frames"); | |
| 684 | + rmSync(frameDir, { recursive: true, force: true }); | |
| 685 | + mkdirSync(frameDir, { recursive: true }); | |
| 686 | + mkdirSync(dirname(RECORDING_FILE), { recursive: true }); | |
| 687 | + rmSync(RECORDING_FILE, { force: true }); | |
| 688 | + return new BootRecorder(frameDir, RECORDING_FILE); | |
| 689 | + }; | |
| 690 | + | |
| 691 | + start = (startedAt: number): void => { | |
| 692 | + this.startedAt = startedAt; | |
| 693 | + if (!this.enabled) return; | |
| 694 | + this.enqueueFrame("waiting for Electron window", null); | |
| 695 | + this.timer = setInterval(() => { | |
| 696 | + this.enqueueFrame( | |
| 697 | + this.page ? "app window reachable" : "waiting for Electron window", | |
| 698 | + this.page, | |
| 699 | + ); | |
| 700 | + }, 1_000); | |
| 701 | + }; | |
| 702 | + | |
| 703 | + attachPage = (page: CdpPage): void => { | |
| 704 | + this.page = page; | |
| 705 | + this.enqueueFrame("app window reachable", page); | |
| 706 | + }; | |
| 707 | + | |
| 708 | + captureReadyFrame = async (): Promise<void> => { | |
| 709 | + if (!this.enabled) return; | |
| 710 | + this.enqueueFrame("managed sidecar ready", this.page); | |
| 711 | + await this.queue; | |
| 712 | + }; | |
| 713 | + | |
| 714 | + stop = async (): Promise<void> => { | |
| 715 | + if (this.timer) { | |
| 716 | + clearInterval(this.timer); | |
| 717 | + this.timer = null; | |
| 718 | + } | |
| 719 | + if (!this.enabled) return; | |
| 720 | + await this.queue; | |
| 721 | + if (this.frame > 0) { | |
| 722 | + execFileSync( | |
| 723 | + "ffmpeg", | |
| 724 | + [ | |
| 725 | + "-y", | |
| 726 | + "-framerate", | |
| 727 | + "1", | |
| 728 | + "-i", | |
| 729 | + join(this.frameDir, "frame-%04d.ppm"), | |
| 730 | + "-pix_fmt", | |
| 731 | + "yuv420p", | |
| 732 | + "-movflags", | |
| 733 | + "+faststart", | |
| 734 | + this.outputPath, | |
| 735 | + ], | |
| 736 | + { stdio: "ignore" }, | |
| 737 | + ); | |
| 738 | + } | |
| 739 | + }; | |
| 740 | + | |
| 741 | + recordingStats = (): { | |
| 742 | + readonly path: string; | |
| 743 | + readonly size: number; | |
| 744 | + readonly frames: number; | |
| 745 | + } | null => { | |
| 746 | + if (!existsSync(this.outputPath)) return null; | |
| 747 | + return { path: this.outputPath, size: statSync(this.outputPath).size, frames: this.frame }; | |
| 748 | + }; | |
| 749 | + | |
| 750 | + private enqueueFrame = (state: string, page: CdpPage | null): void => { | |
| 751 | + if (!this.enabled) return; | |
| 752 | + this.queue = this.queue | |
| 753 | + .then(() => this.writeFrame(state, page)) | |
| 754 | + .catch((error) => { | |
| 755 | + this.enabled = false; | |
| 756 | + console.warn(`[stale-launchd-repro] recording disabled after frame error: ${error}`); | |
| 757 | + }); | |
| 758 | + }; | |
| 759 | + | |
| 760 | + private writeFrame = async (state: string, page: CdpPage | null): Promise<void> => { | |
| 761 | + this.frame += 1; | |
| 762 | + const elapsedSeconds = (Date.now() - this.startedAt) / 1000; | |
| 763 | + const output = join(this.frameDir, `frame-${String(this.frame).padStart(4, "0")}.ppm`); | |
| 764 | + const label = `${elapsedSeconds.toFixed(1)}s ${state}`; | |
| 765 | + if (!page) { | |
| 766 | + makeLabelPpm(output, label, RECORDING_HEIGHT); | |
| 767 | + return; | |
| 768 | + } | |
| 769 | + const raw = join(this.frameDir, `raw-${String(this.frame).padStart(4, "0")}.png`); | |
| 770 | + const labelFrame = join(this.frameDir, `label-${String(this.frame).padStart(4, "0")}.ppm`); | |
| 771 | + await page.screenshot(raw); | |
| 772 | + makeLabelPpm(labelFrame, label, 110); | |
| 773 | + execFileSync( | |
| 774 | + "ffmpeg", | |
| 775 | + [ | |
| 776 | + "-y", | |
| 777 | + "-i", | |
| 778 | + raw, | |
| 779 | + "-i", | |
| 780 | + labelFrame, | |
| 781 | + "-filter_complex", | |
| 782 | + `[0:v]scale=${RECORDING_WIDTH}:${RECORDING_HEIGHT}:force_original_aspect_ratio=decrease,pad=${RECORDING_WIDTH}:${RECORDING_HEIGHT}:(ow-iw)/2:(oh-ih)/2:color=black[base];[base][1:v]overlay=0:0`, | |
| 783 | + "-frames:v", | |
| 784 | + "1", | |
| 785 | + output, | |
| 786 | + ], | |
| 787 | + { stdio: "ignore" }, | |
| 788 | + ); | |
| 789 | + rmSync(raw, { force: true }); | |
| 790 | + rmSync(labelFrame, { force: true }); | |
| 791 | + }; | |
| 792 | +} | |
| 793 | + | |
| 794 | +const run = async (runDir: string): Promise<void> => { | |
| 795 | + const cleanHome = mkdtempSync(join(tmpdir(), "executor-pkg-clean-boot-control-")); | |
| 796 | + const staleHome = mkdtempSync(join(tmpdir(), "executor-pkg-stale-launchd-")); | |
| 797 | + const launchdSnapshot = captureLaunchdService(); | |
| 798 | + const staleRecorder = BootRecorder.create(runDir); | |
| 799 | + | |
| 800 | + try { | |
| 801 | + const clean = await bootPackagedApp({ | |
| 802 | + home: cleanHome, | |
| 803 | + env: cleanBootEnv(cleanHome), | |
| 804 | + recorder: null, | |
| 805 | + expectKickstartFailure: false, | |
| 806 | + }); | |
| 807 | + | |
| 808 | + writeStaleLaunchAgentPlist(staleHome); | |
| 809 | + await launchctl(["bootout", serviceTarget()]); | |
| 810 | + const stale = await bootPackagedApp({ | |
| 811 | + home: staleHome, | |
| 812 | + env: packagedAppEnv(staleHome), | |
| 813 | + recorder: staleRecorder, | |
| 814 | + expectKickstartFailure: true, | |
| 815 | + }); | |
| 816 | + | |
| 817 | + /* | |
| 818 | + * Regression pin for the current broken behavior: | |
| 819 | + * a stale plist makes `service status` report registered even though launchd | |
| 820 | + * cannot find the label. The packaged app logs the kickstart failure, then | |
| 821 | + * still pays the full 15s wait before falling back to managed-spawn. | |
| 822 | + * | |
| 823 | + * When the fix lands, invert the >=14s stale assertions. The desired | |
| 824 | + * behavior is a fast-fail to managed-spawn, close to the clean control. | |
| 825 | + */ | |
| 826 | + expect(stale.matchedLogLine, "stale boot logs the launchd kickstart failure").toContain( | |
| 827 | + "launchctl kickstart failed (exit 113)", | |
| 828 | + ); | |
| 829 | + expect(stale.matchedLogLine, "stale boot says launchd cannot find the label").toContain( | |
| 830 | + 'Could not find service "sh.executor.daemon"', | |
| 831 | + ); | |
| 832 | + expect( | |
| 833 | + stale.pageCdpMs, | |
| 834 | + "window/CDP is not reachable until the dead wait is paid", | |
| 835 | + ).toBeGreaterThanOrEqual(14_000); | |
| 836 | + expect( | |
| 837 | + stale.sidecarReadyMs, | |
| 838 | + "managed sidecar readiness is delayed by the dead wait", | |
| 839 | + ).toBeGreaterThanOrEqual(14_000); | |
| 840 | + expect(stale.connectionKind, "the app eventually recovers through managed-spawn").toBe( | |
| 841 | + "desktop-sidecar", | |
| 842 | + ); | |
| 843 | + expect( | |
| 844 | + new URL(stale.origin).port, | |
| 845 | + "managed recovery must not be a launchd attach to the stale plist port", | |
| 846 | + ).not.toBe(String(STALE_PLIST_PORT)); | |
| 847 | + expect( | |
| 848 | + clean.sidecarReadyMs, | |
| 849 | + "the clean control skips supervision and boots quickly", | |
| 850 | + ).toBeLessThan(10_000); | |
| 851 | + expect( | |
| 852 | + stale.sidecarReadyMs, | |
| 853 | + "the stale launchd path is dramatically slower than the clean control", | |
| 854 | + ).toBeGreaterThan(clean.sidecarReadyMs + 5_000); | |
| 855 | + | |
| 856 | + const recording = staleRecorder?.recordingStats(); | |
| 857 | + emitEvidence( | |
| 858 | + `[stale-launchd-repro] clean=${clean.sidecarReadyMs}ms stale=${stale.sidecarReadyMs}ms stalePageCdp=${stale.pageCdpMs}ms cleanOrigin=${clean.origin} staleOrigin=${stale.origin} staleConnectionKind=${stale.connectionKind} staleManifestKind=${stale.manifestKind} stalePid=${stale.manifestPid}`, | |
| 859 | + ); | |
| 860 | + emitEvidence(`[stale-launchd-repro] matchedLogLine=${stale.matchedLogLine}`); | |
| 861 | + if (recording) { | |
| 862 | + emitEvidence( | |
| 863 | + `[stale-launchd-repro] recording=${recording.path} frames=${recording.frames} size=${recording.size}`, | |
| 864 | + ); | |
| 865 | + } | |
| 866 | + } finally { | |
| 867 | + await restoreLaunchdService(launchdSnapshot); | |
| 868 | + rmSync(cleanHome, { recursive: true, force: true }); | |
| 869 | + rmSync(staleHome, { recursive: true, force: true }); | |
| 870 | + } | |
| 871 | +}; | |
| 872 | + | |
| 873 | +if (!macAquaAvailable() || !packagedSingleInstanceAvailable()) { | |
| 874 | + it.skip(`${SCENARIO_NAME} (needs macOS Aqua and no already-running Executor.app)`, () => {}); | |
| 875 | +} else { | |
| 876 | + scenario( | |
| 877 | + SCENARIO_NAME, | |
| 878 | + { timeout: 360_000 }, | |
| 879 | + Effect.gen(function* () { | |
| 880 | + requireBundle(); | |
| 881 | + const runDir = yield* RunDir; | |
| 882 | + yield* Effect.promise(() => run(runDir)); | |
| 883 | + }), | |
| 884 | + ); | |
| 885 | +} | |
Binary file not shown.