History

Desktop boot: show window immediately, self-heal stale launchd state, fix boot-path latency (#1303)

* Desktop boot: show a window immediately and self-heal stale launchd state Two boot-time fixes in the desktop main process: Show a lightweight "Starting Executor" window at app.whenReady, before any service or sidecar work, then load the real web UI into the same window once the connection resolves (no second-window flash). A pending second-instance now focuses that startup window instead of no-opping while connection is null. Classify the supervised service by both registered and running instead of ignoring the running signal: - registered and running: attach; only kick if the attach still misses it. - registered but not running (the stale plist/unit/task case): restart, and if the OS manager cannot start it, reinstall the service (install is already idempotent) before waiting. If install also fails, fall back to managed spawn immediately. The 15s attach wait is only paid after a kick or install actually succeeds, so a stale registration no longer costs a dead wait. * Desktop boot: keep login-shell PATH capture off the read path captureUserPath shells out to $SHELL -ilc on every bundled-executor call, including read-only `service status` that boot runs first. Skip the capture for `service status` (process env PATH is enough for a read) and memoize it for the rest of the process lifetime so install/restart pay the shell probe at most once instead of on every call. * e2e: invert the stale launchd repro to guard the fast, self-healing boot The repro pinned the broken behavior (stale plist forces a >=14s dead wait). Flip it to guard the fix: the startup window must appear within a few seconds, the kickstart failure must be followed by a self-heal-or-fallback log line rather than a dead wait, boot-to-ready must stay under the fast budget and close to the clean control, and recovery must either repair launchd or fall back cleanly to managed spawn without attaching to the stale plist port. The E2E_RECORD recording now captures the fixed fast boot.

Rhys Sullivan committed 4 months ago

Commit a968cc51 parent 7f618056 files · +340 −108Browse files

Unified · Split

6 files
  1. apps/desktop/src/main/crash-screen.ts +98 −0
  2. apps/desktop/src/main/index.ts +101 −45
  3. apps/desktop/src/main/service.ts +29 −11
  4. e2e/desktop-packaged/stale-launchd-registration.test.ts +114 −54
  5. e2e/recordings/stale-launchd-fast-boot.mp4 +0 −0
  6. e2e/recordings/stale-launchd-slow-boot.mp4 +0 −0
apps/desktop/src/main/crash-screen.ts+98−0
@@ -11,6 +11,104 @@ export interface CrashScreenOptions { · 10 unmodified lines
11 readonly reported: boolean;11 readonly reported: boolean;
12}12}
13 13
14export const startupWindowHtml = (): string => `<!doctype html>
15<html>
16 <head>
17 <meta charset="utf-8" />
18 <title>Executor</title>
19 <style>
20 :root {
21 color-scheme: light dark;
22 --background: #0a0a0a;
23 --foreground: #fafafa;
24 --muted: #a1a1aa;
25 --line: #27272a;
26 --pulse: #fafafa;
27 }
28 @media (prefers-color-scheme: light) {
29 :root {
30 --background: #f7f7f4;
31 --foreground: #18181b;
32 --muted: #60646c;
33 --line: #d8d8d0;
34 --pulse: #18181b;
35 }
36 }
37 body {
38 margin: 0;
39 min-height: 100vh;
40 display: grid;
41 place-items: center;
42 background: var(--background);
43 color: var(--foreground);
44 font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
45 }
46 main {
47 width: min(22rem, calc(100vw - 4rem));
48 display: grid;
49 gap: 1rem;
50 justify-items: center;
51 text-align: center;
52 }
53 .mark {
54 width: 2.25rem;
55 height: 2.25rem;
56 border: 1px solid var(--line);
57 border-radius: 8px;
58 display: grid;
59 place-items: center;
60 font-size: 0.82rem;
61 font-weight: 650;
62 letter-spacing: 0;
63 }
64 h1 {
65 margin: 0;
66 font-size: 1.05rem;
67 font-weight: 600;
68 letter-spacing: 0;
69 }
70 p {
71 margin: -0.35rem 0 0;
72 color: var(--muted);
73 font-size: 0.82rem;
74 line-height: 1.5;
75 }
76 .activity {
77 width: 8rem;
78 height: 2px;
79 overflow: hidden;
80 border-radius: 999px;
81 background: var(--line);
82 }
83 .activity::after {
84 content: "";
85 display: block;
86 width: 45%;
87 height: 100%;
88 border-radius: inherit;
89 background: var(--pulse);
90 animation: slide 1.15s ease-in-out infinite;
91 }
92 @keyframes slide {
93 0% { transform: translateX(-110%); opacity: 0.35; }
94 50% { opacity: 0.9; }
95 100% { transform: translateX(245%); opacity: 0.35; }
96 }
97 @media (prefers-reduced-motion: reduce) {
98 .activity::after { animation: none; transform: none; width: 100%; opacity: 0.55; }
99 }
100 </style>
101 </head>
102 <body>
103 <main>
104 <div class="mark">Ex</div>
105 <h1>Starting Executor&hellip;</h1>
106 <p>Preparing your local workspace.</p>
107 <div class="activity" aria-hidden="true"></div>
108 </main>
109 </body>
110</html>`;
111
14export const sidecarCrashHtml = ({ reported }: CrashScreenOptions): string => `<!doctype html>112export const sidecarCrashHtml = ({ reported }: CrashScreenOptions): string => `<!doctype html>
15<html>113<html>
16 <head>114 <head>
apps/desktop/src/main/index.ts+101−45
@@ -34,7 +34,7 @@ import { · 33 unmodified lines
34 initErrorReporting,34 initErrorReporting,
35 reportAProblem,35 reportAProblem,
36} from "./diagnostics";36} from "./diagnostics";
37import { sidecarCrashHtml } from "./crash-screen";37import { sidecarCrashHtml, startupWindowHtml } from "./crash-screen";
38import { replaceSupervisedDaemonForDesktop } from "./supervised-connection";38import { replaceSupervisedDaemonForDesktop } from "./supervised-connection";
39import {39import {
40 bundledExecutorPath,40 bundledExecutorPath,
@@ -137,6 +137,9 @@ const webUrlForConnection = (conn: SidecarConnection): string => { · 96 unmodified lines
137 return url.toString();137 return url.toString();
138};138};
139 139
140const htmlDataUrl = (html: string): string =>
141 `data:text/html;charset=utf-8,${encodeURIComponent(html)}`;
142
140// The supervised daemon (and the desktop sidecar) own this data dir — the same143// The supervised daemon (and the desktop sidecar) own this data dir — the same
141// path the CLI's `executor web`/daemon uses, so desktop and CLI share state.144// path the CLI's `executor web`/daemon uses, so desktop and CLI share state.
142const DESKTOP_DATA_DIR = join(homedir(), ".executor");145const DESKTOP_DATA_DIR = join(homedir(), ".executor");
@@ -216,6 +219,40 @@ const confirmEnableBackgroundService = async (): Promise<boolean> => { · 73 unmodified lines
216 return response === 0;219 return response === 0;
217};220};
218 221
222const acceptSupervisedConnection = async (
223 attached: SidecarConnection,
224): Promise<SidecarConnection | null> => {
225 if (!shouldReplaceDaemonForDesktop(attached)) return attached;
226 const settings = getServerSettings();
227 return replaceSupervisedDaemonForDesktop(attached, {
228 install: () =>
229 installSupervisedService({
230 port: settings.port,
231 dataDir: DESKTOP_DATA_DIR,
232 }),
233 waitForAttach: () => waitForSupervisedAttach(30_000, { port: settings.port }),
234 attach: attachToSupervisedDaemon,
235 onInstallFailure: (error) => {
236 log.warn("Failed to replace older supervised daemon; re-checking before falling back", error);
237 },
238 });
239};
240
241const installSupervisedAndWait = async (reason: string): Promise<SidecarConnection | null> => {
242 const settings = getServerSettings();
243 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: install failure falls back to managed-spawn so the app still launches
244 try {
245 await installSupervisedService({
246 port: settings.port,
247 dataDir: DESKTOP_DATA_DIR,
248 });
249 } catch (error) {
250 log.warn(`Failed to install supervised service after ${reason}; using managed sidecar`, error);
251 return null;
252 }
253 return waitForSupervisedAttach(15_000, { port: settings.port });
254};
255
219/**256/**
220 * Resolve a connection to the OS-supervised daemon, installing it on first run257 * Resolve a connection to the OS-supervised daemon, installing it on first run
221 * (with consent). Returns null when supervision is unavailable or the user258 * (with consent). Returns null when supervision is unavailable or the user
@@ -225,23 +262,7 @@ const ensureSupervisedConnection = async (): Promise<SidecarConnection | null> = · 3 unmodified lines
225 // 1. Already running → attach.262 // 1. Already running → attach.
226 const attached = await attachToSupervisedDaemon();263 const attached = await attachToSupervisedDaemon();
227 if (attached) {264 if (attached) {
228 if (!shouldReplaceDaemonForDesktop(attached)) return attached;265 return acceptSupervisedConnection(attached);
229 const settings = getServerSettings();
230 return replaceSupervisedDaemonForDesktop(attached, {
231 install: () =>
232 installSupervisedService({
233 port: settings.port,
234 dataDir: DESKTOP_DATA_DIR,
235 }),
236 waitForAttach: () => waitForSupervisedAttach(30_000, { port: settings.port }),
237 attach: attachToSupervisedDaemon,
238 onInstallFailure: (error) => {
239 log.warn(
240 "Failed to replace older supervised daemon; re-checking before falling back",
241 error,
242 );
243 },
244 });
245 }266 }
246 267
247 // Headless/e2e: the first-run background-service prompt and the systemd/268 // Headless/e2e: the first-run background-service prompt and the systemd/
@@ -254,32 +275,41 @@ const ensureSupervisedConnection = async (): Promise<SidecarConnection | null> = · 6 unmodified lines
254 const status = await supervisedServiceStatus();275 const status = await supervisedServiceStatus();
255 if (!status.supported) return null;276 if (!status.supported) return null;
256 277
257 // 2. Registered but not currently serving → kick it and wait.278 // 2. Registered and running, but the first attach missed it. Kick once and
279 // wait only when the kick command succeeds.
258 if (status.registered) {280 if (status.registered) {
259 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: a restart failure just falls through to managed-spawn281 if (status.running) {
282 const attachedAfterStatus = await attachToSupervisedDaemon();
283 if (attachedAfterStatus) return acceptSupervisedConnection(attachedAfterStatus);
284 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: a restart failure just falls through to managed-spawn
285 try {
286 await restartSupervisedService();
287 } catch (error) {
288 log.warn(
289 "Failed to restart running supervised service after attach failed; using managed sidecar",
290 error,
291 );
292 return null;
293 }
294 return waitForSupervisedAttach(15_000);
295 }
296
297 // 3. Registered but not running usually means a stale plist/unit/task. Try
298 // restart first, then reinstall the service if the OS manager cannot start it.
299 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: restart failure triggers supervised service self-heal
260 try {300 try {
261 await restartSupervisedService();301 await restartSupervisedService();
262 } catch (error) {302 } catch (error) {
263 log.warn("Failed to kickstart supervised service", error);303 log.warn("Failed to restart registered supervised service; reinstalling", error);
304 return installSupervisedAndWait("registered service restart failure");
264 }305 }
265 return waitForSupervisedAttach(15_000);306 return waitForSupervisedAttach(15_000);
266 }307 }
267 308
268 // 3. First run → ask, then install + start. The unit carries no secret; the309 // 4. First run → ask, then install + start. The unit carries no secret; the
269 // supervised daemon mints/loads its bearer from auth.json under DESKTOP_DATA_DIR.310 // supervised daemon mints/loads its bearer from auth.json under DESKTOP_DATA_DIR.
270 if (!(await confirmEnableBackgroundService())) return null;311 if (!(await confirmEnableBackgroundService())) return null;
271 const settings = getServerSettings();312 return installSupervisedAndWait("first-run prompt");
272 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: install failure falls back to managed-spawn so the app still launches
273 try {
274 await installSupervisedService({
275 port: settings.port,
276 dataDir: DESKTOP_DATA_DIR,
277 });
278 } catch (error) {
279 log.error("Failed to install supervised service; using managed sidecar", error);
280 return null;
281 }
282 return waitForSupervisedAttach(15_000);
283};313};
284 314
285// Crash monitor for the supervised daemon: the OS service manager restarts it315// Crash monitor for the supervised daemon: the OS service manager restarts it
@@ -308,8 +338,7 @@ const armSupervisedMonitor = () => { · 22 unmodified lines
308 if (supervisedMonitorMisses < SUPERVISED_MONITOR_MISSES_BEFORE_DOWN) return;338 if (supervisedMonitorMisses < SUPERVISED_MONITOR_MISSES_BEFORE_DOWN) return;
309 if (!supervisedDaemonDown && window) {339 if (!supervisedDaemonDown && window) {
310 supervisedDaemonDown = true;340 supervisedDaemonDown = true;
311 const html = sidecarCrashHtml({ reported: errorReportingEnabled });341 showCrashScreen(window);
312 void window.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(html)}`);
313 }342 }
314 return;343 return;
315 }344 }
@@ -408,14 +437,12 @@ const installDockIcon = () => { · 92 unmodified lines
408 log.info(`[dock-icon] set to ${iconPath} (${image.getSize().width}×${image.getSize().height})`);437 log.info(`[dock-icon] set to ${iconPath} (${image.getSize().width}×${image.getSize().height})`);
409};438};
410 439
411const createWindow = async (conn: SidecarConnection) => {440const createMainBrowserWindow = (options: { readonly show: boolean }): BrowserWindow => {
412 const windowState = windowStateKeeper({441 const windowState = windowStateKeeper({
413 defaultWidth: 1280,442 defaultWidth: 1280,
414 defaultHeight: 800,443 defaultHeight: 800,
415 });444 });
416 445
417 installBearerAuthHeader(conn.baseUrl, conn.authToken);
418
419 const linuxIcon = resolveLinuxIcon();446 const linuxIcon = resolveLinuxIcon();
420 447
421 const window = new BrowserWindow({448 const window = new BrowserWindow({
@@ -431,7 +458,7 @@ const createWindow = async (conn: SidecarConnection) => { · 9 unmodified lines
431 // header, which is offset to clear them (.desktop-macos-titlebar).458 // header, which is offset to clear them (.desktop-macos-titlebar).
432 minWidth: 768,459 minWidth: 768,
433 minHeight: 480,460 minHeight: 480,
434 show: false,461 show: options.show,
435 backgroundColor: "#0a0a0a",462 backgroundColor: "#0a0a0a",
436 autoHideMenuBar: true,463 autoHideMenuBar: true,
437 ...(process.platform === "darwin"464 ...(process.platform === "darwin"
@@ -488,15 +515,45 @@ const createWindow = async (conn: SidecarConnection) => { · 50 unmodified lines
488 return { action: "deny" };515 return { action: "deny" };
489 });516 });
490 517
518 return window;
519};
520
521const showStartupWindow = async (): Promise<void> => {
522 const window = liveMainWindow() ?? createMainBrowserWindow({ show: true });
523 if (window.isMinimized()) window.restore();
524 if (!window.isVisible()) window.show();
525 window.focus();
526
527 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: local startup HTML should not block service startup
528 try {
529 await window.loadURL(htmlDataUrl(startupWindowHtml()));
530 } catch (error) {
531 log.warn("Failed to load startup window", error);
532 }
533};
534
535const showCrashScreen = (window: BrowserWindow | null = liveMainWindow()): void => {
536 if (!window) return;
537 const html = sidecarCrashHtml({ reported: errorReportingEnabled });
538 void window.loadURL(htmlDataUrl(html));
539};
540
541const createWindow = async (conn: SidecarConnection) => {
542 installBearerAuthHeader(conn.baseUrl, conn.authToken);
543
544 const existingWindow = liveMainWindow();
545 const window = existingWindow ?? createMainBrowserWindow({ show: false });
546
491 // A supervised daemon can pass the health probe and still disappear before547 // A supervised daemon can pass the health probe and still disappear before
492 // navigation begins. Treat that as a failed connection instead of leaving the548 // navigation begins. Treat that as a failed connection instead of leaving the
493 // user with a visible BrowserWindow that only shows the black background.549 // user with a visible BrowserWindow that only shows the black background.
494 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: Electron navigation rejects when the sidecar vanishes during startup550 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: Electron navigation rejects when the sidecar vanishes during startup
495 try {551 try {
496 await window.loadURL(webUrlForConnection(conn));552 await window.loadURL(webUrlForConnection(conn));
553 if (!window.isDestroyed() && !window.isVisible()) window.show();
497 } catch (error) {554 } catch (error) {
498 log.error("Failed to load Executor web UI", error);555 log.error("Failed to load Executor web UI", error);
499 destroyWindow(window);556 if (!existingWindow) destroyWindow(window);
500 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: caller decides whether to fall back or surface startup failure557 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: caller decides whether to fall back or surface startup failure
501 throw error;558 throw error;
502 }559 }
@@ -916,6 +973,7 @@ const runUpdateCheck = async ({ alertOnFail }: UpdateCheckOptions) => { · 413 unmodified lines
916// here, the boot-time update check never runs (it sits after a successful973// here, the boot-time update check never runs (it sits after a successful
917// sidecar start), so a broken app could never self-update its way out.974// sidecar start), so a broken app could never self-update its way out.
918const handleFatalSidecarFailure = async (error: unknown) => {975const handleFatalSidecarFailure = async (error: unknown) => {
976 showCrashScreen();
919 if (app.isPackaged) {977 if (app.isPackaged) {
920 // Install whatever finishes downloading by the time the user quits the978 // Install whatever finishes downloading by the time the user quits the
921 // failure dialog; if it downloads while the dialog is open, the regular979 // failure dialog; if it downloads while the dialog is open, the regular
@@ -1035,6 +1093,7 @@ const installApplicationMenu = () => { · 113 unmodified lines
1035const boot = async () => {1093const boot = async () => {
1036 installDockIcon();1094 installDockIcon();
1037 installApplicationMenu();1095 installApplicationMenu();
1096 await showStartupWindow();
1038 setupAutoUpdater();1097 setupAutoUpdater();
1039 applyFakeUpdateFromEnv();1098 applyFakeUpdateFromEnv();
1040 registerIpcHandlers();1099 registerIpcHandlers();
@@ -1042,10 +1101,7 @@ const boot = async () => { · 1 unmodified line
1042 // every request with no explanation. Swap in the crash screen — its1101 // every request with no explanation. Swap in the crash screen — its
1043 // buttons drive the regular preload bridge (restart / export diagnostics).1102 // buttons drive the regular preload bridge (restart / export diagnostics).
1044 onUnexpectedSidecarExit(() => {1103 onUnexpectedSidecarExit(() => {
1045 const window = liveMainWindow();1104 showCrashScreen();
1046 if (!window) return;
1047 const html = sidecarCrashHtml({ reported: errorReportingEnabled });
1048 void window.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(html)}`);
1049 // A crashing sidecar may be a broken release — quietly stage any1105 // A crashing sidecar may be a broken release — quietly stage any
1050 // available update so the install prompt appears on its own (same1106 // available update so the install prompt appears on its own (same
1051 // self-heal as the fatal startup path).1107 // self-heal as the fatal startup path).
apps/desktop/src/main/service.ts+29−11
@@ -38,6 +38,8 @@ export const bundledExecutorPath = (): string => · 37 unmodified lines
38 38
39const executorAvailable = (): boolean => app.isPackaged && existsSync(bundledExecutorPath());39const executorAvailable = (): boolean => app.isPackaged && existsSync(bundledExecutorPath());
40 40
41let userPathCapture: Promise<string | undefined> | null = null;
42
41const captureUserPath = async (): Promise<string | undefined> => {43const captureUserPath = async (): Promise<string | undefined> => {
42 const shell = process.env.SHELL;44 const shell = process.env.SHELL;
43 if (!shell) return process.env.PATH;45 if (!shell) return process.env.PATH;
@@ -53,24 +55,37 @@ const captureUserPath = async (): Promise<string | undefined> => { · 9 unmodified lines
53 }55 }
54};56};
55 57
56const serviceEnv = async (dataDir: string): Promise<NodeJS.ProcessEnv> => ({58const memoizedUserPath = (): Promise<string | undefined> => {
57 ...process.env,59 userPathCapture ??= captureUserPath();
58 ...(await captureUserPath().then((path) => (path ? { PATH: path } : {}))),60 return userPathCapture;
59 EXECUTOR_DATA_DIR: dataDir,61};
60 EXECUTOR_SCOPE_DIR: dataDir,62
61 EXECUTOR_CLIENT: "desktop",63const serviceEnv = async (
62 ...sidecarCrashReportingEnv(),64 dataDir: string,
63});65 options: { readonly captureUserPath: boolean },
66): Promise<NodeJS.ProcessEnv> => {
67 const path = options.captureUserPath ? await memoizedUserPath() : process.env.PATH;
68 return {
69 ...process.env,
70 ...(path ? { PATH: path } : {}),
71 EXECUTOR_DATA_DIR: dataDir,
72 EXECUTOR_SCOPE_DIR: dataDir,
73 EXECUTOR_CLIENT: "desktop",
74 ...sidecarCrashReportingEnv(),
75 };
76};
64 77
65const runExecutor = async (78const runExecutor = async (
66 args: ReadonlyArray<string>,79 args: ReadonlyArray<string>,
67 options: { readonly dataDir: string },80 options: { readonly dataDir: string; readonly captureUserPath?: boolean },
68): Promise<CommandResult> => {81): Promise<CommandResult> => {
69 const bin = bundledExecutorPath();82 const bin = bundledExecutorPath();
70 try {83 try {
71 const { stdout, stderr } = await execFileAsync(bin, [...args], {84 const { stdout, stderr } = await execFileAsync(bin, [...args], {
72 encoding: "utf8",85 encoding: "utf8",
73 env: await serviceEnv(options.dataDir),86 env: await serviceEnv(options.dataDir, {
87 captureUserPath: options.captureUserPath ?? true,
88 }),
74 });89 });
75 return { code: 0, stdout, stderr };90 return { code: 0, stdout, stderr };
76 } catch (error) {91 } catch (error) {
@@ -92,7 +107,10 @@ const statusValue = (stdout: string, key: "Registered" | "Running"): boolean => · 15 unmodified lines
92export const supervisedServiceStatus = async (): Promise<SupervisedServiceStatus> => {107export const supervisedServiceStatus = async (): Promise<SupervisedServiceStatus> => {
93 if (!executorAvailable()) return { supported: false, registered: false, running: false };108 if (!executorAvailable()) return { supported: false, registered: false, running: false };
94 const dataDir = join(app.getPath("home"), ".executor");109 const dataDir = join(app.getPath("home"), ".executor");
95 const result = await runExecutor(["service", "status"], { dataDir });110 const result = await runExecutor(["service", "status"], {
111 dataDir,
112 captureUserPath: false,
113 });
96 if (result.code !== 0) {114 if (result.code !== 0) {
97 serviceLog.warn(`service status failed: ${result.stderr || result.stdout}`);115 serviceLog.warn(`service status failed: ${result.stderr || result.stdout}`);
98 return { supported: true, registered: false, running: false };116 return { supported: true, registered: false, running: false };
e2e/desktop-packaged/stale-launchd-registration.test.ts+114−54
@@ -23,8 +23,8 @@ import { RunDir } from "../src/services"; · 22 unmodified lines
23const execFileAsync = promisify(execFile);23const execFileAsync = promisify(execFile);
24const SERVICE_LABEL = "sh.executor.daemon";24const SERVICE_LABEL = "sh.executor.daemon";
25const SCENARIO_NAME =25const SCENARIO_NAME =
26 "Desktop packaged supervised attach: stale launchd registration pins the 15s dead boot wait";26 "Desktop packaged supervised attach: stale launchd registration self-heals without a dead boot wait";
27const RECORDING_FILE = resolve("recordings/stale-launchd-slow-boot.mp4");27const RECORDING_FILE = resolve("recordings/stale-launchd-fast-boot.mp4");
28const RECORDING_WIDTH = 1280;28const RECORDING_WIDTH = 1280;
29const RECORDING_HEIGHT = 720;29const RECORDING_HEIGHT = 720;
30const STALE_PLIST_PORT = 58_251;30const STALE_PLIST_PORT = 58_251;
@@ -72,12 +72,15 @@ interface LaunchTiming { · 41 unmodified lines
72 72
73interface BootResult {73interface BootResult {
74 readonly pageCdpMs: number;74 readonly pageCdpMs: number;
75 readonly startupWindowMs: number;
75 readonly sidecarReadyMs: number;76 readonly sidecarReadyMs: number;
76 readonly connectionKind: string;77 readonly connectionKind: string;
77 readonly origin: string;78 readonly origin: string;
78 readonly manifestKind: string;79 readonly manifestKind: string;
79 readonly manifestPid: number;80 readonly manifestPid: number;
80 readonly matchedLogLine: string | null;81 readonly kickstartFailureLine: string | null;
82 readonly recoveryLogLine: string | null;
83 readonly launchdPrintSucceeded: boolean;
81}84}
82 85
83declare global {86declare global {
@@ -273,6 +276,17 @@ const launchctl = async (args: ReadonlyArray<string>): Promise<boolean> => { · 189 unmodified lines
273 }276 }
274};277};
275 278
279const launchctlPrintService = async (): Promise<string | null> => {
280 try {
281 const { stdout, stderr } = await execFileAsync("launchctl", ["print", serviceTarget()], {
282 encoding: "utf8",
283 });
284 return `${stdout}${stderr}`.trim();
285 } catch {
286 return null;
287 }
288};
289
276const captureLaunchdService = (): LaunchdServiceSnapshot | null => {290const captureLaunchdService = (): LaunchdServiceSnapshot | null => {
277 if (process.platform !== "darwin") return null;291 if (process.platform !== "darwin") return null;
278 const path = launchAgentPath();292 const path = launchAgentPath();
@@ -482,31 +496,51 @@ const readServerManifest = (home: string): ServerManifest => { · 203 unmodified lines
482 return JSON.parse(readFileSync(manifestPath, "utf8")) as ServerManifest;496 return JSON.parse(readFileSync(manifestPath, "utf8")) as ServerManifest;
483};497};
484 498
485const readKickstartFailureLine = (home: string): string | null => {499interface StaleRecoveryLog {
500 readonly kickstartFailureLine: string | null;
501 readonly recoveryLogLine: string | null;
502}
503
504// The desktop's own classification line for a stale registration; electron-log
505// appends the launchctl error on the same line. Production launchctl fails the
506// kickstart with exit 113 ("Could not find service"); a tart guest's
507// `launchctl asuser` session fails it with exit 125 ("Domain does not support
508// specified action"). Same stale state, same self-heal branch, so match the
509// kickstart failure without pinning the environment-specific exit code.
510const isKickstartFailureLine = (line: string): boolean =>
511 line.includes("Failed to restart registered supervised service; reinstalling") &&
512 line.includes("launchctl kickstart failed");
513
514const isRecoveryLogLine = (line: string): boolean =>
515 line.includes("installed supervised service via bundled executor") ||
516 line.includes("Failed to install supervised service after registered service restart failure") ||
517 line.includes("using managed sidecar");
518
519const readStaleRecoveryLog = (home: string): StaleRecoveryLog => {
486 const logPath = join(home, "Library", "Logs", "Executor", "main.log");520 const logPath = join(home, "Library", "Logs", "Executor", "main.log");
487 if (!existsSync(logPath)) return null;521 if (!existsSync(logPath)) return { kickstartFailureLine: null, recoveryLogLine: null };
488 return (522 const lines = readFileSync(logPath, "utf8").split("\n");
489 readFileSync(logPath, "utf8")523 const kickstartIndex = lines.findIndex(isKickstartFailureLine);
490 .split("\n")524 if (kickstartIndex < 0) return { kickstartFailureLine: null, recoveryLogLine: null };
491 .find(525 const recoveryLogLine =
492 (line) =>526 lines.slice(kickstartIndex + 1).find((line) => isRecoveryLogLine(line)) ?? null;
493 line.includes("launchctl kickstart failed (exit 113)") &&527 return {
494 line.includes('Could not find service "sh.executor.daemon"'),528 kickstartFailureLine: lines[kickstartIndex] ?? null,
495 ) ?? null529 recoveryLogLine,
496 );530 };
497};531};
498 532
499const waitForKickstartFailureLine = async (home: string): Promise<string> => {533const waitForStaleRecoveryLog = async (home: string): Promise<StaleRecoveryLog> => {
500 const deadline = Date.now() + 30_000;534 const deadline = Date.now() + 30_000;
501 for (;;) {535 for (;;) {
502 const line = readKickstartFailureLine(home);536 const lines = readStaleRecoveryLog(home);
503 if (line) return line;537 if (lines.kickstartFailureLine && lines.recoveryLogLine) return lines;
504 if (Date.now() >= deadline) {538 if (Date.now() >= deadline) {
505 const logPath = join(home, "Library", "Logs", "Executor", "main.log");539 const logPath = join(home, "Library", "Logs", "Executor", "main.log");
506 const tail = existsSync(logPath)540 const tail = existsSync(logPath)
507 ? readFileSync(logPath, "utf8").split("\n").slice(-20).join("\n")541 ? readFileSync(logPath, "utf8").split("\n").slice(-20).join("\n")
508 : "<main.log missing>";542 : "<main.log missing>";
509 throw new Error(`Timed out waiting for kickstart failure log line\n${tail}`);543 throw new Error(`Timed out waiting for stale launchd recovery log lines\n${tail}`);
510 }544 }
511 await new Promise((resolveWait) => setTimeout(resolveWait, 250));545 await new Promise((resolveWait) => setTimeout(resolveWait, 250));
512 }546 }
@@ -516,7 +550,8 @@ const bootPackagedApp = async (input: { · 3 unmodified lines
516 readonly home: string;550 readonly home: string;
517 readonly env: NodeJS.ProcessEnv;551 readonly env: NodeJS.ProcessEnv;
518 readonly recorder: BootRecorder | null;552 readonly recorder: BootRecorder | null;
519 readonly expectKickstartFailure: boolean;553 readonly expectStartupWindow: boolean;
554 readonly expectStaleRecoveryLog: boolean;
520}): Promise<BootResult> => {555}): Promise<BootResult> => {
521 let app: PackagedApp | undefined;556 let app: PackagedApp | undefined;
522 const startedAt = Date.now();557 const startedAt = Date.now();
@@ -524,6 +559,15 @@ const bootPackagedApp = async (input: { · 1 unmodified line
524 const launched = await launchPackaged(input.env, input.recorder);559 const launched = await launchPackaged(input.env, input.recorder);
525 app = launched.app;560 app = launched.app;
526 const page = app.cdp;561 const page = app.cdp;
562 const startupWindowVisible = await page
563 .waitForText("Starting Executor", 5_000)
564 .then(() => true)
565 .catch(() => false);
566 const startupWindowMs = startupWindowVisible ? Date.now() - startedAt : launched.pageCdpMs;
567 expect(
568 !input.expectStartupWindow || startupWindowVisible,
569 "startup window shows before service recovery finishes when required",
570 ).toBe(true);
527 await page.waitForText("Settings", 120_000);571 await page.waitForText("Settings", 120_000);
528 const sidecarReadyMs = Date.now() - startedAt;572 const sidecarReadyMs = Date.now() - startedAt;
529 await input.recorder?.captureReadyFrame();573 await input.recorder?.captureReadyFrame();
@@ -534,17 +578,21 @@ const bootPackagedApp = async (input: { · 4 unmodified lines
534 } | null>("window.executor.getServerConnection()");578 } | null>("window.executor.getServerConnection()");
535 expect(connection, "the app eventually exposes a server connection").not.toBeNull();579 expect(connection, "the app eventually exposes a server connection").not.toBeNull();
536 const manifest = readServerManifest(input.home);580 const manifest = readServerManifest(input.home);
537 const matchedLogLine = input.expectKickstartFailure581 const recoveryLog = input.expectStaleRecoveryLog
538 ? await waitForKickstartFailureLine(input.home)582 ? await waitForStaleRecoveryLog(input.home)
539 : readKickstartFailureLine(input.home);583 : readStaleRecoveryLog(input.home);
584 const launchdPrintSucceeded = (await launchctlPrintService()) !== null;
540 return {585 return {
541 pageCdpMs: launched.pageCdpMs,586 pageCdpMs: launched.pageCdpMs,
587 startupWindowMs,
542 sidecarReadyMs,588 sidecarReadyMs,
543 connectionKind: connection!.kind,589 connectionKind: connection!.kind,
544 origin: connection!.origin,590 origin: connection!.origin,
545 manifestKind: manifest.kind,591 manifestKind: manifest.kind,
546 manifestPid: manifest.pid,592 manifestPid: manifest.pid,
547 matchedLogLine,593 kickstartFailureLine: recoveryLog.kickstartFailureLine,
594 recoveryLogLine: recoveryLog.recoveryLogLine,
595 launchdPrintSucceeded,
548 };596 };
549 } finally {597 } finally {
550 await input.recorder?.stop();598 await input.recorder?.stop();
@@ -680,7 +728,7 @@ class BootRecorder { · 129 unmodified lines
680 728
681 static create = (runDir: string): BootRecorder | null => {729 static create = (runDir: string): BootRecorder | null => {
682 if (process.env.E2E_RECORD !== "1") return null;730 if (process.env.E2E_RECORD !== "1") return null;
683 const frameDir = join(runDir, "stale-launchd-slow-boot-frames");731 const frameDir = join(runDir, "stale-launchd-fast-boot-frames");
684 rmSync(frameDir, { recursive: true, force: true });732 rmSync(frameDir, { recursive: true, force: true });
685 mkdirSync(frameDir, { recursive: true });733 mkdirSync(frameDir, { recursive: true });
686 mkdirSync(dirname(RECORDING_FILE), { recursive: true });734 mkdirSync(dirname(RECORDING_FILE), { recursive: true });
@@ -707,7 +755,7 @@ class BootRecorder { · 20 unmodified lines
707 755
708 captureReadyFrame = async (): Promise<void> => {756 captureReadyFrame = async (): Promise<void> => {
709 if (!this.enabled) return;757 if (!this.enabled) return;
710 this.enqueueFrame("managed sidecar ready", this.page);758 this.enqueueFrame("executor ready", this.page);
711 await this.queue;759 await this.queue;
712 };760 };
713 761
@@ -802,7 +850,8 @@ const run = async (runDir: string): Promise<void> => { · 88 unmodified lines
802 home: cleanHome,850 home: cleanHome,
803 env: cleanBootEnv(cleanHome),851 env: cleanBootEnv(cleanHome),
804 recorder: null,852 recorder: null,
805 expectKickstartFailure: false,853 expectStartupWindow: false,
854 expectStaleRecoveryLog: false,
806 });855 });
807 856
808 writeStaleLaunchAgentPlist(staleHome);857 writeStaleLaunchAgentPlist(staleHome);
@@ -811,53 +860,64 @@ const run = async (runDir: string): Promise<void> => { · 2 unmodified lines
811 home: staleHome,860 home: staleHome,
812 env: packagedAppEnv(staleHome),861 env: packagedAppEnv(staleHome),
813 recorder: staleRecorder,862 recorder: staleRecorder,
814 expectKickstartFailure: true,863 expectStartupWindow: true,
864 expectStaleRecoveryLog: true,
815 });865 });
816 866
817 /*867 /*
818 * Regression pin for the current broken behavior:868 * Regression guard for the production incident:
819 * a stale plist makes `service status` report registered even though launchd869 * a stale plist makes `service status` report registered while launchd cannot
820 * cannot find the label. The packaged app logs the kickstart failure, then870 * find the label. The app must show a window immediately, then repair or
821 * still pays the full 15s wait before falling back to managed-spawn.871 * bypass that stale service without paying the old dead 15s wait.
822 *
823 * When the fix lands, invert the >=14s stale assertions. The desired
824 * behavior is a fast-fail to managed-spawn, close to the clean control.
825 */872 */
826 expect(stale.matchedLogLine, "stale boot logs the launchd kickstart failure").toContain(873 expect(stale.kickstartFailureLine, "stale boot logs the launchd kickstart failure").toContain(
827 "launchctl kickstart failed (exit 113)",874 "launchctl kickstart failed",
828 );
829 expect(stale.matchedLogLine, "stale boot says launchd cannot find the label").toContain(
830 'Could not find service "sh.executor.daemon"',
831 );875 );
876 expect(
877 stale.kickstartFailureLine,
878 "the failed kickstart is classified as a stale registration to repair",
879 ).toContain("Failed to restart registered supervised service; reinstalling");
880 expect(
881 stale.recoveryLogLine,
882 "kickstart failure is followed by self-heal or fallback logging",
883 ).not.toBeNull();
832 expect(884 expect(
833 stale.pageCdpMs,885 stale.pageCdpMs,
834 "window/CDP is not reachable until the dead wait is paid",886 "the Electron page target is reachable before service recovery finishes",
835 ).toBeGreaterThanOrEqual(14_000);887 ).toBeLessThan(3_000);
836 expect(888 expect(
837 stale.sidecarReadyMs,889 stale.startupWindowMs,
838 "managed sidecar readiness is delayed by the dead wait",890 "the startup window appears promptly in the stale launchd case",
839 ).toBeGreaterThanOrEqual(14_000);891 ).toBeLessThan(3_000);
840 expect(stale.connectionKind, "the app eventually recovers through managed-spawn").toBe(892 expect(new URL(stale.origin).port, "recovery must not attach to the stale plist port").not.toBe(
841 "desktop-sidecar",893 String(STALE_PLIST_PORT),
842 );894 );
843 expect(
844 new URL(stale.origin).port,
845 "managed recovery must not be a launchd attach to the stale plist port",
846 ).not.toBe(String(STALE_PLIST_PORT));
847 expect(895 expect(
848 clean.sidecarReadyMs,896 clean.sidecarReadyMs,
849 "the clean control skips supervision and boots quickly",897 "the clean control skips supervision and boots quickly",
850 ).toBeLessThan(10_000);898 ).toBeLessThan(10_000);
851 expect(899 expect(
852 stale.sidecarReadyMs,900 stale.sidecarReadyMs,
853 "the stale launchd path is dramatically slower than the clean control",901 "stale launchd recovery stays within the fast boot budget",
854 ).toBeGreaterThan(clean.sidecarReadyMs + 5_000);902 ).toBeLessThan(10_000);
903 expect(
904 stale.sidecarReadyMs,
905 "the stale launchd path stays close to the clean control",
906 ).toBeLessThan(clean.sidecarReadyMs + 7_500);
907 const repairedLaunchd = stale.launchdPrintSucceeded && stale.manifestKind === "cli-daemon";
908 const managedFallback =
909 !stale.launchdPrintSucceeded && stale.manifestKind === "desktop-sidecar";
910 expect(
911 repairedLaunchd || managedFallback,
912 "stale state is either repaired into launchd or cleanly falls back to managed spawn",
913 ).toBe(true);
855 914
856 const recording = staleRecorder?.recordingStats();915 const recording = staleRecorder?.recordingStats();
857 emitEvidence(916 emitEvidence(
858 `[stale-launchd-repro] clean=${clean.sidecarReadyMs}ms stale=${stale.sidecarReadyMs}ms stalePageCdp=${stale.pageCdpMs}ms cleanOrigin=${clean.origin} staleOrigin=${stale.origin} staleConnectionKind=${stale.connectionKind} staleManifestKind=${stale.manifestKind} stalePid=${stale.manifestPid}`,917 `[stale-launchd-repro] clean=${clean.sidecarReadyMs}ms stale=${stale.sidecarReadyMs}ms staleStartup=${stale.startupWindowMs}ms stalePageCdp=${stale.pageCdpMs}ms cleanOrigin=${clean.origin} staleOrigin=${stale.origin} staleConnectionKind=${stale.connectionKind} staleManifestKind=${stale.manifestKind} stalePid=${stale.manifestPid} launchdPrintAfterBoot=${stale.launchdPrintSucceeded}`,
859 );918 );
860 emitEvidence(`[stale-launchd-repro] matchedLogLine=${stale.matchedLogLine}`);919 emitEvidence(`[stale-launchd-repro] kickstartFailureLine=${stale.kickstartFailureLine}`);
920 emitEvidence(`[stale-launchd-repro] recoveryLogLine=${stale.recoveryLogLine}`);
861 if (recording) {921 if (recording) {
862 emitEvidence(922 emitEvidence(
863 `[stale-launchd-repro] recording=${recording.path} frames=${recording.frames} size=${recording.size}`,923 `[stale-launchd-repro] recording=${recording.path} frames=${recording.frames} size=${recording.size}`,
e2e/recordings/stale-launchd-fast-boot.mp4+0−0

Binary file not shown.

e2e/recordings/stale-launchd-slow-boot.mp4+0−0

Binary file not shown.