History

fix(desktop,web): contain renderer memory growth and recover from renderer OOM crashes (#5148)

Theo Browne committed 3 months ago

Commit 36caf341 parent 90e37784 files · +173 −8Browse files

Unified · Split

4 files
  1. apps/desktop/src/window/DesktopWindow.ts +41 −4
  2. apps/web/src/components/Sidebar.logic.ts +6 −2
  3. packages/client-runtime/src/state/threadReducer.test.ts +88 −0
  4. packages/client-runtime/src/state/threadReducer.ts +38 −2
apps/desktop/src/window/DesktopWindow.ts+41−4
@@ -1,3 +1,4 @@
1+import * as Clock from "effect/Clock";
12 import * as Context from "effect/Context";
23 import * as Effect from "effect/Effect";
34 import * as Fiber from "effect/Fiber";
@@ -25,6 +26,12 @@ const TITLEBAR_LIGHT_SYMBOL_COLOR = "#1f2937"; · 21 unmodified lines
2526 const TITLEBAR_DARK_SYMBOL_COLOR = "#f8fafc";
2627 const MAIN_WINDOW_BOUNDS_PERSIST_DEBOUNCE_MS = 500;
2728 const DEVELOPMENT_LOAD_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const;
29+// Renderer crash (usually V8 OOM on long sessions) recovery: reload after a
30+// short delay, at most MAX_ATTEMPTS times per rolling WINDOW so a renderer
31+// that dies on boot cannot reload-loop forever.
32+const RENDERER_RECOVERY_RELOAD_DELAY_MS = 500;
33+const RENDERER_RECOVERY_MAX_ATTEMPTS = 3;
34+const RENDERER_RECOVERY_WINDOW_MS = 60_000;
2835 const DEVELOPMENT_RETRYABLE_LOAD_ERROR_CODES = new Set([
2936 -2, // ERR_FAILED
3037 -7, // ERR_TIMED_OUT
@@ -549,6 +556,7 @@ export const make = Effect.gen(function* () { · 518 unmodified lines
549556
550557 let developmentLoadRetryIndex = 0;
551558 let developmentLoadRetryFiber: Fiber.Fiber<void, never> | undefined;
559+ let rendererRecoveryTimestamps: number[] = [];
552560 const clearDevelopmentLoadRetry = () => {
553561 if (developmentLoadRetryFiber === undefined) {
554562 return;
@@ -630,10 +638,39 @@ export const make = Effect.gen(function* () { · 75 unmodified lines
630638 },
631639 );
632640 window.webContents.on("render-process-gone", (_event, details) => {
633− void runPromise(
634− logWindowWarning("main window render process gone", {
635− reason: details.reason,
636− exitCode: details.exitCode,
641+ const recoverable =
642+ details.reason === "crashed" ||
643+ details.reason === "oom" ||
644+ details.reason === "abnormal-exit";
645+ // Long sessions can OOM the renderer (V8 heap exhaustion from
646+ // accumulated thread state). Without a reload the user is left staring
647+ // at a dead white window while agents keep running invisibly, so
648+ // recover by reloading — the renderer rehydrates from the backend,
649+ // which is unaffected. Recovery attempts are bounded so a renderer
650+ // that dies immediately on boot cannot reload-loop forever.
651+ runFork(
652+ Effect.gen(function* () {
653+ const now = yield* Clock.currentTimeMillis;
654+ rendererRecoveryTimestamps = rendererRecoveryTimestamps.filter(
655+ (timestamp) => now - timestamp < RENDERER_RECOVERY_WINDOW_MS,
656+ );
657+ const shouldRecover =
658+ recoverable &&
659+ !window.isDestroyed() &&
660+ rendererRecoveryTimestamps.length < RENDERER_RECOVERY_MAX_ATTEMPTS;
661+ yield* logWindowWarning("main window render process gone", {
662+ reason: details.reason,
663+ exitCode: details.exitCode,
664+ recovering: shouldRecover,
665+ });
666+ if (!shouldRecover) {
667+ return;
668+ }
669+ rendererRecoveryTimestamps.push(now);
670+ yield* Effect.sleep(RENDERER_RECOVERY_RELOAD_DELAY_MS);
671+ if (!window.isDestroyed()) {
672+ loadApplication();
673+ }
637674 }),
638675 );
639676 });
apps/web/src/components/Sidebar.logic.ts+6−2
@@ -16,8 +16,12 @@ import { resolveServerBackedAppStageLabel } from "../branding.logic"; · 15 unmodified lines
1616 export const THREAD_SELECTION_SAFE_SELECTOR = "[data-thread-item], [data-thread-selection-safe]";
1717 export const THREAD_JUMP_HINT_SHOW_DELAY_MS = 100;
1818 // Visible sidebar rows are prewarmed into the thread-detail cache so opening a
19−// nearby thread usually reuses an already-hot subscription.
20−export const SIDEBAR_THREAD_PREWARM_LIMIT = 10;
19+// nearby thread usually reuses an already-hot subscription. Each prewarmed
20+// thread holds a live, fully hydrated detail subscription (all messages and
21+// activities, growing as agents work) for as long as the row stays visible,
22+// so this limit is a direct renderer-heap and server-load multiplier — keep
23+// it small; cold opens still render instantly from the cached snapshot.
24+export const SIDEBAR_THREAD_PREWARM_LIMIT = 3;
2125
2226 type SidebarProject = {
2327 id: string;
packages/client-runtime/src/state/threadReducer.test.ts+88−0
@@ -633,6 +633,94 @@ describe("applyThreadDetailEvent", () => { · 632 unmodified lines
633633 expect(result.thread.activities[0]?.id).toBe("activity-0");
634634 }
635635 });
636+
637+ it("replaces earlier resolvable context-window updates for the same turn", () => {
638+ const contextWindowActivity = (id: string, sequence: number, usedTokens: unknown) => ({
639+ id: EventId.make(id),
640+ tone: "info" as const,
641+ kind: "context-window.updated",
642+ summary: "Context window updated",
643+ payload: { usedTokens },
644+ turnId: TurnId.make("turn-1"),
645+ sequence,
646+ createdAt: "2026-04-01T11:00:00.000Z",
647+ });
648+ const otherTurnActivity = contextWindowActivity("activity-other-turn", 2, 500);
649+ const existingActivities = [
650+ contextWindowActivity("activity-cw-1", 1, 1_000),
651+ { ...otherTurnActivity, turnId: TurnId.make("turn-0") },
652+ // Malformed row (no usedTokens): must survive, and must not be
653+ // treated as the latest value by consumers.
654+ contextWindowActivity("activity-cw-malformed", 3, undefined),
655+ contextWindowActivity("activity-cw-2", 4, 2_000),
656+ ];
657+
658+ const result = applyThreadDetailEvent(
659+ { ...baseThread, activities: existingActivities },
660+ {
661+ ...baseEventFields,
662+ sequence: 20,
663+ occurredAt: "2026-04-01T11:02:00.000Z",
664+ aggregateKind: "thread",
665+ aggregateId: ThreadId.make("thread-1"),
666+ type: "thread.activity-appended",
667+ payload: {
668+ threadId: ThreadId.make("thread-1"),
669+ activity: contextWindowActivity("activity-cw-3", 5, 3_000),
670+ },
671+ },
672+ );
673+
674+ expect(result.kind).toBe("updated");
675+ if (result.kind === "updated") {
676+ const ids = result.thread.activities.map((activity) => activity.id);
677+ // Same-turn resolvable rows collapse to the newest; the other turn's
678+ // row and the malformed row are untouched.
679+ expect(ids).toEqual(["activity-other-turn", "activity-cw-malformed", "activity-cw-3"]);
680+ }
681+ });
682+
683+ it("does not collapse context-window history for a malformed update", () => {
684+ const resolvable = {
685+ id: EventId.make("activity-cw-resolvable"),
686+ tone: "info" as const,
687+ kind: "context-window.updated",
688+ summary: "Context window updated",
689+ payload: { usedTokens: 1_000 },
690+ turnId: TurnId.make("turn-1"),
691+ sequence: 1,
692+ createdAt: "2026-04-01T11:00:00.000Z",
693+ };
694+
695+ const result = applyThreadDetailEvent(
696+ { ...baseThread, activities: [resolvable] },
697+ {
698+ ...baseEventFields,
699+ sequence: 21,
700+ occurredAt: "2026-04-01T11:03:00.000Z",
701+ aggregateKind: "thread",
702+ aggregateId: ThreadId.make("thread-1"),
703+ type: "thread.activity-appended",
704+ payload: {
705+ threadId: ThreadId.make("thread-1"),
706+ activity: {
707+ ...resolvable,
708+ id: EventId.make("activity-cw-broken"),
709+ payload: { usedTokens: Number.NaN },
710+ sequence: 2,
711+ },
712+ },
713+ },
714+ );
715+
716+ expect(result.kind).toBe("updated");
717+ if (result.kind === "updated") {
718+ // The resolvable row must survive so consumers can still derive a
719+ // usage value by walking backwards past the malformed row.
720+ const ids = result.thread.activities.map((activity) => activity.id);
721+ expect(ids).toEqual(["activity-cw-resolvable", "activity-cw-broken"]);
722+ }
723+ });
636724 });
637725
638726 describe("thread.turn-diff-completed", () => {
packages/client-runtime/src/state/threadReducer.ts+38−2
@@ -35,6 +35,24 @@ const activityOrder = O.combineAll<OrchestrationThreadActivity>([ · 34 unmodified lines
3535 O.mapInput(O.String, (a) => a.id),
3636 ]);
3737
38+/**
39+ * Matches the validity rule in `deriveLatestContextWindowSnapshot` (and the
40+ * server's snapshot-side `dropStaleContextWindowActivities`): rows without a
41+ * finite, non-negative `usedTokens` are skipped during the consumer's backward
42+ * walk, so they must not replace an earlier resolvable row here.
43+ */
44+function isResolvableContextWindowActivity(activity: OrchestrationThreadActivity): boolean {
45+ if (activity.kind !== "context-window.updated") {
46+ return false;
47+ }
48+ const payload =
49+ activity.payload && typeof activity.payload === "object"
50+ ? (activity.payload as Record<string, unknown>)
51+ : null;
52+ const usedTokens = payload?.usedTokens;
53+ return typeof usedTokens === "number" && Number.isFinite(usedTokens) && usedTokens >= 0;
54+}
55+
3856 /**
3957 * Apply a single orchestration event to an `OrchestrationThread`, returning
4058 * the updated thread, a deletion signal, or an "unchanged" marker when the
@@ -509,10 +527,28 @@ export function applyThreadDetailEvent( · 468 unmodified lines
509527
510528 // ── Activities ──────────────────────────────────────────────────
511529 case "thread.activity-appended": {
530+ const activity = event.payload.activity;
531+ // A resolvable context-window update supersedes earlier resolvable ones
532+ // for the same turn: consumers only read the latest value (walking the
533+ // array backwards), and providers stream these updates continuously, so
534+ // retaining the history grows the thread by thousands of rows over a
535+ // long session. Mirrors the server-side snapshot rule in
536+ // dropStaleContextWindowActivities; retention stays per turn so a
537+ // thread.reverted that discards turns can still resolve a value from
538+ // the turns that survive.
539+ const supersedesContextWindow = isResolvableContextWindowActivity(activity);
512540 const activities = pipe(
513541 thread.activities,
514− Arr.filter((activity) => activity.id !== event.payload.activity.id),
515− Arr.append(event.payload.activity),
542+ Arr.filter(
543+ (entry) =>
544+ entry.id !== activity.id &&
545+ !(
546+ supersedesContextWindow &&
547+ entry.turnId === activity.turnId &&
548+ isResolvableContextWindowActivity(entry)
549+ ),
550+ ),
551+ Arr.append(activity),
516552 Arr.sort(activityOrder),
517553 );
518554