import { EventEmitter } from "node:events"; import type { AppSettings, AppSettingsUpdateInput, ProjectAddInput, ProjectAddResult, ProjectListResult, ProjectRemoveInput, ProjectUpdateScriptsInput, ProjectUpdateScriptsResult, StateBootstrapResult, StateCatchUpInput, StateCatchUpResult, StateEvent, StateListMessagesInput, StateListMessagesResult, ThreadsCreateInput, ThreadsDeleteInput, ThreadsMarkVisitedInput, ThreadsUpdateBranchInput, ThreadsUpdateModelInput, ThreadsUpdateResult, ThreadsUpdateTerminalStateInput, ThreadsUpdateTitleInput, } from "@t3tools/contracts"; import { createLogger } from "./logger"; import type { LiveStoreStateMirror } from "./livestore/liveStoreEngine"; import { diffCatchUpResults, diffListMessagesResults, diffStateSnapshots } from "./livestore/parity"; import type { ApplyCheckpointRevertInput, StateSyncEngine } from "./stateSyncEngine"; interface LiveStoreReadPilotEvents { stateEvent: [event: StateEvent]; } type LiveStoreReadSource = "delegate" | "livestore"; type LiveStoreReadRoute = "state.bootstrap" | "state.catchUp" | "state.listMessages"; type LiveStoreFallbackReason = "mirror-error" | "empty-mirror"; export interface LiveStoreReadPilotMetrics { routeReadCounts: Record; fallbackCounts: Record; } export interface LiveStoreReadPilotStateSyncEngineOptions { delegate: StateSyncEngine; mirror: LiveStoreStateMirror; enableBootstrapParityCheck?: boolean; enableCatchUpParityCheck?: boolean; enableListMessagesParityCheck?: boolean; disableDelegateReadFallback?: boolean; } export class LiveStoreReadPilotStateSyncEngine extends EventEmitter implements StateSyncEngine { private readonly delegate: StateSyncEngine; private readonly mirror: LiveStoreStateMirror; private readonly logger = createLogger("livestore-read-pilot"); private readonly unsubscribeDelegate: () => void; private readonly enableBootstrapParityCheck: boolean; private readonly enableCatchUpParityCheck: boolean; private readonly enableListMessagesParityCheck: boolean; private readonly disableDelegateReadFallback: boolean; private closed = false; private bootstrapSource: LiveStoreReadSource = "delegate"; private catchUpSource: LiveStoreReadSource = "delegate"; private listMessagesSource: LiveStoreReadSource = "delegate"; private bootstrapParityState: "unknown" | "in-parity" | "drift" = "unknown"; private catchUpParityState: "unknown" | "in-parity" | "drift" = "unknown"; private listMessagesParityState: "unknown" | "in-parity" | "drift" = "unknown"; private readonly metrics: LiveStoreReadPilotMetrics = { routeReadCounts: { "state.bootstrap": { livestore: 0, delegate: 0 }, "state.catchUp": { livestore: 0, delegate: 0 }, "state.listMessages": { livestore: 0, delegate: 0 }, }, fallbackCounts: { "state.bootstrap": { "mirror-error": 0, "empty-mirror": 0 }, "state.catchUp": { "mirror-error": 0, "empty-mirror": 0 }, "state.listMessages": { "mirror-error": 0, "empty-mirror": 0 }, }, }; constructor(options: LiveStoreReadPilotStateSyncEngineOptions) { super(); this.delegate = options.delegate; this.mirror = options.mirror; this.enableBootstrapParityCheck = options.enableBootstrapParityCheck ?? false; this.enableCatchUpParityCheck = options.enableCatchUpParityCheck ?? false; this.enableListMessagesParityCheck = options.enableListMessagesParityCheck ?? false; this.disableDelegateReadFallback = options.disableDelegateReadFallback ?? false; this.unsubscribeDelegate = this.delegate.onStateEvent((event) => { this.emit("stateEvent", event); void this.mirror .mirrorStateEvent(event) .then((mirrored) => { if (mirrored === false) { this.logger.warn("mirror state event in read pilot reported unsuccessful write", { seq: event.seq, }); } }) .catch((error) => { this.logger.warn("failed to mirror state event in read pilot", { error, seq: event.seq, }); }); }); } onStateEvent(listener: (event: StateEvent) => void): () => void { this.on("stateEvent", listener); return () => { this.off("stateEvent", listener); }; } loadSnapshot(): StateBootstrapResult { try { const snapshot = this.mirror.debugReadSnapshot(); if (this.enableBootstrapParityCheck) { this.checkBootstrapParity(snapshot); } if (snapshot.lastStateSeq === 0 && !this.disableDelegateReadFallback) { const delegateSnapshot = this.delegate.loadSnapshot(); this.logReadSourceChange("state.bootstrap", this.bootstrapSource, "delegate", { lastStateSeq: delegateSnapshot.lastStateSeq, }); this.recordReadRoute("state.bootstrap", "delegate"); this.recordFallback("state.bootstrap", "empty-mirror"); this.bootstrapSource = "delegate"; return delegateSnapshot; } this.logReadSourceChange("state.bootstrap", this.bootstrapSource, "livestore", { lastStateSeq: snapshot.lastStateSeq, }); this.recordReadRoute("state.bootstrap", "livestore"); this.bootstrapSource = "livestore"; return snapshot; } catch (error) { if (this.disableDelegateReadFallback) { this.logger.error("failed to read bootstrap from livestore mirror with fallback disabled", { error, }); throw error; } this.logger.warn("failed to read bootstrap from livestore mirror; using delegate", { error }); this.recordFallback("state.bootstrap", "mirror-error"); } const snapshot = this.delegate.loadSnapshot(); this.logReadSourceChange("state.bootstrap", this.bootstrapSource, "delegate", { lastStateSeq: snapshot.lastStateSeq, }); this.recordReadRoute("state.bootstrap", "delegate"); this.bootstrapSource = "delegate"; return snapshot; } private checkBootstrapParity(mirrorSnapshot: StateBootstrapResult): void { let delegateSnapshot: StateBootstrapResult; try { delegateSnapshot = this.delegate.loadSnapshot(); } catch (error) { this.logger.warn("bootstrap parity check failed to read delegate snapshot", { error }); return; } const diffs = diffStateSnapshots(delegateSnapshot, mirrorSnapshot); if (diffs.length === 0) { if (this.bootstrapParityState !== "in-parity") { this.bootstrapParityState = "in-parity"; this.logger.info("livestore read pilot bootstrap parity check passed", { lastStateSeq: mirrorSnapshot.lastStateSeq, }); } return; } this.bootstrapParityState = "drift"; this.logger.warn("livestore read pilot bootstrap parity drift detected", { diffCount: diffs.length, samplePaths: diffs.slice(0, 5).map((diff) => diff.path), delegateLastStateSeq: delegateSnapshot.lastStateSeq, mirrorLastStateSeq: mirrorSnapshot.lastStateSeq, }); } listMessages(raw: StateListMessagesInput): StateListMessagesResult { try { const result = this.mirror.debugListMessages(raw); if (this.enableListMessagesParityCheck) { this.checkListMessagesParity(raw, result); } this.logReadSourceChange("state.listMessages", this.listMessagesSource, "livestore", { threadId: raw.threadId, }); this.recordReadRoute("state.listMessages", "livestore"); this.listMessagesSource = "livestore"; return result; } catch (error) { if (this.disableDelegateReadFallback) { this.logger.error( "failed to list messages from livestore mirror with fallback disabled", { error, threadId: raw.threadId, }, ); throw error; } this.recordFallback("state.listMessages", "mirror-error"); this.logger.warn("failed to list messages from livestore mirror; using delegate", { error, threadId: raw.threadId, }); const result = this.delegate.listMessages(raw); this.logReadSourceChange("state.listMessages", this.listMessagesSource, "delegate", { threadId: raw.threadId, }); this.recordReadRoute("state.listMessages", "delegate"); this.listMessagesSource = "delegate"; return result; } } private checkListMessagesParity( raw: StateListMessagesInput, mirrorResult: StateListMessagesResult, ): void { let delegateResult: StateListMessagesResult; try { delegateResult = this.delegate.listMessages(raw); } catch (error) { this.logger.warn("list-messages parity check failed to read delegate listMessages", { error }); return; } const diffs = diffListMessagesResults(delegateResult, mirrorResult); if (diffs.length === 0) { if (this.listMessagesParityState !== "in-parity") { this.listMessagesParityState = "in-parity"; this.logger.info("livestore read pilot list-messages parity check passed", { threadId: raw.threadId, offset: raw.offset ?? 0, limit: raw.limit ?? 200, }); } return; } this.listMessagesParityState = "drift"; this.logger.warn("livestore read pilot list-messages parity drift detected", { threadId: raw.threadId, offset: raw.offset ?? 0, limit: raw.limit ?? 200, diffCount: diffs.length, sampleDiffs: diffs.slice(0, 5), }); } catchUp(raw: StateCatchUpInput): StateCatchUpResult { const afterSeq = raw.afterSeq ?? 0; try { const result = this.mirror.debugCatchUp(afterSeq); if (this.enableCatchUpParityCheck) { this.checkCatchUpParity(raw, result); } this.logReadSourceChange("state.catchUp", this.catchUpSource, "livestore", { afterSeq, }); this.recordReadRoute("state.catchUp", "livestore"); this.catchUpSource = "livestore"; return result; } catch (error) { if (this.disableDelegateReadFallback) { this.logger.error("failed to catch up from livestore mirror with fallback disabled", { error, afterSeq, }); throw error; } this.recordFallback("state.catchUp", "mirror-error"); this.logger.warn("failed to catch up from livestore mirror; using delegate", { error, afterSeq, }); const result = this.delegate.catchUp(raw); this.logReadSourceChange("state.catchUp", this.catchUpSource, "delegate", { afterSeq, }); this.recordReadRoute("state.catchUp", "delegate"); this.catchUpSource = "delegate"; return result; } } debugReadMetrics(): LiveStoreReadPilotMetrics { return { routeReadCounts: { "state.bootstrap": { ...this.metrics.routeReadCounts["state.bootstrap"] }, "state.catchUp": { ...this.metrics.routeReadCounts["state.catchUp"] }, "state.listMessages": { ...this.metrics.routeReadCounts["state.listMessages"] }, }, fallbackCounts: { "state.bootstrap": { ...this.metrics.fallbackCounts["state.bootstrap"] }, "state.catchUp": { ...this.metrics.fallbackCounts["state.catchUp"] }, "state.listMessages": { ...this.metrics.fallbackCounts["state.listMessages"] }, }, }; } isReadFallbackDisabled(): boolean { return this.disableDelegateReadFallback; } private checkCatchUpParity(raw: StateCatchUpInput, mirrorResult: StateCatchUpResult): void { let delegateResult: StateCatchUpResult; try { delegateResult = this.delegate.catchUp(raw); } catch (error) { this.logger.warn("catch-up parity check failed to read delegate catch-up", { error }); return; } const diffs = diffCatchUpResults(delegateResult, mirrorResult); if (diffs.length === 0) { if (this.catchUpParityState !== "in-parity") { this.catchUpParityState = "in-parity"; this.logger.info("livestore read pilot catch-up parity check passed", { afterSeq: raw.afterSeq ?? 0, lastStateSeq: mirrorResult.lastStateSeq, }); } return; } this.catchUpParityState = "drift"; this.logger.warn("livestore read pilot catch-up parity drift detected", { afterSeq: raw.afterSeq ?? 0, diffCount: diffs.length, sampleDiffs: diffs.slice(0, 5), delegateLastStateSeq: delegateResult.lastStateSeq, mirrorLastStateSeq: mirrorResult.lastStateSeq, }); } private logReadSourceChange( route: LiveStoreReadRoute, previous: LiveStoreReadSource, next: LiveStoreReadSource, metadata: Record, ): void { if (previous === next) { return; } this.logger.info(`serving ${route} from ${next}`, { source: next, previousSource: previous, ...metadata, }); } private recordReadRoute(route: LiveStoreReadRoute, source: LiveStoreReadSource): void { this.metrics.routeReadCounts[route][source] += 1; } private recordFallback(route: LiveStoreReadRoute, reason: LiveStoreFallbackReason): void { this.metrics.fallbackCounts[route][reason] += 1; } getAppSettings(): AppSettings { return this.delegate.getAppSettings(); } updateAppSettings(raw: AppSettingsUpdateInput): AppSettings { return this.delegate.updateAppSettings(raw); } createThread(raw: ThreadsCreateInput): ThreadsUpdateResult { return this.delegate.createThread(raw); } updateThreadTerminalState(raw: ThreadsUpdateTerminalStateInput): ThreadsUpdateResult { return this.delegate.updateThreadTerminalState(raw); } updateThreadModel(raw: ThreadsUpdateModelInput): ThreadsUpdateResult { return this.delegate.updateThreadModel(raw); } updateThreadTitle(raw: ThreadsUpdateTitleInput): ThreadsUpdateResult { return this.delegate.updateThreadTitle(raw); } updateThreadBranch(raw: ThreadsUpdateBranchInput): ThreadsUpdateResult { return this.delegate.updateThreadBranch(raw); } markThreadVisited(raw: ThreadsMarkVisitedInput): ThreadsUpdateResult { return this.delegate.markThreadVisited(raw); } deleteThread(raw: ThreadsDeleteInput): void { this.delegate.deleteThread(raw); } listProjects(): ProjectListResult { return this.delegate.listProjects(); } addProject(raw: ProjectAddInput): ProjectAddResult { return this.delegate.addProject(raw); } removeProject(raw: ProjectRemoveInput): void { this.delegate.removeProject(raw); } updateProjectScripts(raw: ProjectUpdateScriptsInput): ProjectUpdateScriptsResult { return this.delegate.updateProjectScripts(raw); } applyCheckpointRevert(input: ApplyCheckpointRevertInput): void { this.delegate.applyCheckpointRevert(input); } close(): void { if (this.closed) { return; } this.closed = true; this.logger.info("livestore read pilot metrics", { metrics: this.metrics }); this.unsubscribeDelegate(); this.removeAllListeners(); void this.mirror.dispose(); } }