import { assert, it } from "@effect/vitest"; import { CommandId, ContextTransferId, EventId, MessageId, NodeId, type OrchestrationV2AppThread, type OrchestrationV2Run, ProjectId, ProviderDriverKind, ProviderInstanceId, ProviderSessionId, ProviderThreadId, RunId, RuntimeRequestId, ThreadId, TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import { EffectOutboxV2, layer as effectOutboxLayer } from "./EffectOutbox.ts"; import { ProjectionStoreV2, layer as projectionStoreLayer } from "./ProjectionStore.ts"; const TestLayer = Layer.mergeAll(projectionStoreLayer, effectOutboxLayer).pipe( Layer.provideMerge(SqlitePersistenceMemory), ); const providerInstanceId = ProviderInstanceId.make("codex"); const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" }; const driver = ProviderDriverKind.make("codex"); const createThread = Effect.fn(function* ( name: string, overrides: Partial = {}, ) { const projections = yield* ProjectionStoreV2; const now = yield* DateTime.now; const threadId = ThreadId.make(`thread:recovery:${name}`); const thread: OrchestrationV2AppThread = { createdBy: "user", creationSource: "web", id: threadId, projectId: ProjectId.make("project:recovery"), title: name, providerInstanceId, modelSelection, runtimeMode: "full-access", interactionMode: "default", branch: null, worktreePath: null, activeProviderThreadId: null, lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, forkedFrom: null, createdAt: now, updatedAt: now, archivedAt: null, settledOverride: null, settledAt: null, lastVisitedAt: null, deletedAt: null, ...overrides, }; yield* projections.apply({ id: EventId.make(`event:${threadId}:created`), type: "thread.created", threadId, occurredAt: now, payload: thread, }); return threadId; }); const createRun = Effect.fn(function* ( threadId: ThreadId, status: OrchestrationV2Run["status"], overrides: Partial = {}, ) { const projections = yield* ProjectionStoreV2; const now = yield* DateTime.now; const ordinal = overrides.ordinal ?? 1; const runId = RunId.make(`run:${threadId}:${ordinal}`); yield* projections.apply({ id: EventId.make(`event:${runId}:created`), type: "run.created", threadId, runId, occurredAt: now, payload: { id: runId, threadId, ordinal, providerInstanceId, modelSelection, providerThreadId: null, userMessageId: MessageId.make(`message:${runId}`), rootNodeId: null, activeAttemptId: null, status, requestedAt: now, startedAt: now, completedAt: status === "completed" ? now : null, checkpointId: null, contextHandoffId: null, ...overrides, }, }); return runId; }); it.effect("selects unfinished recovery work without reading settled thread histories", () => Effect.gen(function* () { const projections = yield* ProjectionStoreV2; const outbox = yield* EffectOutboxV2; const sql = yield* SqlClient.SqlClient; const now = yield* DateTime.now; for (let index = 0; index < 600; index += 1) { yield* createRun(yield* createThread(`settled-${index}`), "completed"); } const queued = yield* createThread("queued"); const archived = yield* createThread("archived", { archivedAt: now }); const deleted = yield* createThread("deleted", { deletedAt: now }); const blocked = yield* createThread("blocked"); for (const threadId of [queued, archived, deleted, blocked]) { yield* createRun(threadId, "queued"); } yield* createRun(blocked, "waiting", { ordinal: 2 }); const background = yield* createThread("background"); yield* createRun(background, "completed"); yield* projections.apply({ id: EventId.make("event:recovery:background"), type: "turn-item.updated", threadId: background, occurredAt: now, payload: { id: TurnItemId.make("item:recovery:background"), threadId: background, runId: null, nodeId: null, providerThreadId: null, providerTurnId: null, nativeItemRef: null, parentItemId: null, ordinal: 1, type: "dynamic_tool", status: "waiting", title: null, startedAt: now, completedAt: null, updatedAt: now, toolName: "background", input: null, output: null, }, }); const outboxOnly = yield* createThread("outbox-only"); yield* outbox.enqueue([ { id: "effect:recovery:outbox-only", commandId: CommandId.make("command:recovery:outbox-only"), threadId: outboxOnly, request: { type: "provider-turn.start", runId: RunId.make("run:outbox-only") }, }, ]); const requestOnly = yield* createThread("request-only"); yield* projections.apply({ id: EventId.make("event:recovery:request-only"), type: "runtime-request.updated", threadId: requestOnly, occurredAt: now, payload: { id: RuntimeRequestId.make("request:recovery:pending"), nodeId: NodeId.make("node:recovery:request"), providerTurnId: null, nativeRequestRef: null, kind: "user_input", status: "pending", responseCapability: { type: "not_resumable", reason: "Process stopped" }, createdAt: now, resolvedAt: null, }, }); const delivery = yield* createThread("delivery", { archivedAt: now }); yield* createRun(delivery, "completed", { delegatedCompletion: { disposition: "open", nextGeneration: 2, delivery: { generation: 1, messageId: MessageId.make("message:delivery"), taskIds: [] }, }, }); // This historical payload cannot be decoded. Candidate discovery must not // materialize it while deciding which threads have work to reconcile. yield* sql` UPDATE orchestration_v2_projection_runs SET payload_json = '{broken' WHERE thread_id = ${ThreadId.make("thread:recovery:settled-599")} `; assert.deepEqual(yield* projections.getRecoveryThreadIds("queued-runs"), [queued]); assert.deepEqual( new Set(yield* projections.getRecoveryThreadIds("runtime")), new Set([queued, archived, blocked, background, outboxOnly, requestOnly]), ); assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [delivery]); assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []); assert.deepEqual(yield* projections.getUnreadableThreadIds(), [ ThreadId.make("thread:recovery:settled-599"), ]); }).pipe(Effect.provide(TestLayer)), ); it.effect("recovers terminal subagent results until their cross-thread transfer exists", () => Effect.gen(function* () { const projections = yield* ProjectionStoreV2; const sql = yield* SqlClient.SqlClient; const now = yield* DateTime.now; const parent = yield* createThread("subagent-parent"); const children: Array = []; for (const name of ["terminal", "archived", "deleted", "running"]) { const child = yield* createThread(`subagent-${name}`, { lineage: { parentThreadId: parent, relationshipToParent: "subagent", rootThreadId: parent }, forkedFrom: { type: "node", nodeId: NodeId.make(`node:${name}`) }, archivedAt: name === "archived" ? now : null, deletedAt: name === "deleted" ? now : null, }); yield* createRun(child, name === "running" ? "running" : "completed"); children.push(child); } assert.deepEqual( new Set(yield* projections.getRecoveryThreadIds("subagent-results")), new Set(children.slice(0, 2)), ); const completed = children[0]!; const transferId = ContextTransferId.make("transfer:recovery:subagent-result"); yield* projections.apply({ id: EventId.make("event:recovery:subagent-result"), type: "context-transfer.created", threadId: parent, occurredAt: now, payload: { id: transferId, type: "subagent_result", sourceThreadId: completed, targetThreadId: parent, sourcePoint: { threadId: completed }, basePoint: null, sourceProviderInstanceId: providerInstanceId, targetProviderInstanceId: providerInstanceId, targetRunId: null, status: "pending", resolution: null, createdBy: "system", error: null, createdAt: now, updatedAt: now, consumedAt: null, }, }); const archivedChild = children[1]; assert.isDefined(archivedChild); assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), [archivedChild]); assert.deepEqual(yield* projections.getUnreadableThreadIds(), []); yield* sql` UPDATE orchestration_v2_projection_context_transfers SET payload_json = '{}' WHERE context_transfer_id = ${transferId} `; assert.deepEqual( new Set(yield* projections.getUnreadableThreadIds()), new Set([parent, completed]), ); }).pipe(Effect.provide(TestLayer)), ); it.effect("includes shared sessions and provider-owned background rosters in recovery", () => Effect.gen(function* () { const projections = yield* ProjectionStoreV2; const now = yield* DateTime.now; const first = yield* createThread("shared-first"); const second = yield* createThread("shared-second", { archivedAt: now }); const providerSessionId = ProviderSessionId.make("session:recovery:shared"); for (const threadId of [first, second]) { yield* projections.apply({ id: EventId.make(`event:${threadId}:session`), type: "provider-session.attached", threadId, driver, providerInstanceId, occurredAt: now, payload: { id: providerSessionId, driver, providerInstanceId, status: "ready", cwd: "/workspace", model: modelSelection.model, capabilities: CodexProviderCapabilitiesV2, createdAt: now, updatedAt: now, lastError: null, }, }); } const roster = yield* createThread("roster"); yield* projections.apply({ id: EventId.make("event:recovery:roster"), type: "provider-thread.updated", threadId: roster, driver, providerInstanceId, occurredAt: now, payload: { id: ProviderThreadId.make("provider-thread:recovery:roster"), appThreadId: roster, ownerNodeId: null, driver, providerInstanceId, providerSessionId: null, nativeThreadRef: null, nativeConversationHeadRef: null, status: "idle", firstRunOrdinal: null, lastRunOrdinal: null, handoffIds: [], forkedFrom: null, createdAt: now, updatedAt: now, pendingBackgroundTasks: [{ taskId: "background", description: "Still running" }], }, }); assert.deepEqual( new Set(yield* projections.getRecoveryThreadIds("runtime")), new Set([first, second, roster]), ); assert.deepEqual(yield* projections.getUnreadableThreadIds(), []); const sql = yield* SqlClient.SqlClient; yield* sql` UPDATE orchestration_v2_projection_provider_sessions SET payload_json = '{}' WHERE provider_session_id = ${providerSessionId} `; assert.deepEqual( new Set(yield* projections.getUnreadableThreadIds()), new Set([first, second]), ); }).pipe(Effect.provide(TestLayer)), ); it.effect("marks fork descendants unreadable when their source is missing or corrupt", () => Effect.gen(function* () { const projections = yield* ProjectionStoreV2; const sql = yield* SqlClient.SqlClient; const source = yield* createThread("source"); const sourceRun = yield* createRun(source, "completed"); const fork = yield* createThread("fork", { forkedFrom: { type: "run", threadId: source, runId: sourceRun }, }); const descendant = yield* createThread("descendant", { forkedFrom: { type: "run", threadId: fork, runId: RunId.make("run:fork") }, }); assert.deepEqual(yield* projections.getUnreadableThreadIds(), []); yield* sql` UPDATE orchestration_v2_projection_runs SET payload_json = '{}' WHERE run_id = ${sourceRun} `; assert.deepEqual( new Set(yield* projections.getUnreadableThreadIds()), new Set([source, fork, descendant]), ); yield* sql`DELETE FROM orchestration_v2_projection_threads WHERE thread_id = ${source}`; assert.deepEqual( new Set(yield* projections.getUnreadableThreadIds()), new Set([fork, descendant]), ); }).pipe(Effect.provide(TestLayer)), );