import type { EnvironmentId, OrchestrationV2DomainEvent, OrchestrationV2ThreadShell, Project, ThreadId, } from "@t3tools/contracts"; import { RelayApi, type RelayAgentActivityPublishProofPayload, type RelayAgentActivityState, } from "@t3tools/contracts/relay"; import { projectThreadAwarenessV2 } from "@t3tools/shared/agentAwareness"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import { normalizeRelayIssuer, RELAY_ACTIVITY_PUBLISH_TYP, signRelayJwt, } from "@t3tools/shared/relayJwt"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; import * as HttpApiClient from "effect/unstable/httpapi/HttpApiClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { isAgentActivityPublishingEnabledValue, PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, RELAY_ISSUER_SECRET, RELAY_URL_SECRET, } from "../cloud/config.ts"; import { getOrCreateEnvironmentKeyPairFromSecretStore } from "../cloud/environmentKeys.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import { forkParked } from "../serverActivation.ts"; export class AgentAwarenessRelay extends Context.Service< AgentAwarenessRelay, { readonly publishThread: (threadId: ThreadId) => Effect.Effect; readonly drain: Effect.Effect; readonly start: () => Effect.Effect; } >()("t3/relay/AgentAwarenessRelay") {} export function eventThreadId(event: OrchestrationV2DomainEvent): ThreadId { return event.threadId; } export function shouldPublishAgentAwarenessEvent( event: Pick, ): boolean { // projectThreadAwarenessV2 reads thread metadata, run status, and pending requests. // Message bodies and tool progress cannot change the published activity. switch (event.type) { case "thread.created": case "thread.archived": case "thread.unarchived": case "thread.deleted": case "thread.metadata-updated": case "thread.model-selection-updated": case "thread.provider-switched": case "run.created": case "run.updated": case "runtime-request.updated": return true; case "thread.settled": case "thread.unsettled": case "thread.snoozed": case "thread.unsnoozed": case "thread.pinned": case "thread.unpinned": case "thread.pin-reordered": case "thread.visited": case "thread.marked-unread": case "thread.runtime-mode-updated": case "thread.interaction-mode-updated": case "run-attempt.created": case "run-attempt.updated": case "node.updated": case "subagent.updated": case "provider-session.attached": case "provider-session.updated": case "provider-session.detached": case "provider-thread.updated": case "provider-turn.updated": case "message.updated": case "turn-item.updated": case "plan.updated": case "checkpoint-scope.created": case "checkpoint.captured": case "checkpoint.rollback-requested": case "context-handoff.updated": case "context-transfer.created": case "context-transfer.updated": return false; } } export const makeAgentAwarenessPublishWorker = Effect.fnUntraced(function* ( publish: (threadId: ThreadId) => Effect.Effect, ) { const queued = new Set(); const worker = yield* makeDrainableWorker((threadId: ThreadId) => Effect.sync(() => queued.delete(threadId)).pipe(Effect.andThen(publish(threadId))), ); const enqueue = (threadId: ThreadId) => Effect.suspend(() => { if (queued.has(threadId)) return Effect.void; // Removing the ID when processing starts allows one new queue entry // for updates received while the current snapshot is being published. queued.add(threadId); return worker.enqueue(threadId); }).pipe(Effect.uninterruptible); return { enqueue, drain: worker.drain }; }); export function agentAwarenessPublishIdentity(state: RelayAgentActivityState | null): string { if (state === null) { return "null"; } const { updatedAt: _updatedAt, ...meaningfulState } = state; return JSON.stringify(meaningfulState); } export function isAgentActivityPublishingEnabled(value: string | null): boolean { return isAgentActivityPublishingEnabledValue(value); } export function resolveAgentActivityPublishingStartupState(input: { readonly relayConfigured: boolean; readonly publishEnabled: boolean; }): "waiting-for-link" | "disabled" | "enabled" { if (!input.relayConfigured) { return "waiting-for-link"; } return input.publishEnabled ? "enabled" : "disabled"; } const RELAY_AGENT_ACTIVITY_DETAIL_MAX_LENGTH = 160; const REDACTED_RELAY_AGENT_FAILURE_DETAIL = "The agent run failed."; const RELAY_AGENT_ACTIVITY_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000] as const; export function sanitizeRelayAgentActivityState( state: RelayAgentActivityState | null, ): RelayAgentActivityState | null { if (state === null) { return null; } const { detail: _detail, ...rest } = state; const detail = (state.phase === "failed" ? REDACTED_RELAY_AGENT_FAILURE_DETAIL : state.detail) ?.trim() .slice(0, RELAY_AGENT_ACTIVITY_DETAIL_MAX_LENGTH) .trim(); return detail ? { ...rest, detail } : rest; } function relayEnvironmentClient(token: string) { return HttpClient.mapRequest(HttpClientRequest.setHeader("authorization", `Bearer ${token}`)); } function deliveryStats( deliveries: ReadonlyArray<{ readonly ok: boolean; readonly queued?: boolean | undefined; readonly kind: string; readonly apnsStatus?: number | null; readonly apnsReason?: string | null; }>, ) { let queued = 0; let successful = 0; let failed = 0; const failedReasons: string[] = []; const kinds = new Set(); for (const delivery of deliveries) { kinds.add(delivery.kind); if (delivery.queued) { queued += 1; continue; } if (delivery.ok) { successful += 1; continue; } failed += 1; failedReasons.push(`${delivery.apnsStatus ?? "transport"}:${delivery.apnsReason ?? "unknown"}`); } return { total: deliveries.length, queued, successful, failed, kinds: [...kinds], failedReasons, }; } export function signRelayAgentActivityPublishProof(input: { readonly privateKey: string; readonly payload: RelayAgentActivityPublishProofPayload; }) { return signRelayJwt({ privateKey: input.privateKey, typ: RELAY_ACTIVITY_PUBLISH_TYP, payload: input.payload, }); } const makePublishProof = Effect.fn("makePublishProof")(function* (input: { readonly privateKey: string; readonly relayIssuer: string; readonly environmentId: string; readonly threadId: ThreadId; readonly state: RelayAgentActivityState | null; readonly jti: string; }) { const now = yield* DateTime.now; const expiresAt = DateTime.add(now, { minutes: 5 }); const payload = { iss: `t3-env:${input.environmentId}`, aud: normalizeRelayIssuer(input.relayIssuer), sub: input.environmentId, jti: input.jti, iat: Math.floor(now.epochMilliseconds / 1_000), exp: Math.floor(expiresAt.epochMilliseconds / 1_000), environmentId: input.environmentId as RelayAgentActivityPublishProofPayload["environmentId"], threadId: input.threadId, state: input.state, } satisfies RelayAgentActivityPublishProofPayload; return yield* signRelayAgentActivityPublishProof({ privateKey: input.privateKey, payload }); }); // Compact, log-safe view of the fields the awareness phase ladder reads. export function describeThreadShellForAwareness( thread: Option.Option, ): Record { if (Option.isNone(thread)) { return { found: false }; } const shell = thread.value; return { found: true, status: shell.status, activityRunStatus: shell.activityRunStatus ?? null, activeRunId: shell.activeRunId ?? null, latestRunId: shell.latestRunId ?? null, pendingRuntimeRequestKind: shell.pendingRuntimeRequest?.kind ?? null, hasActionableProposedPlan: shell.hasActionableProposedPlan, }; } export function resolveAgentAwarenessRelayPublishSnapshot(input: { readonly environmentId: EnvironmentId; readonly threadId: ThreadId; readonly thread: Option.Option; readonly project: Option.Option; }): { readonly projectId: string | null; readonly state: RelayAgentActivityState | null; readonly reason: "snapshot" | "thread-not-found" | "project-not-found"; } { if (Option.isNone(input.thread)) { return { projectId: null, state: null, reason: "thread-not-found", }; } if (Option.isNone(input.project)) { return { projectId: input.thread.value.projectId, state: null, reason: "project-not-found", }; } return { projectId: input.thread.value.projectId, state: sanitizeRelayAgentActivityState( projectThreadAwarenessV2({ environmentId: input.environmentId, project: input.project.value, thread: input.thread.value, }), ), reason: "snapshot", }; } export function resolveAgentAwarenessRelayActiveThreadIds(input: { readonly environmentId: EnvironmentId; readonly projects: ReadonlyArray>; readonly threads: ReadonlyArray; }): ReadonlyArray { const projectById = new Map(input.projects.map((project) => [project.id, project])); return input.threads .filter((thread) => { const project = projectById.get(thread.projectId); if (!project) { return false; } return ( projectThreadAwarenessV2({ environmentId: input.environmentId, project, thread, }) !== null ); }) .map((thread) => thread.id); } export const make = Effect.gen(function* () { const secrets = yield* ServerSecretStore.ServerSecretStore; const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; const threads = yield* ThreadManagement.ThreadManagementService; const projects = yield* ProjectService.ProjectService; const crypto = yield* Crypto.Crypto; const scope = yield* Effect.scope; const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets); const activeSnapshotPublishedRef = yield* Ref.make(false); const publishedStateByThreadRef = yield* Ref.make(new Map()); const readSecretString = (name: string) => secrets .get(name) .pipe( Effect.map((bytes) => Option.isSome(bytes) ? new TextDecoder().decode(bytes.value) : null, ), ); const readRelayConfig = Effect.gen(function* () { const [url, issuer, environmentCredential] = yield* Effect.all([ readSecretString(RELAY_URL_SECRET), readSecretString(RELAY_ISSUER_SECRET), readSecretString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), ]); return url && environmentCredential ? { url, issuer: issuer ?? url, environmentCredential } : null; }); const readPublishAgentActivityEnabled = readSecretString(PUBLISH_AGENT_ACTIVITY_SECRET).pipe( Effect.map(isAgentActivityPublishingEnabled), ); const makeRelayClient = (relayConfig: { readonly url: string; readonly environmentCredential: string; }) => HttpApiClient.make(RelayApi, { baseUrl: relayConfig.url, transformClient: relayEnvironmentClient(relayConfig.environmentCredential), }).pipe(Effect.provide(FetchHttpClient.layer)); let cachedRelayClient: Effect.Success> | undefined; let publishedRelayConfig: NonNullable> | undefined; // Deadlines for publishes that need confirmation (tombstones and // first-state completions). The confirming publish is re-enqueued through // the same drainable worker as every other publish, so a confirmed // tombstone can never race an in-flight live update; a recovered state // clears the deadline. Assigned after the worker exists. const publishConfirmDeadlines = new Map(); let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect = () => Effect.void; const publishRetries = new Map< ThreadId, { readonly attempts: number; timer: Fiber.Fiber | undefined } >(); const cancelPublishRetry = Effect.fnUntraced(function* (threadId: ThreadId) { const retry = publishRetries.get(threadId); publishRetries.delete(threadId); if (retry?.timer !== undefined) yield* Fiber.interrupt(retry.timer); }); const cancelPublishRetries = Effect.suspend(() => Effect.forEach([...publishRetries.keys()], cancelPublishRetry, { discard: true }), ); let schedulePublishRetry: (threadId: ThreadId) => Effect.Effect = () => Effect.void; const resetPublishedConnection = Effect.gen(function* () { publishedRelayConfig = undefined; cachedRelayClient = undefined; publishConfirmDeadlines.clear(); yield* Ref.set(publishedStateByThreadRef, new Map()); }); const publishThreadUnsafe = Effect.fn("publishThreadUnsafe")(function* (threadId: ThreadId) { const publishAgentActivity = yield* readPublishAgentActivityEnabled; if (!publishAgentActivity) { yield* cancelPublishRetries; yield* resetPublishedConnection; yield* Effect.logDebug("agent activity publish skipped; publication disabled", { threadId, }); return; } const relayConfig = yield* readRelayConfig; if (!relayConfig) { yield* cancelPublishRetries; yield* resetPublishedConnection; yield* Effect.logDebug("agent activity publish skipped; relay link credentials unavailable", { threadId, }); return; } if ( publishedRelayConfig?.url !== relayConfig.url || publishedRelayConfig.issuer !== relayConfig.issuer || publishedRelayConfig.environmentCredential !== relayConfig.environmentCredential ) { yield* resetPublishedConnection; publishedRelayConfig = relayConfig; } const environmentId = yield* serverEnvironment.getEnvironmentId; const publishState = (input: { readonly projectId: string | null; readonly state: RelayAgentActivityState | null; readonly reason: string; }) => Effect.gen(function* () { if (cachedRelayClient === undefined) { cachedRelayClient = yield* makeRelayClient(relayConfig); } const relayClient = cachedRelayClient; const proof = yield* makePublishProof({ privateKey: cloudLinkKeyPair.privateKey, relayIssuer: relayConfig.issuer, environmentId, threadId, state: input.state, jti: yield* crypto.randomUUIDv4, }); yield* Effect.logInfo("publishing agent activity for thread", { environmentId, threadId, projectId: input.projectId, statePhase: input.state?.phase ?? null, hasState: input.state !== null, reason: input.reason, }); const response = yield* relayClient.server.publishAgentActivity({ params: { environmentId, threadId, }, payload: { state: input.state, proof, }, }); yield* Effect.logInfo("agent activity publish completed", { environmentId, threadId, ok: response.ok, deliveries: deliveryStats(response.deliveries), }); }); // Per-thread shell read: this publish runs for every activity-relevant // domain event, so materializing the full shell here would make the cost // of one thread's activity proportional to how many threads exist. const threadShell = yield* threads.getThreadShell(threadId); const thread = threadShell === null || threadShell.archivedAt !== null ? Option.none() : Option.some(threadShell); const project = Option.isSome(thread) ? yield* projects.getById(thread.value.projectId) : Option.none(); const snapshot = resolveAgentAwarenessRelayPublishSnapshot({ environmentId, threadId, thread, project, }); const publishIdentity = agentAwarenessPublishIdentity(snapshot.state); const publishedStateByThread = yield* Ref.get(publishedStateByThreadRef); if (publishedStateByThread.get(threadId) === publishIdentity) { // The projection is back at (or never left) the last published state, so // any pending deferred confirmation is moot. Leaving the deadline in // place would let a much later transient null find it already expired // and publish a tombstone immediately, skipping the deferral window. publishConfirmDeadlines.delete(threadId); yield* Effect.logDebug("agent activity publish skipped; projected state unchanged", { environmentId, threadId, reason: snapshot.reason, }); return; } // Two projections need confirmation before publishing, because both can // appear transiently while the projector is mid-write and publishing them // immediately is destructive or noisy: // - null (tombstone) while the previous published state was live: deletes // the thread from every armed card mid-conversation. // - completed as the thread's FIRST published state: sessions boot at // "ready" before their first turn, which projects as completed for an // instant and sends a spurious Done notification at thread birth. // Defer, schedule a re-publish through the ordinary worker queue, and // only publish if the projection still holds when it drains. const requiresConfirmation = (snapshot.state === null && publishedStateByThread.get(threadId) !== agentAwarenessPublishIdentity(null)) || (snapshot.state?.phase === "completed" && !publishedStateByThread.has(threadId)); if (requiresConfirmation) { const nowMs = (yield* DateTime.now).epochMilliseconds; const deadline = publishConfirmDeadlines.get(threadId); if (deadline === undefined) { publishConfirmDeadlines.set(threadId, nowMs + 5_000); yield* Effect.logInfo("agent activity publish deferred pending confirmation", { environmentId, threadId, reason: snapshot.reason, statePhase: snapshot.state?.phase ?? null, shell: describeThreadShellForAwareness(thread), }); yield* schedulePublishConfirm(threadId); return; } if (nowMs < deadline) { return; } yield* Effect.logInfo("agent activity deferred publish confirmed", { environmentId, threadId, reason: snapshot.reason, statePhase: snapshot.state?.phase ?? null, shell: describeThreadShellForAwareness(thread), }); } else { publishConfirmDeadlines.delete(threadId); } if (snapshot.reason === "thread-not-found") { yield* Effect.logDebug("publishing agent activity tombstone; thread not found", { environmentId, threadId, }); } else if (snapshot.reason === "project-not-found") { yield* Effect.logDebug("publishing agent activity tombstone; project not found", { environmentId, threadId, projectId: snapshot.projectId, }); } yield* publishState({ projectId: snapshot.projectId, state: snapshot.state, reason: snapshot.reason, }); publishConfirmDeadlines.delete(threadId); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => { const nextPublishedStates = new Map(publishedStates); nextPublishedStates.set(threadId, publishIdentity); return nextPublishedStates; }); }); const processThreadPublish = Effect.fnUntraced(function* (threadId: ThreadId) { const retry = publishRetries.get(threadId); if (retry?.timer !== undefined) { const timer = retry.timer; retry.timer = undefined; yield* Fiber.interrupt(timer); } yield* publishThreadUnsafe(threadId).pipe( Effect.tap(() => cancelPublishRetry(threadId)), Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) return Effect.void; return Effect.logWarning("agent activity publish failed", { threadId, cause: Cause.pretty(cause), }).pipe(Effect.andThen(schedulePublishRetry(threadId))); }), Effect.withSpan("AgentAwarenessRelay.publishThread"), withRelayClientTracing, ); }); const worker = yield* makeAgentAwarenessPublishWorker(processThreadPublish); const enqueueThreadPublish = (threadId: ThreadId) => cancelPublishRetry(threadId).pipe(Effect.andThen(worker.enqueue(threadId))); const publishThread: AgentAwarenessRelay["Service"]["publishThread"] = (threadId) => enqueueThreadPublish(threadId).pipe(Effect.andThen(worker.drain)); schedulePublishRetry = Effect.fnUntraced(function* (threadId: ThreadId) { const attempts = publishRetries.get(threadId)?.attempts ?? 0; const delayMs = RELAY_AGENT_ACTIVITY_RETRY_DELAYS_MS[attempts]; if (delayMs === undefined) { // Keep the exhausted budget until fresh activity or a successful publish // clears it; an old confirmation timer must not start another retry series. yield* Effect.logWarning("agent activity publish retry budget exhausted", { threadId }); return; } const retry = { attempts: attempts + 1, timer: undefined as Fiber.Fiber | undefined, }; publishRetries.set(threadId, retry); const timer = yield* Effect.sleep(delayMs).pipe( Effect.andThen( Effect.suspend(() => { if (publishRetries.get(threadId) !== retry) return Effect.void; retry.timer = undefined; return worker.enqueue(threadId); }), ), Effect.forkIn(scope), ); if (publishRetries.get(threadId) !== retry) { yield* Fiber.interrupt(timer); } else { retry.timer = timer; } }); const publishActiveThreadsUnsafe = Effect.gen(function* () { const publishAgentActivity = yield* readPublishAgentActivityEnabled.pipe( Effect.orElseSucceed(() => false), ); if (!publishAgentActivity) { yield* Effect.logDebug("agent activity snapshot skipped; publication disabled"); return false; } const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); if (!relayConfig) { yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable"); return false; } const environmentId = yield* serverEnvironment.getEnvironmentId; const [projectSnapshot, shellSnapshot] = yield* Effect.all([ projects.snapshot, threads.getShellSnapshot(), ]); const activeThreadIds = resolveAgentAwarenessRelayActiveThreadIds({ environmentId, projects: projectSnapshot.projects, threads: shellSnapshot.threads, }); if (activeThreadIds.length === 0) { yield* Effect.logDebug("agent activity snapshot has no publishable threads"); return true; } yield* Effect.logInfo("publishing active agent activity snapshot", { count: activeThreadIds.length, }); yield* Effect.forEach(activeThreadIds, enqueueThreadPublish, { discard: true }); yield* worker.drain; return true; }); const publishActiveThreadsOnceWhenConfigured = (logEnabledWhenReady: boolean) => Effect.gen(function* () { while (!(yield* Ref.get(activeSnapshotPublishedRef))) { const published = yield* publishActiveThreadsUnsafe.pipe(Effect.orElseSucceed(() => false)); if (published) { yield* Ref.set(activeSnapshotPublishedRef, true); if (logEnabledWhenReady) { const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); yield* Effect.logInfo("agent activity publishing enabled after link reconciliation", { relayUrl: relayConfig?.url, }); } return; } yield* Effect.sleep("5 seconds"); } }); schedulePublishConfirm = (threadId) => Effect.forkIn( Effect.sleep("5 seconds").pipe( Effect.andThen(worker.enqueue(threadId)), Effect.catchCause((cause) => Effect.logWarning("deferred agent activity confirmation failed", { threadId, cause: Cause.pretty(cause), }), ), ), scope, ).pipe(Effect.asVoid); const start: AgentAwarenessRelay["Service"]["start"] = Effect.fn("AgentAwarenessRelay.start")( function* () { const [relayConfig, publishEnabled] = yield* Effect.all([ readRelayConfig.pipe(Effect.orElseSucceed(() => null)), readPublishAgentActivityEnabled.pipe(Effect.orElseSucceed(() => false)), ]); const startupState = resolveAgentActivityPublishingStartupState({ relayConfigured: relayConfig !== null, publishEnabled, }); switch (startupState) { case "waiting-for-link": yield* Effect.logInfo( "agent activity publishing standby; waiting for T3 Connect link reconciliation", ); break; case "disabled": yield* Effect.logInfo("agent activity publishing disabled by T3 Connect configuration"); break; case "enabled": yield* Effect.logInfo("agent activity publishing enabled", { relayUrl: relayConfig?.url, }); break; } yield* forkParked( Effect.sleep("1 second").pipe( Effect.andThen(publishActiveThreadsOnceWhenConfigured(startupState !== "enabled")), ), ); yield* forkParked( Stream.runForEach(threads.streamDomainEvents, (event) => { const threadId = eventThreadId(event); if (!shouldPublishAgentAwarenessEvent(event)) { return Effect.void; } return Effect.logDebug("agent activity publishing queued thread publish", { eventType: event.type, threadId, }).pipe(Effect.andThen(enqueueThreadPublish(threadId))); }), ); }, ); return AgentAwarenessRelay.of({ publishThread, drain: worker.drain, start, }); }); export const layer = Layer.effect(AgentAwarenessRelay, make);