History

feat(server): measure provider turn token usage (#9132)

Theo Browne committed 2 months ago

Commit 1587f241 parent dab5f6e14 files · +3865 −20Browse files

Unified · Split

14 files
  1. apps/server/src/provider/Layers/ClaudeAdapter.test.ts +250 −6
  2. apps/server/src/provider/Layers/ClaudeAdapter.ts +94 −0
  3. apps/server/src/provider/Layers/CodexAdapter.test.ts +446 −0
  4. apps/server/src/provider/Layers/CodexAdapter.ts +253 −2
  5. apps/server/src/provider/Layers/OpenCodeAdapter.test.ts +745 −0
  6. apps/server/src/provider/Layers/OpenCodeAdapter.ts +152 −4
  7. apps/server/src/provider/Layers/ProviderService.test.ts +1237 −4
  8. apps/server/src/provider/Layers/ProviderService.ts +484 −1
  9. apps/server/src/telemetry/AnalyticsService.test.ts +43 −0
  10. docs/README.md +1 −0
  11. docs/internals/product-analytics.md +68 −3
  12. docs/user/telemetry.md +15 −0
  13. packages/contracts/src/providerRuntime.test.ts +46 −0
  14. packages/contracts/src/providerRuntime.ts +31 −0
apps/server/src/provider/Layers/ClaudeAdapter.test.ts+250−6
@@ -24,6 +24,7 @@ import { · 23 unmodified lines
2424 import { createModelSelection } from "@t3tools/shared/model";
2525 import { assert, describe, it } from "@effect/vitest";
2626 import * as Context from "effect/Context";
27+import * as Deferred from "effect/Deferred";
2728 import * as Effect from "effect/Effect";
2829 import * as Fiber from "effect/Fiber";
2930 import * as Layer from "effect/Layer";
@@ -1899,7 +1900,8 @@ describe("ClaudeAdapterLive", () => { · 1869 unmodified lines
18991900 return Effect.gen(function* () {
19001901 const adapter = yield* ClaudeAdapter;
19011902
1902− const runtimeEventsFiber = yield* Stream.take(adapter.streamEvents, 8).pipe(
1903+ const runtimeEventsFiber = yield* adapter.streamEvents.pipe(
1904+ Stream.takeUntil((event) => event.type === "turn.completed"),
19031905 Stream.runCollect,
19041906 Effect.forkChild,
19051907 );
@@ -1937,6 +1939,17 @@ describe("ClaudeAdapterLive", () => { · 31 unmodified lines
19371939 },
19381940 } as unknown as SDKMessage);
19391941
1942+ harness.query.emit({
1943+ type: "system",
1944+ subtype: "task_started",
1945+ task_id: "task-agent-1",
1946+ description: "Review the database layer",
1947+ task_type: "local_agent",
1948+ tool_use_id: "tool-task-1",
1949+ uuid: "task-agent-1-uuid",
1950+ session_id: "sdk-session-task",
1951+ } as unknown as SDKMessage);
1952+
19401953 harness.query.emit({
19411954 type: "assistant",
19421955 session_id: "sdk-session-task",
@@ -1953,6 +1966,12 @@ describe("ClaudeAdapterLive", () => { · 10 unmodified lines
19531966 subtype: "success",
19541967 is_error: false,
19551968 errors: [],
1969+ usage: {
1970+ input_tokens: 100,
1971+ cache_read_input_tokens: 40,
1972+ cache_creation_input_tokens: 10,
1973+ output_tokens: 20,
1974+ },
19561975 session_id: "sdk-session-task",
19571976 uuid: "result-task-1",
19581977 } as unknown as SDKMessage);
@@ -1964,6 +1983,11 @@ describe("ClaudeAdapterLive", () => { · 5 unmodified lines
19641983 assert.equal(toolStarted.payload.itemType, "collab_agent_tool_call");
19651984 assert.equal(toolStarted.payload.title, "Subagent task");
19661985 }
1986+ const completed = runtimeEvents.find((event) => event.type === "turn.completed");
1987+ assert.equal(completed?.type, "turn.completed");
1988+ if (completed?.type === "turn.completed") {
1989+ assert.equal(completed.payload.tokenUsage?.hasSubagents, true);
1990+ }
19671991 }).pipe(
19681992 Effect.provideService(Random.Random, makeDeterministicRandomService()),
19691993 Effect.provide(harness.layer),
@@ -1975,7 +1999,8 @@ describe("ClaudeAdapterLive", () => { · 5 unmodified lines
19751999 return Effect.gen(function* () {
19762000 const adapter = yield* ClaudeAdapter;
19772001
1978− const runtimeEventsFiber = yield* Stream.take(adapter.streamEvents, 6).pipe(
2002+ const runtimeEventsFiber = yield* adapter.streamEvents.pipe(
2003+ Stream.takeUntil((event) => event.type === "turn.completed"),
19792004 Stream.runCollect,
19802005 Effect.forkChild,
19812006 );
@@ -1998,6 +2023,12 @@ describe("ClaudeAdapterLive", () => { · 16 unmodified lines
19982023 is_error: false,
19992024 errors: ["Error: Request was aborted."],
20002025 stop_reason: "tool_use",
2026+ usage: {
2027+ input_tokens: 12,
2028+ cache_read_input_tokens: 3,
2029+ cache_creation_input_tokens: 1,
2030+ output_tokens: 4,
2031+ },
20012032 session_id: "sdk-session-abort",
20022033 uuid: "result-abort",
20032034 } as unknown as SDKMessage);
@@ -2011,6 +2042,7 @@ describe("ClaudeAdapterLive", () => { · 7 unmodified lines
20112042 "session.state.changed",
20122043 "turn.started",
20132044 "thread.started",
2045+ "thread.token-usage.updated",
20142046 "turn.completed",
20152047 ],
20162048 );
@@ -2022,6 +2054,15 @@ describe("ClaudeAdapterLive", () => { · 5 unmodified lines
20222054 assert.equal(turnCompleted.payload.state, "interrupted");
20232055 assert.equal(turnCompleted.payload.errorMessage, "Error: Request was aborted.");
20242056 assert.equal(turnCompleted.payload.stopReason, "tool_use");
2057+ assert.deepEqual(turnCompleted.payload.tokenUsage, {
2058+ usageStatus: "partial",
2059+ usageScope: "main_agent",
2060+ inputTokens: 16,
2061+ cachedInputTokens: 3,
2062+ cacheCreationTokens: 1,
2063+ outputTokens: 4,
2064+ hasSubagents: false,
2065+ });
20252066 }
20262067 }).pipe(
20272068 Effect.provideService(Random.Random, makeDeterministicRandomService()),
@@ -2509,7 +2550,10 @@ describe("ClaudeAdapterLive", () => { · 481 unmodified lines
25092550 session_id: "sdk-session-result-usage",
25102551 usage: {
25112552 input_tokens: 400,
2553+ cache_read_input_tokens: 90,
2554+ cache_creation_input_tokens: 10,
25122555 output_tokens: 50,
2556+ output_tokens_details: { thinking_tokens: 30 },
25132557 },
25142558 modelUsage: {
25152559 [SYNTHETIC_CLAUDE_CAPABLE_MODEL]: {
@@ -2527,16 +2571,211 @@ describe("ClaudeAdapterLive", () => { · 11 unmodified lines
25272571 assert.deepEqual(usageEvent.payload.usage, {
25282572 usedTokens: 200,
25292573 lastUsedTokens: 200,
2530− totalProcessedTokens: 450,
2574+ totalProcessedTokens: 550,
25312575 inputTokens: 180,
25322576 outputTokens: 20,
25332577 maxTokens: 200000,
25342578 });
25352579 }
2536− assert.equal(
2537− runtimeEvents.find((event) => event.type === "turn.completed")?.type,
2538− "turn.completed",
2580+ const completed = runtimeEvents.find((event) => event.type === "turn.completed");
2581+ assert.equal(completed?.type, "turn.completed");
2582+ if (completed?.type === "turn.completed") {
2583+ assert.deepEqual(completed.payload.tokenUsage, {
2584+ usageStatus: "complete",
2585+ usageScope: "main_agent",
2586+ inputTokens: 500,
2587+ cachedInputTokens: 90,
2588+ cacheCreationTokens: 10,
2589+ reasoningTokens: 30,
2590+ outputTokens: 50,
2591+ hasSubagents: false,
2592+ });
2593+ }
2594+ }).pipe(
2595+ Effect.provideService(Random.Random, makeDeterministicRandomService()),
2596+ Effect.provide(harness.layer),
2597+ );
2598+ });
2599+
2600+ it.effect("treats omitted Claude cache counters as zero contributions", () => {
2601+ const harness = makeHarness();
2602+ return Effect.gen(function* () {
2603+ const adapter = yield* ClaudeAdapter;
2604+ const completedFiber = yield* adapter.streamEvents.pipe(
2605+ Stream.filter((event) => event.type === "turn.completed"),
2606+ Stream.runHead,
2607+ Effect.forkChild,
25392608 );
2609+ yield* adapter.startSession({
2610+ threadId: THREAD_ID,
2611+ provider: ProviderDriverKind.make("claudeAgent"),
2612+ runtimeMode: "full-access",
2613+ });
2614+ yield* adapter.sendTurn({
2615+ threadId: THREAD_ID,
2616+ input: "uncached turn",
2617+ attachments: [],
2618+ });
2619+ harness.query.emit({
2620+ type: "result",
2621+ subtype: "success",
2622+ is_error: false,
2623+ duration_ms: 100,
2624+ duration_api_ms: 90,
2625+ num_turns: 1,
2626+ result: "done",
2627+ stop_reason: "end_turn",
2628+ session_id: "sdk-session-uncached-usage",
2629+ usage: {
2630+ input_tokens: 42,
2631+ output_tokens: 9,
2632+ },
2633+ } as unknown as SDKMessage);
2634+
2635+ const completed = yield* Fiber.join(completedFiber);
2636+ assert.equal(completed._tag, "Some");
2637+ if (completed._tag === "Some" && completed.value.type === "turn.completed") {
2638+ assert.deepEqual(completed.value.payload.tokenUsage, {
2639+ usageStatus: "complete",
2640+ usageScope: "main_agent",
2641+ inputTokens: 42,
2642+ outputTokens: 9,
2643+ hasSubagents: false,
2644+ });
2645+ }
2646+ }).pipe(
2647+ Effect.provideService(Random.Random, makeDeterministicRandomService()),
2648+ Effect.provide(harness.layer),
2649+ );
2650+ });
2651+
2652+ it.effect("uses per-turn result usage across consecutive Claude turns", () => {
2653+ const harness = makeHarness();
2654+ return Effect.gen(function* () {
2655+ const adapter = yield* ClaudeAdapter;
2656+ const firstTurnSettled = yield* Deferred.make<void>();
2657+ const completedFiber = yield* adapter.streamEvents.pipe(
2658+ Stream.tap((event) =>
2659+ event.type === "session.state.changed" && event.payload.reason === "api_retry:1/2"
2660+ ? Deferred.succeed(firstTurnSettled, undefined).pipe(Effect.asVoid)
2661+ : Effect.void,
2662+ ),
2663+ Stream.filter((event) => event.type === "turn.completed"),
2664+ Stream.take(2),
2665+ Stream.runCollect,
2666+ Effect.forkChild,
2667+ );
2668+
2669+ yield* adapter.startSession({
2670+ threadId: THREAD_ID,
2671+ provider: ProviderDriverKind.make("claudeAgent"),
2672+ runtimeMode: "full-access",
2673+ });
2674+ yield* adapter.sendTurn({
2675+ threadId: THREAD_ID,
2676+ input: "first turn",
2677+ attachments: [],
2678+ });
2679+ harness.query.emit({
2680+ type: "result",
2681+ subtype: "success",
2682+ is_error: false,
2683+ duration_ms: 100,
2684+ duration_api_ms: 90,
2685+ num_turns: 1,
2686+ result: "first",
2687+ stop_reason: "end_turn",
2688+ session_id: "sdk-session-consecutive-usage",
2689+ usage: {
2690+ input_tokens: 100,
2691+ cache_read_input_tokens: 20,
2692+ cache_creation_input_tokens: 5,
2693+ output_tokens: 10,
2694+ },
2695+ modelUsage: {
2696+ "claude-opus-4-6": {
2697+ inputTokens: 10_000,
2698+ outputTokens: 1_000,
2699+ cacheReadInputTokens: 2_000,
2700+ cacheCreationInputTokens: 500,
2701+ webSearchRequests: 0,
2702+ costUSD: 1,
2703+ contextWindow: 200_000,
2704+ maxOutputTokens: 64_000,
2705+ },
2706+ },
2707+ } as unknown as SDKMessage);
2708+ harness.query.emit({
2709+ type: "system",
2710+ subtype: "api_retry",
2711+ attempt: 1,
2712+ max_retries: 2,
2713+ retry_delay_ms: 1,
2714+ error_status: 502,
2715+ error: { type: "api_error" },
2716+ session_id: "sdk-session-consecutive-usage",
2717+ uuid: "consecutive-usage-barrier",
2718+ } as unknown as SDKMessage);
2719+ yield* Deferred.await(firstTurnSettled);
2720+
2721+ yield* adapter.sendTurn({
2722+ threadId: THREAD_ID,
2723+ input: "second turn",
2724+ attachments: [],
2725+ });
2726+ harness.query.emit({
2727+ type: "result",
2728+ subtype: "success",
2729+ is_error: false,
2730+ duration_ms: 80,
2731+ duration_api_ms: 70,
2732+ num_turns: 2,
2733+ result: "second",
2734+ stop_reason: "end_turn",
2735+ session_id: "sdk-session-consecutive-usage",
2736+ usage: {
2737+ input_tokens: 30,
2738+ cache_read_input_tokens: 2,
2739+ cache_creation_input_tokens: 3,
2740+ output_tokens: 7,
2741+ },
2742+ modelUsage: {
2743+ "claude-opus-4-6": {
2744+ inputTokens: 20_000,
2745+ outputTokens: 2_000,
2746+ cacheReadInputTokens: 4_000,
2747+ cacheCreationInputTokens: 1_000,
2748+ webSearchRequests: 0,
2749+ costUSD: 2,
2750+ contextWindow: 200_000,
2751+ maxOutputTokens: 64_000,
2752+ },
2753+ },
2754+ } as unknown as SDKMessage);
2755+
2756+ const completed = Array.from(yield* Fiber.join(completedFiber));
2757+ assert.equal(completed[0]?.type, "turn.completed");
2758+ assert.equal(completed[1]?.type, "turn.completed");
2759+ if (completed[0]?.type === "turn.completed" && completed[1]?.type === "turn.completed") {
2760+ assert.deepEqual(completed[0].payload.tokenUsage, {
2761+ usageStatus: "complete",
2762+ usageScope: "main_agent",
2763+ inputTokens: 125,
2764+ cachedInputTokens: 20,
2765+ cacheCreationTokens: 5,
2766+ outputTokens: 10,
2767+ hasSubagents: false,
2768+ });
2769+ assert.deepEqual(completed[1].payload.tokenUsage, {
2770+ usageStatus: "complete",
2771+ usageScope: "main_agent",
2772+ inputTokens: 35,
2773+ cachedInputTokens: 2,
2774+ cacheCreationTokens: 3,
2775+ outputTokens: 7,
2776+ hasSubagents: false,
2777+ });
2778+ }
25402779 }).pipe(
25412780 Effect.provideService(Random.Random, makeDeterministicRandomService()),
25422781 Effect.provide(harness.layer),
@@ -2941,6 +3180,11 @@ describe("ClaudeAdapterLive", () => { · 398 unmodified lines
29413180 assert.equal(String(turnCompleted.turnId), String(turn.turnId));
29423181 assert.equal(turnCompleted.payload.state, "interrupted");
29433182 assert.equal(turnCompleted.payload.errorMessage, "Claude runtime interrupted.");
3183+ assert.deepEqual(turnCompleted.payload.tokenUsage, {
3184+ usageStatus: "unavailable",
3185+ usageScope: "main_agent",
3186+ hasSubagents: false,
3187+ });
29443188 }
29453189
29463190 const sessionExited = runtimeEvents[5];
apps/server/src/provider/Layers/ClaudeAdapter.ts+94−0
@@ -24,6 +24,7 @@ import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview"; · 23 unmodified lines
2424 import { type ClaudeScopedLimitNames, claudeRateLimitEventToUpdate } from "./claudeUsageLimits.ts";
2525 import {
2626 ApprovalRequestId,
27+ classifyTaskAgentKind,
2728 type CanonicalItemType,
2829 type CanonicalRequestType,
2930 type ClaudeSettings,
@@ -38,6 +39,7 @@ import { · 8 unmodified lines
3839 type ProviderSendTurnInput,
3940 type ProviderSession,
4041 type ThreadTokenUsageSnapshot,
42+ type TurnTokenUsage,
4143 type ProviderUserInputAnswers,
4244 type RuntimeContentStreamKind,
4345 RuntimeItemId,
@@ -154,6 +156,7 @@ interface ClaudeTurnState { · 110 unmodified lines
154156 readonly capturedProposedPlanKeys: Set<string>;
155157 latestAssistantUsage: unknown | undefined;
156158 compactedSinceLatestAssistantUsage: boolean;
159+ hasSubagents: boolean;
157160 nextSyntheticAssistantBlockIndex: number;
158161 }
159162
@@ -643,6 +646,85 @@ function normalizeClaudeActiveTokenUsage( · 483 unmodified lines
643646 });
644647 }
645648
649+function normalizeClaudeTurnTokenUsage(
650+ result: SDKResultMessage | undefined,
651+ hasSubagents: boolean,
652+ terminalStatus: ProviderRuntimeTurnStatus,
653+): TurnTokenUsage {
654+ const usage = result?.usage as Record<string, unknown> | undefined;
655+ if (!usage) {
656+ return {
657+ usageStatus: "unavailable",
658+ usageScope: "main_agent",
659+ hasSubagents,
660+ };
661+ }
662+
663+ const uncachedInputTokens = finiteNonNegativeInteger(usage.input_tokens);
664+ const cachedInputTokens = finiteNonNegativeInteger(usage.cache_read_input_tokens);
665+ const cacheCreationTokens = finiteNonNegativeInteger(usage.cache_creation_input_tokens);
666+ const rawOutputTokens = finiteNonNegativeInteger(usage.output_tokens);
667+ const outputDetails = usage.output_tokens_details as Record<string, unknown> | undefined;
668+ const thinkingTokens = finiteNonNegativeInteger(outputDetails?.thinking_tokens);
669+ const cachedInputContribution = usage.cache_read_input_tokens == null ? 0 : cachedInputTokens;
670+ const cacheCreationContribution =
671+ usage.cache_creation_input_tokens == null ? 0 : cacheCreationTokens;
672+ const inputTokens =
673+ uncachedInputTokens !== undefined &&
674+ cachedInputContribution !== undefined &&
675+ cacheCreationContribution !== undefined
676+ ? uncachedInputTokens + cachedInputContribution + cacheCreationContribution
677+ : undefined;
678+ const hasKnownUsage =
679+ uncachedInputTokens !== undefined ||
680+ cachedInputTokens !== undefined ||
681+ cacheCreationTokens !== undefined ||
682+ rawOutputTokens !== undefined;
683+ const hasPositiveUsage =
684+ (uncachedInputTokens ?? 0) +
685+ (cachedInputTokens ?? 0) +
686+ (cacheCreationTokens ?? 0) +
687+ (rawOutputTokens ?? 0) >
688+ 0;
689+
690+ if (!hasKnownUsage || (result?.subtype !== "success" && !hasPositiveUsage)) {
691+ return {
692+ usageStatus: "unavailable",
693+ usageScope: "main_agent",
694+ hasSubagents,
695+ };
696+ }
697+
698+ const commonUsage = {
699+ usageScope: "main_agent",
700+ ...(cachedInputTokens !== undefined ? { cachedInputTokens } : {}),
701+ ...(cacheCreationTokens !== undefined ? { cacheCreationTokens } : {}),
702+ ...(thinkingTokens !== undefined && rawOutputTokens !== undefined
703+ ? { reasoningTokens: Math.min(rawOutputTokens, thinkingTokens) }
704+ : {}),
705+ hasSubagents,
706+ } as const;
707+ if (
708+ terminalStatus === "completed" &&
709+ result?.subtype === "success" &&
710+ inputTokens !== undefined &&
711+ rawOutputTokens !== undefined
712+ ) {
713+ return {
714+ ...commonUsage,
715+ usageStatus: "complete",
716+ inputTokens,
717+ outputTokens: rawOutputTokens,
718+ };
719+ }
720+ return {
721+ ...commonUsage,
722+ usageStatus: "partial",
723+ ...(inputTokens !== undefined ? { inputTokens } : {}),
724+ ...(rawOutputTokens !== undefined ? { outputTokens: rawOutputTokens } : {}),
725+ };
726+}
727+
646728 function compactBoundaryTokenUsageSnapshot(
647729 message: Record<string, unknown>,
648730 contextWindow?: number,
@@ -2503,6 +2585,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 1854 unmodified lines
25032585 ? { totalCostUsd: result.total_cost_usd }
25042586 : {}),
25052587 ...(errorMessage ? { errorMessage } : {}),
2588+ tokenUsage: normalizeClaudeTurnTokenUsage(result, turnState.hasSubagents, status),
25062589 },
25072590 providerRefs: nativeProviderRefs(context),
25082591 });
@@ -3047,6 +3130,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 538 unmodified lines
30473130 capturedProposedPlanKeys: new Set(),
30483131 latestAssistantUsage: undefined,
30493132 compactedSinceLatestAssistantUsage: false,
3133+ hasSubagents: false,
30503134 nextSyntheticAssistantBlockIndex: -1,
30513135 };
30523136 context.session = {
@@ -3353,6 +3437,15 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 300 unmodified lines
33533437 )
33543438 : undefined;
33553439 const owningAgentId = launchingTool?.agentId;
3440+ if (
3441+ context.turnState &&
3442+ classifyTaskAgentKind({
3443+ taskType: message.task_type,
3444+ ...(owningAgentId ? { agentId: owningAgentId } : {}),
3445+ }) === "agent"
3446+ ) {
3447+ context.turnState.hasSubagents = true;
3448+ }
33563449 // Model/effort: the Agent tool's input carries explicit overrides;
33573450 // absent ones inherit the session's selection (SDK behavior).
33583451 // Subagent assistant snapshots refine model with the authoritative API
@@ -4746,6 +4839,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 1387 unmodified lines
47464839 capturedProposedPlanKeys: new Set(),
47474840 latestAssistantUsage: undefined,
47484841 compactedSinceLatestAssistantUsage: false,
4842+ hasSubagents: false,
47494843 nextSyntheticAssistantBlockIndex: -1,
47504844 };
47514845
apps/server/src/provider/Layers/CodexAdapter.test.ts+446−0
@@ -597,7 +597,453 @@ function startLifecycleRuntime() { · 596 unmodified lines
597597 });
598598 }
599599
600+function codexTokenUsageEvent(input: {
601+ readonly id: string;
602+ readonly turnId: string;
603+ readonly inputTokens: number;
604+ readonly cachedInputTokens: number;
605+ readonly cacheCreationTokens: number;
606+ readonly outputTokens: number;
607+ readonly reasoningTokens: number;
608+ readonly last?: {
609+ readonly inputTokens: number;
610+ readonly cachedInputTokens: number;
611+ readonly cacheCreationTokens: number;
612+ readonly outputTokens: number;
613+ readonly reasoningTokens: number;
614+ };
615+}): ProviderEvent {
616+ const totalTokens = input.inputTokens + input.outputTokens;
617+ const last = input.last ?? input;
618+ return {
619+ id: asEventId(input.id),
620+ kind: "notification",
621+ provider: ProviderDriverKind.make("codex"),
622+ threadId: asThreadId("thread-1"),
623+ turnId: asTurnId(input.turnId),
624+ createdAt: "2026-01-01T00:00:00.000Z",
625+ method: "thread/tokenUsage/updated",
626+ payload: {
627+ threadId: "thread-1",
628+ turnId: input.turnId,
629+ tokenUsage: {
630+ total: {
631+ inputTokens: input.inputTokens,
632+ cachedInputTokens: input.cachedInputTokens,
633+ cacheWriteInputTokens: input.cacheCreationTokens,
634+ outputTokens: input.outputTokens,
635+ reasoningOutputTokens: input.reasoningTokens,
636+ totalTokens,
637+ },
638+ last: {
639+ inputTokens: last.inputTokens,
640+ cachedInputTokens: last.cachedInputTokens,
641+ cacheWriteInputTokens: last.cacheCreationTokens,
642+ outputTokens: last.outputTokens,
643+ reasoningOutputTokens: last.reasoningTokens,
644+ totalTokens: last.inputTokens + last.outputTokens,
645+ },
646+ },
647+ },
648+ };
649+}
650+
651+function codexTurnEvent(method: "turn/started" | "turn/completed", turnId: string): ProviderEvent {
652+ return {
653+ id: asEventId(`evt-${method}-${turnId}`),
654+ kind: "notification",
655+ provider: ProviderDriverKind.make("codex"),
656+ threadId: asThreadId("thread-1"),
657+ turnId: asTurnId(turnId),
658+ createdAt: "2026-01-01T00:00:00.000Z",
659+ method,
660+ payload:
661+ method === "turn/started"
662+ ? {}
663+ : {
664+ threadId: "thread-1",
665+ turn: { id: turnId, items: [], status: "completed" },
666+ },
667+ };
668+}
669+
600670 lifecycleLayer("CodexAdapterLive lifecycle", (it) => {
671+ it.effect("calculates one Codex turn total from cumulative counters", () =>
672+ Effect.gen(function* () {
673+ const { adapter, runtime } = yield* startLifecycleRuntime();
674+ const completedFiber = yield* adapter.streamEvents.pipe(
675+ Stream.filter((event) => event.type === "turn.completed"),
676+ Stream.runHead,
677+ Effect.forkChild,
678+ );
679+
680+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-usage"));
681+ yield* runtime.emit(
682+ codexTokenUsageEvent({
683+ id: "evt-usage-1",
684+ turnId: "turn-usage",
685+ inputTokens: 100,
686+ cachedInputTokens: 40,
687+ cacheCreationTokens: 10,
688+ outputTokens: 20,
689+ reasoningTokens: 8,
690+ }),
691+ );
692+ // Codex can repeat both notifications without new work.
693+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-usage"));
694+ yield* runtime.emit(
695+ codexTokenUsageEvent({
696+ id: "evt-usage-duplicate",
697+ turnId: "turn-usage",
698+ inputTokens: 100,
699+ cachedInputTokens: 40,
700+ cacheCreationTokens: 10,
701+ outputTokens: 20,
702+ reasoningTokens: 8,
703+ }),
704+ );
705+ yield* runtime.emit({
706+ id: asEventId("evt-collab-activity"),
707+ kind: "notification",
708+ provider: ProviderDriverKind.make("codex"),
709+ threadId: asThreadId("thread-1"),
710+ turnId: asTurnId("turn-usage"),
711+ createdAt: "2026-01-01T00:00:00.000Z",
712+ method: "collabAgent/activity",
713+ payload: {
714+ agentThreadId: "child-1",
715+ agentPath: "/root/child-1",
716+ activityKind: "started",
717+ },
718+ });
719+ yield* runtime.emit(
720+ codexTokenUsageEvent({
721+ id: "evt-usage-2",
722+ turnId: "turn-usage",
723+ inputTokens: 150,
724+ cachedInputTokens: 60,
725+ cacheCreationTokens: 15,
726+ outputTokens: 30,
727+ reasoningTokens: 12,
728+ }),
729+ );
730+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-usage"));
731+
732+ const completed = yield* Fiber.join(completedFiber);
733+ NodeAssert.equal(completed._tag, "Some");
734+ if (completed._tag === "Some" && completed.value.type === "turn.completed") {
735+ NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, {
736+ usageStatus: "complete",
737+ usageScope: "main_agent",
738+ inputTokens: 150,
739+ cachedInputTokens: 60,
740+ cacheCreationTokens: 15,
741+ outputTokens: 30,
742+ reasoningTokens: 12,
743+ hasSubagents: true,
744+ });
745+ }
746+ }),
747+ );
748+
749+ it.effect("does not charge a late prior-turn update to the next Codex turn", () =>
750+ Effect.gen(function* () {
751+ const { adapter, runtime } = yield* startLifecycleRuntime();
752+ const completedFiber = yield* adapter.streamEvents.pipe(
753+ Stream.filter((event) => event.type === "turn.completed"),
754+ Stream.take(2),
755+ Stream.runCollect,
756+ Effect.forkChild,
757+ );
758+
759+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-first"));
760+ yield* runtime.emit(
761+ codexTokenUsageEvent({
762+ id: "evt-late-1",
763+ turnId: "turn-first",
764+ inputTokens: 100,
765+ cachedInputTokens: 40,
766+ cacheCreationTokens: 10,
767+ outputTokens: 20,
768+ reasoningTokens: 8,
769+ }),
770+ );
771+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-first"));
772+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-second"));
773+ // A late update for the finished turn lands after the next turn starts.
774+ yield* runtime.emit(
775+ codexTokenUsageEvent({
776+ id: "evt-late-2",
777+ turnId: "turn-first",
778+ inputTokens: 150,
779+ cachedInputTokens: 60,
780+ cacheCreationTokens: 15,
781+ outputTokens: 30,
782+ reasoningTokens: 12,
783+ }),
784+ );
785+ yield* runtime.emit(
786+ codexTokenUsageEvent({
787+ id: "evt-late-3",
788+ turnId: "turn-second",
789+ inputTokens: 170,
790+ cachedInputTokens: 65,
791+ cacheCreationTokens: 16,
792+ outputTokens: 35,
793+ reasoningTokens: 14,
794+ }),
795+ );
796+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-second"));
797+
798+ const completed = Array.from(yield* Fiber.join(completedFiber));
799+ const second = completed[1];
800+ NodeAssert.equal(second?.type, "turn.completed");
801+ if (second?.type === "turn.completed") {
802+ NodeAssert.deepStrictEqual(second.payload.tokenUsage, {
803+ usageStatus: "complete",
804+ usageScope: "main_agent",
805+ inputTokens: 20,
806+ cachedInputTokens: 5,
807+ cacheCreationTokens: 1,
808+ outputTokens: 5,
809+ reasoningTokens: 2,
810+ hasSubagents: false,
811+ });
812+ }
813+ }),
814+ );
815+
816+ it.effect("clamps Codex cache and reasoning subsets to their totals", () =>
817+ Effect.gen(function* () {
818+ const { adapter, runtime } = yield* startLifecycleRuntime();
819+ const completedFiber = yield* adapter.streamEvents.pipe(
820+ Stream.filter((event) => event.type === "turn.completed"),
821+ Stream.runHead,
822+ Effect.forkChild,
823+ );
824+
825+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-clamp"));
826+ yield* runtime.emit(
827+ codexTokenUsageEvent({
828+ id: "evt-clamp-1",
829+ turnId: "turn-clamp",
830+ inputTokens: 100,
831+ cachedInputTokens: 140,
832+ cacheCreationTokens: 120,
833+ outputTokens: 20,
834+ reasoningTokens: 30,
835+ }),
836+ );
837+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-clamp"));
838+
839+ const completed = yield* Fiber.join(completedFiber);
840+ NodeAssert.equal(completed._tag, "Some");
841+ if (completed._tag === "Some" && completed.value.type === "turn.completed") {
842+ NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, {
843+ usageStatus: "complete",
844+ usageScope: "main_agent",
845+ inputTokens: 100,
846+ cachedInputTokens: 100,
847+ cacheCreationTokens: 100,
848+ outputTokens: 20,
849+ reasoningTokens: 20,
850+ hasSubagents: false,
851+ });
852+ }
853+ }),
854+ );
855+
856+ it.effect("counts the last response when Codex resets its running total mid-turn", () =>
857+ Effect.gen(function* () {
858+ const { adapter, runtime } = yield* startLifecycleRuntime();
859+ const completedFiber = yield* adapter.streamEvents.pipe(
860+ Stream.filter((event) => event.type === "turn.completed"),
861+ Stream.runHead,
862+ Effect.forkChild,
863+ );
864+
865+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-reset"));
866+ yield* runtime.emit(
867+ codexTokenUsageEvent({
868+ id: "evt-reset-1",
869+ turnId: "turn-reset",
870+ inputTokens: 5_000,
871+ cachedInputTokens: 4_000,
872+ cacheCreationTokens: 100,
873+ outputTokens: 500,
874+ reasoningTokens: 200,
875+ last: {
876+ inputTokens: 100,
877+ cachedInputTokens: 80,
878+ cacheCreationTokens: 10,
879+ outputTokens: 20,
880+ reasoningTokens: 8,
881+ },
882+ }),
883+ );
884+ // Codex restarted its cumulative total. The new total is smaller than
885+ // the previous one, so only `last` is counted for this update.
886+ yield* runtime.emit(
887+ codexTokenUsageEvent({
888+ id: "evt-reset-2",
889+ turnId: "turn-reset",
890+ inputTokens: 150,
891+ cachedInputTokens: 90,
892+ cacheCreationTokens: 5,
893+ outputTokens: 30,
894+ reasoningTokens: 12,
895+ }),
896+ );
897+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-reset"));
898+
899+ const completed = yield* Fiber.join(completedFiber);
900+ NodeAssert.equal(completed._tag, "Some");
901+ if (completed._tag === "Some" && completed.value.type === "turn.completed") {
902+ NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, {
903+ usageStatus: "complete",
904+ usageScope: "main_agent",
905+ inputTokens: 250,
906+ cachedInputTokens: 170,
907+ cacheCreationTokens: 15,
908+ outputTokens: 50,
909+ reasoningTokens: 20,
910+ hasSubagents: false,
911+ });
912+ }
913+ }),
914+ );
915+
916+ it.effect("uses the last response usage when no prior Codex total exists", () =>
917+ Effect.gen(function* () {
918+ const adapter = yield* CodexAdapter;
919+ yield* adapter.startSession({
920+ provider: ProviderDriverKind.make("codex"),
921+ threadId: asThreadId("thread-1"),
922+ resumeCursor: { threadId: "provider-thread-1" },
923+ runtimeMode: "full-access",
924+ });
925+ const runtime = lifecycleRuntimeFactory.lastRuntime;
926+ NodeAssert.ok(runtime);
927+ const firstCompletionsFiber = yield* adapter.streamEvents.pipe(
928+ Stream.filter((event) => event.type === "turn.completed"),
929+ Stream.take(2),
930+ Stream.runCollect,
931+ Effect.forkChild,
932+ );
933+
934+ // Resumed thread: the cumulative total already holds old history, so the
935+ // first update must count only `last`.
936+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-resumed"));
937+ yield* runtime.emit(
938+ codexTokenUsageEvent({
939+ id: "evt-resume-baseline",
940+ turnId: "turn-resumed",
941+ inputTokens: 1_000,
942+ cachedInputTokens: 400,
943+ cacheCreationTokens: 100,
944+ outputTokens: 200,
945+ reasoningTokens: 80,
946+ last: {
947+ inputTokens: 300,
948+ cachedInputTokens: 120,
949+ cacheCreationTokens: 30,
950+ outputTokens: 60,
951+ reasoningTokens: 24,
952+ },
953+ }),
954+ );
955+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-resumed"));
956+
957+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-after-resume"));
958+ yield* runtime.emit(
959+ codexTokenUsageEvent({
960+ id: "evt-after-resume",
961+ turnId: "turn-after-resume",
962+ inputTokens: 1_100,
963+ cachedInputTokens: 440,
964+ cacheCreationTokens: 110,
965+ outputTokens: 220,
966+ reasoningTokens: 88,
967+ }),
968+ );
969+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-after-resume"));
970+
971+ const firstCompletions = Array.from(yield* Fiber.join(firstCompletionsFiber));
972+
973+ yield* adapter.rollbackThread(asThreadId("thread-1"), 1);
974+ const rollbackCompletionFiber = yield* adapter.streamEvents.pipe(
975+ Stream.filter((event) => event.type === "turn.completed"),
976+ Stream.runHead,
977+ Effect.forkChild,
978+ );
979+ // Rollback drops the baseline and Codex shrinks its total, so the first
980+ // update after it counts only `last` again.
981+ yield* runtime.emit(codexTurnEvent("turn/started", "turn-after-rollback"));
982+ yield* runtime.emit(
983+ codexTokenUsageEvent({
984+ id: "evt-after-rollback",
985+ turnId: "turn-after-rollback",
986+ inputTokens: 1_050,
987+ cachedInputTokens: 420,
988+ cacheCreationTokens: 105,
989+ outputTokens: 210,
990+ reasoningTokens: 84,
991+ last: {
992+ inputTokens: 50,
993+ cachedInputTokens: 20,
994+ cacheCreationTokens: 5,
995+ outputTokens: 10,
996+ reasoningTokens: 4,
997+ },
998+ }),
999+ );
1000+ yield* runtime.emit(codexTurnEvent("turn/completed", "turn-after-rollback"));
1001+
1002+ const rollbackCompletion = yield* Fiber.join(rollbackCompletionFiber);
1003+ const completions = [
1004+ ...firstCompletions,
1005+ ...(rollbackCompletion._tag === "Some" ? [rollbackCompletion.value] : []),
1006+ ];
1007+ NodeAssert.deepStrictEqual(
1008+ completions.map((event) =>
1009+ event.type === "turn.completed" ? event.payload.tokenUsage : undefined,
1010+ ),
1011+ [
1012+ {
1013+ usageStatus: "complete",
1014+ usageScope: "main_agent",
1015+ inputTokens: 300,
1016+ cachedInputTokens: 120,
1017+ cacheCreationTokens: 30,
1018+ outputTokens: 60,
1019+ reasoningTokens: 24,
1020+ hasSubagents: false,
1021+ },
1022+ {
1023+ usageStatus: "complete",
1024+ usageScope: "main_agent",
1025+ inputTokens: 100,
1026+ cachedInputTokens: 40,
1027+ cacheCreationTokens: 10,
1028+ outputTokens: 20,
1029+ reasoningTokens: 8,
1030+ hasSubagents: false,
1031+ },
1032+ {
1033+ usageStatus: "complete",
1034+ usageScope: "main_agent",
1035+ inputTokens: 50,
1036+ cachedInputTokens: 20,
1037+ cacheCreationTokens: 5,
1038+ outputTokens: 10,
1039+ reasoningTokens: 4,
1040+ hasSubagents: false,
1041+ },
1042+ ],
1043+ );
1044+ }),
1045+ );
1046+
6011047 it.effect("carries child model metadata through every task event", () =>
6021048 Effect.gen(function* () {
6031049 const { adapter, runtime } = yield* startLifecycleRuntime();
apps/server/src/provider/Layers/CodexAdapter.ts+253−2
@@ -26,6 +26,7 @@ import { · 25 unmodified lines
2626 RuntimeRequestId,
2727 RuntimeTaskId,
2828 type RuntimeTaskUsage,
29+ type TurnTokenUsage,
2930 ProviderApprovalDecision,
3031 ThreadId,
3132 ProviderSendTurnInput,
@@ -99,9 +100,34 @@ interface CodexAdapterSessionContext { · 67 unmodified lines
99100 readonly scope: Scope.Closeable;
100101 readonly runtime: CodexSessionRuntimeShape;
101102 readonly eventFiber: Fiber.Fiber<void, never>;
103+ readonly turnTokenUsage: CodexTurnTokenUsageState;
102104 stopped: boolean;
103105 }
104106
107+type CodexCumulativeTokenUsage = {
108+ readonly inputTokens: number;
109+ readonly cachedInputTokens: number;
110+ readonly cacheCreationTokens?: number;
111+ readonly outputTokens: number;
112+ readonly reasoningTokens: number;
113+};
114+
115+interface CodexTurnTokenUsageAccumulator {
116+ inputTokens: number;
117+ cachedInputTokens: number;
118+ cacheCreationTokens: number | undefined;
119+ outputTokens: number;
120+ reasoningTokens: number;
121+ observed: boolean;
122+ hasSubagents: boolean;
123+}
124+
125+interface CodexTurnTokenUsageState {
126+ baseline: CodexCumulativeTokenUsage | undefined;
127+ activeTurnId: string | undefined;
128+ readonly byTurnId: Map<string, CodexTurnTokenUsageAccumulator>;
129+}
130+
105131 function mapCodexRuntimeError(
106132 threadId: ThreadId,
107133 method: string,
@@ -429,6 +455,162 @@ function normalizeCodexTokenUsage( · 321 unmodified lines
429455 };
430456 }
431457
458+function codexTokenUsageBreakdown(
459+ usage: EffectCodexSchema.V2ThreadTokenUsageUpdatedNotification__TokenUsageBreakdown,
460+): CodexCumulativeTokenUsage {
461+ return {
462+ inputTokens: usage.inputTokens,
463+ cachedInputTokens: usage.cachedInputTokens,
464+ ...(usage.cacheWriteInputTokens !== undefined
465+ ? { cacheCreationTokens: usage.cacheWriteInputTokens }
466+ : {}),
467+ outputTokens: usage.outputTokens,
468+ reasoningTokens: usage.reasoningOutputTokens,
469+ };
470+}
471+
472+function makeCodexTurnTokenUsageState(): CodexTurnTokenUsageState {
473+ return {
474+ baseline: undefined,
475+ activeTurnId: undefined,
476+ byTurnId: new Map(),
477+ };
478+}
479+
480+function getCodexTurnAccumulator(
481+ state: CodexTurnTokenUsageState,
482+ turnId: string,
483+): CodexTurnTokenUsageAccumulator {
484+ const existing = state.byTurnId.get(turnId);
485+ if (existing) return existing;
486+ const created: CodexTurnTokenUsageAccumulator = {
487+ inputTokens: 0,
488+ cachedInputTokens: 0,
489+ cacheCreationTokens: 0,
490+ outputTokens: 0,
491+ reasoningTokens: 0,
492+ observed: false,
493+ hasSubagents: false,
494+ };
495+ state.byTurnId.set(turnId, created);
496+ return created;
497+}
498+
499+/**
500+ * Usage added by one `thread/tokenUsage/updated` notification. Codex reports a
501+ * running `total` for the thread and `last`, the usage of the newest model
502+ * response. Within a turn the growth of `total` equals `last`. Without a prior
503+ * total (first update after resume or rollback), or when Codex reset the
504+ * running total, `last` is the delta.
505+ */
506+function codexTurnTokenUsageDelta(
507+ previous: CodexCumulativeTokenUsage | undefined,
508+ current: CodexCumulativeTokenUsage,
509+ last: CodexCumulativeTokenUsage,
510+): CodexCumulativeTokenUsage {
511+ if (
512+ previous === undefined ||
513+ current.inputTokens < previous.inputTokens ||
514+ current.cachedInputTokens < previous.cachedInputTokens ||
515+ current.outputTokens < previous.outputTokens ||
516+ current.reasoningTokens < previous.reasoningTokens
517+ ) {
518+ return last;
519+ }
520+ return {
521+ inputTokens: current.inputTokens - previous.inputTokens,
522+ cachedInputTokens: current.cachedInputTokens - previous.cachedInputTokens,
523+ ...(current.cacheCreationTokens !== undefined &&
524+ previous.cacheCreationTokens !== undefined &&
525+ current.cacheCreationTokens >= previous.cacheCreationTokens
526+ ? { cacheCreationTokens: current.cacheCreationTokens - previous.cacheCreationTokens }
527+ : {}),
528+ outputTokens: current.outputTokens - previous.outputTokens,
529+ reasoningTokens: current.reasoningTokens - previous.reasoningTokens,
530+ };
531+}
532+
533+function accumulateCodexTurnTokenUsage(
534+ state: CodexTurnTokenUsageState,
535+ turnId: string,
536+ usage: EffectCodexSchema.V2ThreadTokenUsageUpdatedNotification["tokenUsage"],
537+): void {
538+ const current = codexTokenUsageBreakdown(usage.total);
539+ if (state.activeTurnId !== turnId) {
540+ // The total is thread-wide, so every update moves the baseline. A late
541+ // update for a finished turn is not counted toward the live turn.
542+ state.baseline = current;
543+ return;
544+ }
545+
546+ const accumulator = getCodexTurnAccumulator(state, turnId);
547+ const delta = codexTurnTokenUsageDelta(
548+ state.baseline,
549+ current,
550+ codexTokenUsageBreakdown(usage.last),
551+ );
552+ state.baseline = current;
553+
554+ if (
555+ delta.inputTokens > 0 ||
556+ delta.cachedInputTokens > 0 ||
557+ delta.outputTokens > 0 ||
558+ delta.reasoningTokens > 0
559+ ) {
560+ accumulator.observed = true;
561+ }
562+ accumulator.inputTokens += delta.inputTokens;
563+ accumulator.cachedInputTokens += delta.cachedInputTokens;
564+ accumulator.outputTokens += delta.outputTokens;
565+ accumulator.reasoningTokens += delta.reasoningTokens;
566+ if (delta.cacheCreationTokens === undefined) {
567+ accumulator.cacheCreationTokens = undefined;
568+ } else if (accumulator.cacheCreationTokens !== undefined) {
569+ accumulator.cacheCreationTokens += delta.cacheCreationTokens;
570+ }
571+}
572+
573+function completeCodexTurnTokenUsage(
574+ state: CodexTurnTokenUsageState,
575+ turnId: string,
576+ completed: boolean,
577+): TurnTokenUsage {
578+ const usage = state.byTurnId.get(turnId);
579+ state.byTurnId.delete(turnId);
580+ if (state.activeTurnId === turnId) state.activeTurnId = undefined;
581+ if (!usage) {
582+ return {
583+ usageStatus: "unavailable",
584+ usageScope: "main_agent",
585+ hasSubagents: false,
586+ };
587+ }
588+
589+ if (!usage.observed) {
590+ return {
591+ usageStatus: "unavailable",
592+ usageScope: "main_agent",
593+ hasSubagents: usage.hasSubagents,
594+ };
595+ }
596+
597+ // Codex counts cache reads and writes inside inputTokens. Clamp the
598+ // subsets so the record keeps the documented relationships even if a
599+ // counter drifts.
600+ return {
601+ usageStatus: completed ? "complete" : "partial",
602+ usageScope: "main_agent",
603+ inputTokens: usage.inputTokens,
604+ cachedInputTokens: Math.min(usage.inputTokens, usage.cachedInputTokens),
605+ ...(usage.cacheCreationTokens !== undefined
606+ ? { cacheCreationTokens: Math.min(usage.inputTokens, usage.cacheCreationTokens) }
607+ : {}),
608+ outputTokens: usage.outputTokens,
609+ reasoningTokens: Math.min(usage.outputTokens, usage.reasoningTokens),
610+ hasSubagents: usage.hasSubagents,
611+ };
612+}
613+
432614 function toTurnStatus(
433615 value: EffectCodexSchema.V2TurnCompletedNotification["turn"]["status"] | "cancelled",
434616 ): "completed" | "failed" | "cancelled" | "interrupted" {
@@ -2063,6 +2245,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( · 1628 unmodified lines
20632245 }
20642246 : {}),
20652247 };
2248+ const turnTokenUsage = makeCodexTurnTokenUsageState();
20662249 const sessionScope = yield* Scope.make("sequential");
20672250 let sessionScopeTransferred = false;
20682251 yield* Effect.addFinalizer(() =>
@@ -2091,7 +2274,64 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( · 22 unmodified lines
20912274 const eventFiber = yield* Stream.runForEach(runtime.events, (event) =>
20922275 Effect.gen(function* () {
20932276 yield* writeNativeEvent(event);
2094− const runtimeEvents = mapToRuntimeEvents(event, event.threadId);
2277+ if (event.method === "turn/started" && event.turnId) {
2278+ if (turnTokenUsage.activeTurnId !== event.turnId) {
2279+ turnTokenUsage.byTurnId.clear();
2280+ turnTokenUsage.activeTurnId = event.turnId;
2281+ getCodexTurnAccumulator(turnTokenUsage, event.turnId);
2282+ }
2283+ } else if (event.method === "thread/tokenUsage/updated") {
2284+ const payload = readPayload(
2285+ EffectCodexSchema.V2ThreadTokenUsageUpdatedNotification,
2286+ event.payload,
2287+ );
2288+ if (payload) {
2289+ accumulateCodexTurnTokenUsage(turnTokenUsage, payload.turnId, payload.tokenUsage);
2290+ }
2291+ } else if (turnTokenUsage.activeTurnId) {
2292+ const collabPayload =
2293+ typeof event.payload === "object" && event.payload !== null
2294+ ? (event.payload as Record<string, unknown>)
2295+ : undefined;
2296+ const isCollabSpawn =
2297+ event.method === "collabAgent/started" ||
2298+ (event.method === "collabAgent/activity" &&
2299+ collabPayload?.activityKind === "started");
2300+ if (isCollabSpawn && event.turnId === turnTokenUsage.activeTurnId) {
2301+ getCodexTurnAccumulator(turnTokenUsage, turnTokenUsage.activeTurnId).hasSubagents =
2302+ true;
2303+ }
2304+ }
2305+
2306+ const runtimeEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => {
2307+ if (runtimeEvent.type === "turn.completed" && runtimeEvent.turnId) {
2308+ return {
2309+ ...runtimeEvent,
2310+ payload: {
2311+ ...runtimeEvent.payload,
2312+ tokenUsage: completeCodexTurnTokenUsage(
2313+ turnTokenUsage,
2314+ String(runtimeEvent.turnId),
2315+ runtimeEvent.payload.state === "completed",
2316+ ),
2317+ },
2318+ } satisfies ProviderRuntimeEvent;
2319+ }
2320+ if (runtimeEvent.type === "turn.aborted" && runtimeEvent.turnId) {
2321+ return {
2322+ ...runtimeEvent,
2323+ payload: {
2324+ ...runtimeEvent.payload,
2325+ tokenUsage: completeCodexTurnTokenUsage(
2326+ turnTokenUsage,
2327+ String(runtimeEvent.turnId),
2328+ false,
2329+ ),
2330+ },
2331+ } satisfies ProviderRuntimeEvent;
2332+ }
2333+ return runtimeEvent;
2334+ });
20952335 if (runtimeEvents.length === 0) {
20962336 yield* Effect.logDebug("ignoring unhandled Codex provider event", {
20972337 method: event.method,
@@ -2129,6 +2369,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( · 31 unmodified lines
21292369 scope: sessionScope,
21302370 runtime,
21312371 eventFiber,
2372+ turnTokenUsage,
21322373 stopped: false,
21332374 });
21342375 sessionScopeTransferred = true;
@@ -2262,7 +2503,17 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( · 127 unmodified lines
22622503 }
22632504
22642505 return requireSession(threadId).pipe(
2265− Effect.flatMap((session) => session.runtime.rollbackThread(numTurns)),
2506+ Effect.flatMap((session) =>
2507+ session.runtime.rollbackThread(numTurns).pipe(
2508+ Effect.tap(() =>
2509+ Effect.sync(() => {
2510+ session.turnTokenUsage.baseline = undefined;
2511+ session.turnTokenUsage.activeTurnId = undefined;
2512+ session.turnTokenUsage.byTurnId.clear();
2513+ }),
2514+ ),
2515+ ),
2516+ ),
22662517 Effect.mapError((cause) =>
22672518 cause._tag === "ProviderAdapterSessionNotFoundError"
22682519 ? cause
apps/server/src/provider/Layers/OpenCodeAdapter.test.ts+745−0
@@ -3,6 +3,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; · 2 unmodified lines
33 import { it } from "@effect/vitest";
44 import * as Cause from "effect/Cause";
55 import * as Context from "effect/Context";
6+import * as Crypto from "effect/Crypto";
7+import * as Deferred from "effect/Deferred";
68 import * as Effect from "effect/Effect";
79 import * as Exit from "effect/Exit";
810 import * as Fiber from "effect/Fiber";
@@ -2086,6 +2088,557 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { · 2077 unmodified lines
20862088 }),
20872089 );
20882090
2091+ it.effect("marks subagents when a child is proven related by ancestry lookup", () =>
2092+ Effect.gen(function* () {
2093+ const adapter = yield* OpenCodeAdapter;
2094+ const threadId = asThreadId("thread-child-ancestry-usage");
2095+ // The child's `session.created` event was missed. Only the ancestry
2096+ // lookup can prove that `ses_child` belongs to this thread.
2097+ runtimeMock.state.sessionParentById.set("ses_child", "http://127.0.0.1:9999/session");
2098+ runtimeMock.state.sessionStatus = "busy";
2099+ const busy = promiseWithResolvers<unknown>();
2100+ const childPermission = promiseWithResolvers<unknown>();
2101+ const idle = promiseWithResolvers<unknown>();
2102+ runtimeMock.state.subscribedEvents = [busy.promise, childPermission.promise, idle.promise];
2103+
2104+ const eventsFiber = yield* adapter.streamEvents.pipe(
2105+ Stream.filter(
2106+ (event) =>
2107+ event.threadId === threadId &&
2108+ (event.type === "request.opened" || event.type === "turn.completed"),
2109+ ),
2110+ Stream.take(2),
2111+ Stream.runCollect,
2112+ Effect.forkChild,
2113+ );
2114+ yield* adapter.startSession({
2115+ provider: ProviderDriverKind.make("opencode"),
2116+ threadId,
2117+ runtimeMode: "approval-required",
2118+ });
2119+ const sendFiber = yield* adapter
2120+ .sendTurn({
2121+ threadId,
2122+ input: "Delegate to a child",
2123+ modelSelection: createModelSelection(
2124+ ProviderInstanceId.make("opencode"),
2125+ "opencode/kimi-k3",
2126+ ),
2127+ })
2128+ .pipe(Effect.forkChild);
2129+ busy.resolve({
2130+ id: "evt-child-ancestry-busy",
2131+ type: "session.status",
2132+ properties: {
2133+ sessionID: "http://127.0.0.1:9999/session",
2134+ status: { type: "busy" },
2135+ },
2136+ });
2137+ yield* Fiber.join(sendFiber);
2138+
2139+ const requestOpened = promiseWithResolvers<void>();
2140+ runtimeMock.state.sessionGetObserved = (sessionID) => {
2141+ if (sessionID === "ses_child") requestOpened.resolve(undefined);
2142+ };
2143+ childPermission.resolve({
2144+ id: "evt-child-ancestry-permission",
2145+ type: "permission.asked",
2146+ properties: permissionRequest("per_child_ancestry", "ses_child"),
2147+ });
2148+ yield* Effect.promise(() => requestOpened.promise);
2149+ yield* Effect.yieldNow;
2150+ runtimeMock.state.sessionStatus = "idle";
2151+ idle.resolve({
2152+ id: "evt-child-ancestry-idle",
2153+ type: "session.status",
2154+ properties: {
2155+ sessionID: "http://127.0.0.1:9999/session",
2156+ status: { type: "idle" },
2157+ },
2158+ });
2159+
2160+ const events = Array.from(yield* Fiber.join(eventsFiber).pipe(Effect.timeout("1 second")));
2161+ NodeAssert.deepEqual(
2162+ events.map((event) => event.type),
2163+ ["request.opened", "turn.completed"],
2164+ );
2165+ const completed = events[1];
2166+ if (completed?.type === "turn.completed") {
2167+ NodeAssert.deepEqual(completed.payload.tokenUsage, {
2168+ usageStatus: "unavailable",
2169+ usageScope: "main_agent",
2170+ hasSubagents: true,
2171+ });
2172+ }
2173+ yield* adapter.stopSession(threadId);
2174+ }),
2175+ );
2176+
2177+ it.effect("sums owned OpenCode step usage and marks unresolved usage partial", () =>
2178+ Effect.gen(function* () {
2179+ const adapter = yield* OpenCodeAdapter;
2180+ const threadId = asThreadId("thread-step-usage");
2181+ const busy = promiseWithResolvers<unknown>();
2182+ const firstStep = promiseWithResolvers<unknown>();
2183+ const assistantMessage = promiseWithResolvers<unknown>();
2184+ const duplicateStep = promiseWithResolvers<unknown>();
2185+ const secondStep = promiseWithResolvers<unknown>();
2186+ const unresolvedHeader = promiseWithResolvers<unknown>();
2187+ const unresolvedStep = promiseWithResolvers<unknown>();
2188+ const recoveredStep = promiseWithResolvers<unknown>();
2189+ const recoveredIncompleteHeader = promiseWithResolvers<unknown>();
2190+ const recoveredCompleteHeader = promiseWithResolvers<unknown>();
2191+ const childSession = promiseWithResolvers<unknown>();
2192+ const idle = promiseWithResolvers<unknown>();
2193+ runtimeMock.state.subscribedEvents = [
2194+ busy.promise,
2195+ firstStep.promise,
2196+ assistantMessage.promise,
2197+ duplicateStep.promise,
2198+ secondStep.promise,
2199+ unresolvedHeader.promise,
2200+ unresolvedStep.promise,
2201+ recoveredStep.promise,
2202+ recoveredIncompleteHeader.promise,
2203+ recoveredCompleteHeader.promise,
2204+ childSession.promise,
2205+ idle.promise,
2206+ ];
2207+
2208+ const completedFiber = yield* adapter.streamEvents.pipe(
2209+ Stream.filter((event) => event.threadId === threadId && event.type === "turn.completed"),
2210+ Stream.runHead,
2211+ Effect.forkChild,
2212+ );
2213+ yield* adapter.startSession({
2214+ provider: ProviderDriverKind.make("opencode"),
2215+ threadId,
2216+ runtimeMode: "full-access",
2217+ });
2218+ const sendFiber = yield* adapter
2219+ .sendTurn({
2220+ threadId,
2221+ input: "Use two model steps",
2222+ modelSelection: createModelSelection(
2223+ ProviderInstanceId.make("opencode"),
2224+ "opencode/kimi-k3",
2225+ ),
2226+ })
2227+ .pipe(Effect.forkChild);
2228+ busy.resolve({
2229+ id: "evt-step-usage-busy",
2230+ type: "session.status",
2231+ properties: {
2232+ sessionID: "http://127.0.0.1:9999/session",
2233+ status: { type: "busy" },
2234+ },
2235+ });
2236+ yield* Fiber.join(sendFiber);
2237+ const promptMessageId = (runtimeMock.state.promptCalls[0] as { messageID: string }).messageID;
2238+
2239+ const stepPart = {
2240+ id: "step-usage-1",
2241+ sessionID: "http://127.0.0.1:9999/session",
2242+ messageID: "assistant-step-usage-1",
2243+ type: "step-finish",
2244+ reason: "tool-calls",
2245+ cost: 0,
2246+ tokens: {
2247+ input: 100,
2248+ output: 20,
2249+ reasoning: 5,
2250+ cache: { read: 40, write: 10 },
2251+ },
2252+ } as const;
2253+ firstStep.resolve({
2254+ id: "evt-step-usage-1",
2255+ type: "message.part.updated",
2256+ properties: {
2257+ sessionID: "http://127.0.0.1:9999/session",
2258+ part: stepPart,
2259+ },
2260+ });
2261+ assistantMessage.resolve({
2262+ id: "evt-step-usage-assistant",
2263+ type: "message.updated",
2264+ properties: {
2265+ sessionID: "http://127.0.0.1:9999/session",
2266+ info: {
2267+ id: "assistant-step-usage-1",
2268+ role: "assistant",
2269+ parentID: promptMessageId,
2270+ },
2271+ },
2272+ });
2273+ duplicateStep.resolve({
2274+ id: "evt-step-usage-1-duplicate",
2275+ type: "message.part.updated",
2276+ properties: {
2277+ sessionID: "http://127.0.0.1:9999/session",
2278+ part: stepPart,
2279+ },
2280+ });
2281+ secondStep.resolve({
2282+ id: "evt-step-usage-2",
2283+ type: "message.part.updated",
2284+ properties: {
2285+ sessionID: "http://127.0.0.1:9999/session",
2286+ part: {
2287+ ...stepPart,
2288+ id: "step-usage-2",
2289+ tokens: {
2290+ input: 50,
2291+ output: 10,
2292+ reasoning: 2,
2293+ cache: { read: 10, write: 0 },
2294+ },
2295+ },
2296+ },
2297+ });
2298+ unresolvedHeader.resolve({
2299+ id: "evt-step-usage-unresolved-header",
2300+ type: "message.updated",
2301+ properties: {
2302+ sessionID: "http://127.0.0.1:9999/session",
2303+ info: {
2304+ id: "assistant-step-usage-without-parent",
2305+ role: "assistant",
2306+ },
2307+ },
2308+ });
2309+ unresolvedStep.resolve({
2310+ id: "evt-step-usage-unresolved",
2311+ type: "message.part.updated",
2312+ properties: {
2313+ sessionID: "http://127.0.0.1:9999/session",
2314+ part: {
2315+ ...stepPart,
2316+ id: "step-usage-unresolved",
2317+ messageID: "assistant-step-usage-without-parent",
2318+ tokens: {
2319+ input: 1_000,
2320+ output: 1_000,
2321+ reasoning: 0,
2322+ cache: { read: 0, write: 0 },
2323+ },
2324+ },
2325+ },
2326+ });
2327+ recoveredStep.resolve({
2328+ id: "evt-step-usage-recovered",
2329+ type: "message.part.updated",
2330+ properties: {
2331+ sessionID: "http://127.0.0.1:9999/session",
2332+ part: {
2333+ ...stepPart,
2334+ id: "step-usage-recovered",
2335+ messageID: "assistant-step-usage-recovered",
2336+ tokens: {
2337+ input: 30,
2338+ output: 10,
2339+ reasoning: 2,
2340+ cache: { read: 5, write: 1 },
2341+ },
2342+ },
2343+ },
2344+ });
2345+ recoveredIncompleteHeader.resolve({
2346+ id: "evt-step-usage-recovered-incomplete-header",
2347+ type: "message.updated",
2348+ properties: {
2349+ sessionID: "http://127.0.0.1:9999/session",
2350+ info: {
2351+ id: "assistant-step-usage-recovered",
2352+ role: "assistant",
2353+ parentID: "",
2354+ },
2355+ },
2356+ });
2357+ recoveredCompleteHeader.resolve({
2358+ id: "evt-step-usage-recovered-complete-header",
2359+ type: "message.updated",
2360+ properties: {
2361+ sessionID: "http://127.0.0.1:9999/session",
2362+ info: {
2363+ id: "assistant-step-usage-recovered",
2364+ role: "assistant",
2365+ parentID: promptMessageId,
2366+ },
2367+ },
2368+ });
2369+ childSession.resolve({
2370+ id: "evt-step-usage-child",
2371+ type: "session.created",
2372+ properties: {
2373+ info: {
2374+ id: "child-step-usage",
2375+ parentID: "http://127.0.0.1:9999/session",
2376+ },
2377+ },
2378+ });
2379+ idle.resolve({
2380+ id: "evt-step-usage-idle",
2381+ type: "session.status",
2382+ properties: {
2383+ sessionID: "http://127.0.0.1:9999/session",
2384+ status: { type: "idle" },
2385+ },
2386+ });
2387+
2388+ const completed = yield* Fiber.join(completedFiber).pipe(Effect.timeout("1 second"));
2389+ NodeAssert.equal(completed._tag, "Some");
2390+ if (completed._tag === "Some" && completed.value.type === "turn.completed") {
2391+ NodeAssert.deepStrictEqual(completed.value.payload.tokenUsage, {
2392+ usageStatus: "partial",
2393+ usageScope: "main_agent",
2394+ inputTokens: 246,
2395+ cachedInputTokens: 55,
2396+ cacheCreationTokens: 11,
2397+ outputTokens: 49,
2398+ reasoningTokens: 9,
2399+ hasSubagents: true,
2400+ });
2401+ }
2402+
2403+ yield* adapter.stopSession(threadId);
2404+ }),
2405+ );
2406+
2407+ it.effect("keeps the next turn usage while the prior completion is delayed", () =>
2408+ Effect.gen(function* () {
2409+ const threadId = asThreadId("thread-token-usage-terminal-handoff");
2410+ const firstBusy = promiseWithResolvers<unknown>();
2411+ const firstAssistantMessage = promiseWithResolvers<unknown>();
2412+ const firstStep = promiseWithResolvers<unknown>();
2413+ const firstIdle = promiseWithResolvers<unknown>();
2414+ const secondBusy = promiseWithResolvers<unknown>();
2415+ const secondAssistantMessage = promiseWithResolvers<unknown>();
2416+ const secondStep = promiseWithResolvers<unknown>();
2417+ const secondIdle = promiseWithResolvers<unknown>();
2418+ runtimeMock.state.subscribedEvents = [
2419+ firstBusy.promise,
2420+ firstAssistantMessage.promise,
2421+ firstStep.promise,
2422+ firstIdle.promise,
2423+ secondBusy.promise,
2424+ secondAssistantMessage.promise,
2425+ secondStep.promise,
2426+ secondIdle.promise,
2427+ ];
2428+
2429+ const firstStepWriteStarted = yield* Deferred.make<void>();
2430+ const firstStepWriteRelease = yield* Deferred.make<void>();
2431+ const terminalUuidStarted = yield* Deferred.make<void>();
2432+ const terminalUuidRelease = yield* Deferred.make<void>();
2433+ let blockFirstStepWrite = true;
2434+ let blockNextUuid = false;
2435+ const nodeCrypto = yield* Crypto.Crypto;
2436+ const gatedCrypto = {
2437+ ...nodeCrypto,
2438+ randomUUIDv4: Effect.suspend(() => {
2439+ if (!blockNextUuid) return nodeCrypto.randomUUIDv4;
2440+ blockNextUuid = false;
2441+ return Deferred.succeed(terminalUuidStarted, undefined).pipe(
2442+ Effect.andThen(Deferred.await(terminalUuidRelease)),
2443+ Effect.andThen(nodeCrypto.randomUUIDv4),
2444+ );
2445+ }),
2446+ } satisfies Crypto.Crypto;
2447+ const adapter = yield* makeOpenCodeAdapter(openCodeAdapterTestSettings, {
2448+ nativeEventLogger: {
2449+ filePath: "memory://opencode-token-usage-terminal-handoff",
2450+ write: (record) => {
2451+ const eventType = (record as { event?: { type?: unknown } }).event?.type;
2452+ if (blockFirstStepWrite && eventType === "message.part.updated") {
2453+ blockFirstStepWrite = false;
2454+ return Deferred.succeed(firstStepWriteStarted, undefined).pipe(
2455+ Effect.andThen(Deferred.await(firstStepWriteRelease)),
2456+ );
2457+ }
2458+ return Effect.void;
2459+ },
2460+ close: () => Effect.void,
2461+ },
2462+ }).pipe(Effect.provideService(Crypto.Crypto, gatedCrypto));
2463+
2464+ const completedFiber = yield* adapter.streamEvents.pipe(
2465+ Stream.filter((event) => event.threadId === threadId && event.type === "turn.completed"),
2466+ Stream.take(2),
2467+ Stream.runCollect,
2468+ Effect.forkChild,
2469+ );
2470+ yield* adapter.startSession({
2471+ provider: ProviderDriverKind.make("opencode"),
2472+ threadId,
2473+ runtimeMode: "full-access",
2474+ });
2475+
2476+ const firstSend = yield* adapter
2477+ .sendTurn({
2478+ threadId,
2479+ input: "Run the first token handoff turn",
2480+ modelSelection: createModelSelection(
2481+ ProviderInstanceId.make("opencode"),
2482+ "opencode/kimi-k3",
2483+ ),
2484+ })
2485+ .pipe(Effect.forkChild);
2486+ firstBusy.resolve({
2487+ id: "evt-token-handoff-first-busy",
2488+ type: "session.status",
2489+ properties: {
2490+ sessionID: "http://127.0.0.1:9999/session",
2491+ status: { type: "busy" },
2492+ },
2493+ });
2494+ const firstTurn = yield* Fiber.join(firstSend);
2495+ const firstPromptMessageId = (runtimeMock.state.promptCalls[0] as { messageID: string })
2496+ .messageID;
2497+ firstAssistantMessage.resolve({
2498+ id: "evt-token-handoff-first-assistant",
2499+ type: "message.updated",
2500+ properties: {
2501+ sessionID: "http://127.0.0.1:9999/session",
2502+ info: {
2503+ id: "assistant-token-handoff-first",
2504+ role: "assistant",
2505+ parentID: firstPromptMessageId,
2506+ },
2507+ },
2508+ });
2509+ firstStep.resolve({
2510+ id: "evt-token-handoff-first-step",
2511+ type: "message.part.updated",
2512+ properties: {
2513+ sessionID: "http://127.0.0.1:9999/session",
2514+ part: {
2515+ id: "step-token-handoff-first",
2516+ sessionID: "http://127.0.0.1:9999/session",
2517+ messageID: "assistant-token-handoff-first",
2518+ type: "step-finish",
2519+ reason: "stop",
2520+ cost: 0,
2521+ tokens: {
2522+ input: 100,
2523+ output: 20,
2524+ reasoning: 5,
2525+ cache: { read: 40, write: 10 },
2526+ },
2527+ },
2528+ },
2529+ });
2530+ yield* Deferred.await(firstStepWriteStarted);
2531+ blockNextUuid = true;
2532+ firstIdle.resolve({
2533+ id: "evt-token-handoff-first-idle",
2534+ type: "session.status",
2535+ properties: {
2536+ sessionID: "http://127.0.0.1:9999/session",
2537+ status: { type: "idle" },
2538+ },
2539+ });
2540+ yield* Deferred.succeed(firstStepWriteRelease, undefined);
2541+ yield* Deferred.await(terminalUuidStarted);
2542+
2543+ runtimeMock.state.sessionStatus = "busy";
2544+ const secondTurn = yield* adapter.sendTurn({
2545+ threadId,
2546+ input: "Run the second token handoff turn",
2547+ modelSelection: createModelSelection(
2548+ ProviderInstanceId.make("opencode"),
2549+ "opencode/kimi-k3",
2550+ ),
2551+ });
2552+ NodeAssert.notEqual(secondTurn.turnId, firstTurn.turnId);
2553+ const secondPromptMessageId = (runtimeMock.state.promptCalls[1] as { messageID: string })
2554+ .messageID;
2555+ yield* Deferred.succeed(terminalUuidRelease, undefined);
2556+
2557+ secondBusy.resolve({
2558+ id: "evt-token-handoff-second-busy",
2559+ type: "session.status",
2560+ properties: {
2561+ sessionID: "http://127.0.0.1:9999/session",
2562+ status: { type: "busy" },
2563+ },
2564+ });
2565+ secondAssistantMessage.resolve({
2566+ id: "evt-token-handoff-second-assistant",
2567+ type: "message.updated",
2568+ properties: {
2569+ sessionID: "http://127.0.0.1:9999/session",
2570+ info: {
2571+ id: "assistant-token-handoff-second",
2572+ role: "assistant",
2573+ parentID: secondPromptMessageId,
2574+ },
2575+ },
2576+ });
2577+ secondStep.resolve({
2578+ id: "evt-token-handoff-second-step",
2579+ type: "message.part.updated",
2580+ properties: {
2581+ sessionID: "http://127.0.0.1:9999/session",
2582+ part: {
2583+ id: "step-token-handoff-second",
2584+ sessionID: "http://127.0.0.1:9999/session",
2585+ messageID: "assistant-token-handoff-second",
2586+ type: "step-finish",
2587+ reason: "stop",
2588+ cost: 0,
2589+ tokens: {
2590+ input: 90,
2591+ output: 13,
2592+ reasoning: 3,
2593+ cache: { read: 20, write: 5 },
2594+ },
2595+ },
2596+ },
2597+ });
2598+ secondIdle.resolve({
2599+ id: "evt-token-handoff-second-idle",
2600+ type: "session.status",
2601+ properties: {
2602+ sessionID: "http://127.0.0.1:9999/session",
2603+ status: { type: "idle" },
2604+ },
2605+ });
2606+
2607+ const completed = Array.from(
2608+ yield* Fiber.join(completedFiber).pipe(Effect.timeout("1 second")),
2609+ );
2610+ NodeAssert.deepStrictEqual(
2611+ completed.map((event) =>
2612+ event.type === "turn.completed" ? event.payload.tokenUsage : undefined,
2613+ ),
2614+ [
2615+ {
2616+ usageStatus: "complete",
2617+ usageScope: "main_agent",
2618+ inputTokens: 150,
2619+ cachedInputTokens: 40,
2620+ cacheCreationTokens: 10,
2621+ outputTokens: 25,
2622+ reasoningTokens: 5,
2623+ hasSubagents: false,
2624+ },
2625+ {
2626+ usageStatus: "complete",
2627+ usageScope: "main_agent",
2628+ inputTokens: 115,
2629+ cachedInputTokens: 20,
2630+ cacheCreationTokens: 5,
2631+ outputTokens: 16,
2632+ reasoningTokens: 3,
2633+ hasSubagents: false,
2634+ },
2635+ ],
2636+ );
2637+
2638+ yield* adapter.stopSession(threadId);
2639+ }),
2640+ );
2641+
20892642 it.effect("ignores a stale admission status response after the next turn starts", () =>
20902643 Effect.gen(function* () {
20912644 const adapter = yield* OpenCodeAdapter;
@@ -2203,6 +2756,188 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { · 111 unmodified lines
22032756 }),
22042757 );
22052758
2759+ it.effect("ignores a late prior-turn step after the next prompt is admitted", () =>
2760+ Effect.gen(function* () {
2761+ const adapter = yield* OpenCodeAdapter;
2762+ const threadId = asThreadId("thread-token-usage-late-prior-step");
2763+ const firstBusy = promiseWithResolvers<unknown>();
2764+ const firstAssistant = promiseWithResolvers<unknown>();
2765+ const secondBusy = promiseWithResolvers<unknown>();
2766+ const secondAssistant = promiseWithResolvers<unknown>();
2767+ const lateFirstStep = promiseWithResolvers<unknown>();
2768+ const secondStep = promiseWithResolvers<unknown>();
2769+ const secondIdle = promiseWithResolvers<unknown>();
2770+ runtimeMock.state.subscribedEvents = [
2771+ firstBusy.promise,
2772+ firstAssistant.promise,
2773+ secondBusy.promise,
2774+ secondAssistant.promise,
2775+ lateFirstStep.promise,
2776+ secondStep.promise,
2777+ secondIdle.promise,
2778+ ];
2779+
2780+ const terminalsFiber = yield* adapter.streamEvents.pipe(
2781+ Stream.filter(
2782+ (event) =>
2783+ event.threadId === threadId &&
2784+ (event.type === "turn.aborted" || event.type === "turn.completed"),
2785+ ),
2786+ Stream.take(2),
2787+ Stream.runCollect,
2788+ Effect.forkChild,
2789+ );
2790+ yield* adapter.startSession({
2791+ provider: ProviderDriverKind.make("opencode"),
2792+ threadId,
2793+ runtimeMode: "full-access",
2794+ });
2795+
2796+ const firstSend = yield* adapter
2797+ .sendTurn({
2798+ threadId,
2799+ input: "Start the interrupted turn",
2800+ modelSelection: createModelSelection(
2801+ ProviderInstanceId.make("opencode"),
2802+ "opencode/kimi-k3",
2803+ ),
2804+ })
2805+ .pipe(Effect.forkChild);
2806+ firstBusy.resolve({
2807+ id: "evt-token-late-first-busy",
2808+ type: "session.status",
2809+ properties: {
2810+ sessionID: "http://127.0.0.1:9999/session",
2811+ status: { type: "busy" },
2812+ },
2813+ });
2814+ yield* Fiber.join(firstSend);
2815+ const firstPromptMessageId = (runtimeMock.state.promptCalls[0] as { messageID: string })
2816+ .messageID;
2817+ firstAssistant.resolve({
2818+ id: "evt-token-late-first-assistant",
2819+ type: "message.updated",
2820+ properties: {
2821+ sessionID: "http://127.0.0.1:9999/session",
2822+ info: {
2823+ id: "assistant-token-late-first",
2824+ role: "assistant",
2825+ parentID: firstPromptMessageId,
2826+ },
2827+ },
2828+ });
2829+ yield* Effect.yieldNow;
2830+ yield* adapter.interruptTurn(threadId);
2831+
2832+ const secondSend = yield* adapter
2833+ .sendTurn({
2834+ threadId,
2835+ input: "Start the next turn",
2836+ modelSelection: createModelSelection(
2837+ ProviderInstanceId.make("opencode"),
2838+ "opencode/kimi-k3",
2839+ ),
2840+ })
2841+ .pipe(Effect.forkChild);
2842+ while (runtimeMock.state.promptCalls.length < 2) {
2843+ yield* Effect.yieldNow;
2844+ }
2845+ secondBusy.resolve({
2846+ id: "evt-token-late-second-busy",
2847+ type: "session.status",
2848+ properties: {
2849+ sessionID: "http://127.0.0.1:9999/session",
2850+ status: { type: "busy" },
2851+ },
2852+ });
2853+ yield* Fiber.join(secondSend);
2854+ const secondPromptMessageId = (runtimeMock.state.promptCalls[1] as { messageID: string })
2855+ .messageID;
2856+ secondAssistant.resolve({
2857+ id: "evt-token-late-second-assistant",
2858+ type: "message.updated",
2859+ properties: {
2860+ sessionID: "http://127.0.0.1:9999/session",
2861+ info: {
2862+ id: "assistant-token-late-second",
2863+ role: "assistant",
2864+ parentID: secondPromptMessageId,
2865+ },
2866+ },
2867+ });
2868+ lateFirstStep.resolve({
2869+ id: "evt-token-late-first-step",
2870+ type: "message.part.updated",
2871+ properties: {
2872+ sessionID: "http://127.0.0.1:9999/session",
2873+ part: {
2874+ id: "step-token-late-first",
2875+ sessionID: "http://127.0.0.1:9999/session",
2876+ messageID: "assistant-token-late-first",
2877+ type: "step-finish",
2878+ reason: "stop",
2879+ cost: 0,
2880+ tokens: {
2881+ input: 1_000,
2882+ output: 1_000,
2883+ reasoning: 0,
2884+ cache: { read: 0, write: 0 },
2885+ },
2886+ },
2887+ },
2888+ });
2889+ secondStep.resolve({
2890+ id: "evt-token-late-second-step",
2891+ type: "message.part.updated",
2892+ properties: {
2893+ sessionID: "http://127.0.0.1:9999/session",
2894+ part: {
2895+ id: "step-token-late-second",
2896+ sessionID: "http://127.0.0.1:9999/session",
2897+ messageID: "assistant-token-late-second",
2898+ type: "step-finish",
2899+ reason: "stop",
2900+ cost: 0,
2901+ tokens: {
2902+ input: 40,
2903+ output: 10,
2904+ reasoning: 2,
2905+ cache: { read: 5, write: 1 },
2906+ },
2907+ },
2908+ },
2909+ });
2910+ secondIdle.resolve({
2911+ id: "evt-token-late-second-idle",
2912+ type: "session.status",
2913+ properties: {
2914+ sessionID: "http://127.0.0.1:9999/session",
2915+ status: { type: "idle" },
2916+ },
2917+ });
2918+
2919+ const terminals = Array.from(
2920+ yield* Fiber.join(terminalsFiber).pipe(Effect.timeout("1 second")),
2921+ );
2922+ const secondCompleted = terminals.find((event) => event.type === "turn.completed");
2923+ NodeAssert.equal(secondCompleted?.type, "turn.completed");
2924+ if (secondCompleted?.type === "turn.completed") {
2925+ NodeAssert.deepStrictEqual(secondCompleted.payload.tokenUsage, {
2926+ usageStatus: "complete",
2927+ usageScope: "main_agent",
2928+ inputTokens: 46,
2929+ cachedInputTokens: 5,
2930+ cacheCreationTokens: 1,
2931+ outputTokens: 12,
2932+ reasoningTokens: 2,
2933+ hasSubagents: false,
2934+ });
2935+ }
2936+
2937+ yield* adapter.stopSession(threadId);
2938+ }),
2939+ );
2940+
22062941 it.effect("reconciles a sole idle when the matching prompt echo arrives later", () =>
22072942 Effect.gen(function* () {
22082943 const adapter = yield* OpenCodeAdapter;
@@ -5432,6 +6167,11 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { · 3223 unmodified lines
54326167
54336168 return Effect.gen(function* () {
54346169 const adapter = yield* OpenCodeAdapter;
6170+ const startedFiber = yield* adapter.streamEvents.pipe(
6171+ Stream.filter((event) => event.type === "turn.started"),
6172+ Stream.runHead,
6173+ Effect.forkChild,
6174+ );
54356175 yield* adapter.startSession({
54366176 provider: ProviderDriverKind.make("opencode"),
54376177 threadId: asThreadId("thread-custom-instance"),
@@ -5470,6 +6210,11 @@ it.layer(OpenCodeAdapterTestLayer)("OpenCodeAdapterLive", (it) => { · 32 unmodified lines
54706210 }),
54716211 parts: [{ type: "text", text: "Fix it" }],
54726212 });
6213+ const started = yield* Fiber.join(startedFiber);
6214+ NodeAssert.equal(started._tag, "Some");
6215+ if (started._tag === "Some" && started.value.type === "turn.started") {
6216+ NodeAssert.equal(started.value.payload.effort, undefined);
6217+ }
54736218 }).pipe(Effect.provide(adapterLayer));
54746219 });
54756220
apps/server/src/provider/Layers/OpenCodeAdapter.ts+152−4
@@ -9,6 +9,7 @@ import { · 8 unmodified lines
99 RuntimeRequestId,
1010 ThreadId,
1111 type ToolLifecycleItemType,
12+ type TurnTokenUsage,
1213 TurnId,
1314 type UserInputQuestion,
1415 } from "@t3tools/contracts";
@@ -338,6 +339,7 @@ interface OpenCodeSessionContext { · 323 unmodified lines
338339 readonly partById: Map<string, Part>;
339340 readonly emittedTextByPartId: Map<string, string>;
340341 readonly completedAssistantPartIds: Set<string>;
342+ turnTokenUsage: OpenCodeTurnTokenUsageAccumulator | undefined;
341343 activeTurnId: TurnId | undefined;
342344 activeAgent: string | undefined;
343345 activeVariant: string | undefined;
@@ -368,6 +370,75 @@ interface OpenCodeSessionContext { · 24 unmodified lines
368370 readonly sessionScope: Scope.Closeable;
369371 }
370372
373+interface OpenCodeTurnTokenUsageAccumulator {
374+ readonly partIds: Set<string>;
375+ readonly promptMessageIds: Set<string>;
376+ readonly assistantOwnershipByMessageId: Map<string, "owned" | "other" | "unknown">;
377+ readonly unresolvedStepPartIds: Set<string>;
378+ inputTokens: number;
379+ cachedInputTokens: number;
380+ cacheCreationTokens: number;
381+ outputTokens: number;
382+ reasoningTokens: number;
383+ complete: boolean;
384+ hasSubagents: boolean;
385+}
386+
387+function makeOpenCodeTurnTokenUsageAccumulator(): OpenCodeTurnTokenUsageAccumulator {
388+ return {
389+ partIds: new Set(),
390+ promptMessageIds: new Set(),
391+ assistantOwnershipByMessageId: new Map(),
392+ unresolvedStepPartIds: new Set(),
393+ inputTokens: 0,
394+ cachedInputTokens: 0,
395+ cacheCreationTokens: 0,
396+ outputTokens: 0,
397+ reasoningTokens: 0,
398+ complete: true,
399+ hasSubagents: false,
400+ };
401+}
402+
403+function accumulateOpenCodeStepUsage(
404+ accumulator: OpenCodeTurnTokenUsageAccumulator,
405+ part: Extract<Part, { readonly type: "step-finish" }>,
406+): void {
407+ if (accumulator.partIds.has(part.id)) return;
408+ accumulator.partIds.add(part.id);
409+ accumulator.inputTokens += part.tokens.input + part.tokens.cache.read + part.tokens.cache.write;
410+ accumulator.cachedInputTokens += part.tokens.cache.read;
411+ accumulator.cacheCreationTokens += part.tokens.cache.write;
412+ accumulator.outputTokens += part.tokens.output + part.tokens.reasoning;
413+ accumulator.reasoningTokens += part.tokens.reasoning;
414+}
415+
416+function takeOpenCodeTurnTokenUsage(
417+ context: OpenCodeSessionContext,
418+ complete: boolean,
419+): TurnTokenUsage {
420+ const usage = context.turnTokenUsage;
421+ context.turnTokenUsage = undefined;
422+ if (!usage || usage.partIds.size === 0) {
423+ return {
424+ usageStatus: "unavailable",
425+ usageScope: "main_agent",
426+ hasSubagents: usage?.hasSubagents ?? false,
427+ };
428+ }
429+ return {
430+ usageStatus:
431+ complete && usage.complete && usage.unresolvedStepPartIds.size === 0 ? "complete" : "partial",
432+ usageScope: "main_agent",
433+ inputTokens: usage.inputTokens,
434+ cachedInputTokens: usage.cachedInputTokens,
435+ cacheCreationTokens: usage.cacheCreationTokens,
436+ outputTokens: usage.outputTokens,
437+ reasoningTokens: Math.min(usage.outputTokens, usage.reasoningTokens),
438+ hasSubagents: usage.hasSubagents,
439+ };
440+}
441+
371442 export interface OpenCodeAdapterLiveOptions {
372443 readonly instanceId?: ProviderInstanceId;
373444 readonly environment?: NodeJS.ProcessEnv;
@@ -1032,6 +1103,7 @@ export function makeOpenCodeAdapter( · 658 unmodified lines
10321103 ) {
10331104 context.pendingIdleReconciliation = undefined;
10341105 }
1106+ const tokenUsage = takeOpenCodeTurnTokenUsage(context, true);
10351107 context.activeTurnId = undefined;
10361108 context.activeAgent = undefined;
10371109 context.activeVariant = undefined;
@@ -1061,6 +1133,7 @@ export function makeOpenCodeAdapter( · 23 unmodified lines
10611133 type: "turn.completed",
10621134 payload: {
10631135 state: "completed",
1136+ tokenUsage,
10641137 },
10651138 });
10661139 });
@@ -1198,6 +1271,7 @@ export function makeOpenCodeAdapter( · 131 unmodified lines
11981271 deleteContextIfCurrent(context);
11991272 return;
12001273 }
1274+ const tokenUsage = takeOpenCodeTurnTokenUsage(context, false);
12011275 context.promptAdmission = undefined;
12021276 context.activeTurnId = undefined;
12031277 context.activeAgent = undefined;
@@ -1219,6 +1293,7 @@ export function makeOpenCodeAdapter( · 15 unmodified lines
12191293 payload: {
12201294 state: "failed",
12211295 errorMessage: detail,
1296+ tokenUsage,
12221297 },
12231298 });
12241299 yield* emit({
@@ -1396,7 +1471,13 @@ export function makeOpenCodeAdapter( · 171 unmodified lines
13961471 if (cancellation) {
13971472 context.cancellation = undefined;
13981473 }
1474+ let tokenUsage: TurnTokenUsage = {
1475+ usageStatus: "unavailable",
1476+ usageScope: "main_agent",
1477+ hasSubagents: false,
1478+ };
13991479 if (context.activeTurnId === turnId) {
1480+ tokenUsage = takeOpenCodeTurnTokenUsage(context, false);
14001481 context.activeTurnId = undefined;
14011482 context.activeAgent = undefined;
14021483 context.activeVariant = undefined;
@@ -1416,6 +1497,7 @@ export function makeOpenCodeAdapter( · 13 unmodified lines
14161497 type: "turn.aborted",
14171498 payload: {
14181499 reason: "Interrupted by user.",
1500+ tokenUsage,
14191501 },
14201502 });
14211503 if (cancellation) {
@@ -1550,6 +1632,16 @@ export function makeOpenCodeAdapter( · 128 unmodified lines
15501632 }
15511633 });
15521634
1635+ // Records a child session of this thread. A child seen during a live turn
1636+ // means that turn used subagents, whether the relation came from a
1637+ // `session.created` event or a later ancestry lookup after reconnect.
1638+ const addRelatedOpenCodeSession = (context: OpenCodeSessionContext, sessionId: string) => {
1639+ context.relatedSessionIds.add(sessionId);
1640+ if (context.activeTurnId && context.turnTokenUsage) {
1641+ context.turnTokenUsage.hasSubagents = true;
1642+ }
1643+ };
1644+
15531645 const isRelatedOpenCodeSession = Effect.fn("isRelatedOpenCodeSession")(function* (
15541646 context: OpenCodeSessionContext,
15551647 candidateSessionId: string,
@@ -1581,7 +1673,7 @@ export function makeOpenCodeAdapter( · 25 unmodified lines
15811673 let sessionId: string | undefined = candidateSessionId;
15821674 for (let depth = 0; sessionId !== undefined && depth < 32; depth += 1) {
15831675 if (context.relatedSessionIds.has(sessionId)) {
1584− context.relatedSessionIds.add(candidateSessionId);
1676+ addRelatedOpenCodeSession(context, candidateSessionId);
15851677 return true;
15861678 }
15871679 if (seen.has(sessionId)) {
@@ -2078,6 +2170,9 @@ export function makeOpenCodeAdapter( · 490 unmodified lines
20782170 }
20792171 yield* schedulePendingRequestRecovery(context);
20802172 if (!isFirstConnection) {
2173+ if (context.turnTokenUsage) {
2174+ context.turnTokenUsage.complete = false;
2175+ }
20812176 yield* schedulePromptAdmissionRecovery(context, event);
20822177 if (context.activeTurnId !== undefined && context.promptAdmission === undefined) {
20832178 yield* scheduleIdleReconciliation(context, context.activeTurnId, event);
@@ -2097,7 +2192,7 @@ export function makeOpenCodeAdapter( · 13 unmodified lines
20972192 if (event.type === "session.created" || event.type === "session.updated") {
20982193 const session = event.properties.info;
20992194 if (session.parentID && context.relatedSessionIds.has(session.parentID)) {
2100− context.relatedSessionIds.add(session.id);
2195+ addRelatedOpenCodeSession(context, session.id);
21012196 }
21022197 } else if (event.type === "session.deleted") {
21032198 context.relatedSessionIds.delete(event.properties.info.id);
@@ -2221,10 +2316,36 @@ export function makeOpenCodeAdapter( · 117 unmodified lines
22212316 }
22222317 context.messageRoleById.set(event.properties.info.id, event.properties.info.role);
22232318 if (event.properties.info.role === "assistant") {
2319+ const usage = context.turnTokenUsage;
2320+ const parentMessageId =
2321+ typeof event.properties.info.parentID === "string" &&
2322+ event.properties.info.parentID.trim().length > 0
2323+ ? event.properties.info.parentID
2324+ : undefined;
2325+ const observedOwnership =
2326+ parentMessageId === undefined
2327+ ? "unknown"
2328+ : usage?.promptMessageIds.has(parentMessageId)
2329+ ? "owned"
2330+ : "other";
2331+ const priorOwnership = usage?.assistantOwnershipByMessageId.get(
2332+ event.properties.info.id,
2333+ );
2334+ const ownership =
2335+ priorOwnership === undefined || priorOwnership === "unknown"
2336+ ? observedOwnership
2337+ : priorOwnership;
2338+ if (usage) {
2339+ usage.assistantOwnershipByMessageId.set(event.properties.info.id, ownership);
2340+ }
22242341 for (const part of context.partById.values()) {
22252342 if (part.messageID !== event.properties.info.id) {
22262343 continue;
22272344 }
2345+ if (usage && part.type === "step-finish") {
2346+ if (ownership !== "unknown") usage.unresolvedStepPartIds.delete(part.id);
2347+ if (ownership === "owned") accumulateOpenCodeStepUsage(usage, part);
2348+ }
22282349 yield* emitAssistantTextDelta(context, part, turnId, event);
22292350 }
22302351 }
@@ -2290,6 +2411,21 @@ export function makeOpenCodeAdapter( · 59 unmodified lines
22902411 context.partById.set(part.id, part);
22912412 const messageRole = messageRoleForPart(context, part);
22922413
2414+ if (turnId && part.type === "step-finish" && context.turnTokenUsage) {
2415+ const ownership = context.turnTokenUsage.assistantOwnershipByMessageId.get(
2416+ part.messageID,
2417+ );
2418+ if (ownership === "owned") {
2419+ accumulateOpenCodeStepUsage(context.turnTokenUsage, part);
2420+ } else if (
2421+ ownership === "unknown" ||
2422+ (ownership === undefined &&
2423+ context.messageRoleById.get(part.messageID) !== "assistant")
2424+ ) {
2425+ context.turnTokenUsage.unresolvedStepPartIds.add(part.id);
2426+ }
2427+ }
2428+
22932429 if (messageRole === "assistant") {
22942430 yield* emitAssistantTextDelta(context, part, turnId, event);
22952431 }
@@ -2483,6 +2619,7 @@ export function makeOpenCodeAdapter( · 187 unmodified lines
24832619 terminalCancellation.turnSettled = true;
24842620 terminalCancellation.acknowledged = true;
24852621 }
2622+ const tokenUsage = activeTurnId ? takeOpenCodeTurnTokenUsage(context, false) : undefined;
24862623 context.activeTurnId = undefined;
24872624 context.activeAgent = undefined;
24882625 context.activeVariant = undefined;
@@ -2507,6 +2644,7 @@ export function makeOpenCodeAdapter( · 18 unmodified lines
25072644 payload: {
25082645 state: "failed",
25092646 errorMessage: message,
2647+ tokenUsage,
25102648 },
25112649 });
25122650 }
@@ -2826,6 +2964,7 @@ export function makeOpenCodeAdapter( · 313 unmodified lines
28262964 emittedTextByPartId: new Map(),
28272965 messageRoleById: new Map(),
28282966 completedAssistantPartIds: new Set(),
2967+ turnTokenUsage: undefined,
28292968 activeTurnId: undefined,
28302969 activeAgent: undefined,
28312970 activeVariant: undefined,
@@ -2998,6 +3137,10 @@ export function makeOpenCodeAdapter( · 166 unmodified lines
29983137 context.promptAdmission = promptAdmission;
29993138
30003139 context.activeTurnId = turnId;
3140+ if (steeringTurnId === undefined) {
3141+ context.turnTokenUsage = makeOpenCodeTurnTokenUsageAccumulator();
3142+ }
3143+ context.turnTokenUsage?.promptMessageIds.add(messageId);
30013144 context.activeAgent = agent ?? (input.interactionMode === "plan" ? "plan" : undefined);
30023145 context.activeVariant = variant;
30033146 if (steeringTurnId === undefined) {
@@ -3022,7 +3165,6 @@ export function makeOpenCodeAdapter( · 18 unmodified lines
30223165 type: "turn.started",
30233166 payload: {
30243167 model: modelSelection?.model ?? context.session.model,
3025− ...(variant ? { effort: variant } : {}),
30263168 },
30273169 });
30283170 }
@@ -3087,6 +3229,7 @@ export function makeOpenCodeAdapter( · 58 unmodified lines
30873229 }
30883230 return;
30893231 }
3232+ const tokenUsage = takeOpenCodeTurnTokenUsage(context, false);
30903233 context.promptAdmission = undefined;
30913234 context.activeTurnId = undefined;
30923235 context.activeAgent = undefined;
@@ -3103,7 +3246,10 @@ export function makeOpenCodeAdapter( · 10 unmodified lines
31033246 yield* emit({
31043247 ...(yield* buildEventBase({ threadId: input.threadId, turnId })),
31053248 type: "turn.aborted",
3106− payload: { reason: requestError.detail },
3249+ payload: {
3250+ reason: requestError.detail,
3251+ tokenUsage,
3252+ },
31073253 });
31083254 return;
31093255 }
@@ -3131,6 +3277,7 @@ export function makeOpenCodeAdapter( · 21 unmodified lines
31313277 });
31323278 return;
31333279 }
3280+ const tokenUsage = takeOpenCodeTurnTokenUsage(context, false);
31343281 context.promptAdmission = undefined;
31353282 context.activeTurnId = undefined;
31363283 context.activeAgent = undefined;
@@ -3154,6 +3301,7 @@ export function makeOpenCodeAdapter( · 17 unmodified lines
31543301 type: "turn.aborted",
31553302 payload: {
31563303 reason: requestError.detail,
3304+ tokenUsage,
31573305 },
31583306 });
31593307 }),
apps/server/src/provider/Layers/ProviderService.test.ts+1237−4
@@ -314,6 +314,69 @@ function makeFakeCodexAdapter( · 313 unmodified lines
314314 };
315315 }
316316
317+interface RecordedAnalyticsEvent {
318+ readonly event: string;
319+ readonly properties?: Readonly<Record<string, unknown>>;
320+}
321+
322+function makeRecordingAnalytics() {
323+ const events: Array<RecordedAnalyticsEvent> = [];
324+ const layer = Layer.succeed(
325+ AnalyticsService.AnalyticsService,
326+ AnalyticsService.AnalyticsService.of({
327+ record: (event, properties) =>
328+ Effect.sync(() => {
329+ events.push({ event, ...(properties ? { properties } : {}) });
330+ }),
331+ flush: Effect.void,
332+ }),
333+ );
334+
335+ return {
336+ layer,
337+ reset: () => {
338+ events.length = 0;
339+ },
340+ eventsByName: (event: string) => events.filter((entry) => entry.event === event),
341+ };
342+}
343+
344+function makeStaticInstanceRegistry(
345+ entries: ReadonlyArray<readonly [ProviderInstanceId, ProviderAdapterShape<ProviderAdapterError>]>,
346+): ProviderAdapterRegistry.ProviderAdapterRegistry["Service"] {
347+ const adapters = new Map(entries);
348+ const unsupported = (instanceId: ProviderInstanceId) =>
349+ new ProviderUnsupportedError({
350+ provider: ProviderDriverKind.make(instanceId),
351+ });
352+
353+ return {
354+ getByInstance: (instanceId) => {
355+ const adapter = adapters.get(instanceId);
356+ return adapter ? Effect.succeed(adapter) : Effect.fail(unsupported(instanceId));
357+ },
358+ getInstanceInfo: (instanceId) => {
359+ const adapter = adapters.get(instanceId);
360+ return adapter
361+ ? Effect.succeed({
362+ instanceId,
363+ driverKind: adapter.provider,
364+ displayName: undefined,
365+ enabled: true,
366+ continuationIdentity: {
367+ driverKind: adapter.provider,
368+ continuationKey: `${adapter.provider}:instance:${instanceId}`,
369+ },
370+ })
371+ : Effect.fail(unsupported(instanceId));
372+ },
373+ listInstances: () => Effect.succeed(Array.from(adapters.keys())),
374+ subscribeChanges: Effect.flatMap(PubSub.unbounded<void>(), (pubsub) =>
375+ PubSub.subscribe(pubsub),
376+ ),
377+ };
378+}
379+
317380 const advanceTestClock = (ms: number) =>
318381 TestClock.adjust(`${ms} millis`).pipe(Effect.andThen(Effect.yieldNow));
319382
@@ -332,6 +395,7 @@ function makeProviderServiceLayer( · 12 unmodified lines
332395 input: {
333396 readonly directory?: ProviderSessionDirectory.ProviderSessionDirectory["Service"];
334397 readonly supportsConversationRollback?: boolean;
398+ readonly analyticsLayer?: Layer.Layer<AnalyticsService.AnalyticsService>;
335399 readonly registry?: ProviderAdapterRegistry.ProviderAdapterRegistry["Service"];
336400 } = {},
337401 ) {
@@ -365,7 +429,7 @@ function makeProviderServiceLayer( · 27 unmodified lines
365429 Layer.provide(directoryLayer),
366430 Layer.provide(defaultServerSettingsLayer),
367431 Layer.provide(serverConfigTestLayer),
368− Layer.provideMerge(AnalyticsService.layerTest),
432+ Layer.provideMerge(input.analyticsLayer ?? AnalyticsService.layerTest),
369433 Layer.provide(
370434 Layer.succeed(
371435 ProviderEventLoggers.ProviderEventLoggers,
@@ -440,6 +504,124 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => · 68 unmodified lines
440504 }),
441505 );
442506
507+it.effect("ProviderServiceLive flushes deferred completions during shutdown", () =>
508+ Effect.gen(function* () {
509+ const recordedAnalytics = makeRecordingAnalytics();
510+ const codex = makeFakeCodexAdapter();
511+ const registry = makeStaticInstanceRegistry([[codexInstanceId, codex.adapter]]);
512+ const providerAdapterLayer = Layer.succeed(
513+ ProviderAdapterRegistry.ProviderAdapterRegistry,
514+ registry,
515+ );
516+ const runtimeRepositoryLayer = ProviderSessionRuntime.layer.pipe(
517+ Layer.provide(SqlitePersistenceMemory),
518+ );
519+ const directoryLayer = ProviderSessionDirectoryLive.pipe(Layer.provide(runtimeRepositoryLayer));
520+ const providerLayer = Layer.mergeAll(
521+ makeProviderServiceLive().pipe(
522+ Layer.provide(providerAdapterLayer),
523+ Layer.provide(directoryLayer),
524+ Layer.provide(defaultServerSettingsLayer),
525+ Layer.provide(serverConfigTestLayer),
526+ Layer.provide(recordedAnalytics.layer),
527+ Layer.provide(
528+ Layer.succeed(
529+ ProviderEventLoggers.ProviderEventLoggers,
530+ ProviderEventLoggers.NoOpProviderEventLoggers,
531+ ),
532+ ),
533+ ),
534+ directoryLayer,
535+ runtimeRepositoryLayer,
536+ NodeServices.layer,
537+ );
538+ const scope = yield* Scope.make();
539+ const runtimeServices = yield* Layer.build(providerLayer).pipe(Scope.provide(scope));
540+ const provider = yield* ProviderService.ProviderService.pipe(Effect.provide(runtimeServices));
541+ const threadId = asThreadId("thread-turn-analytics-stop-all-deferred");
542+ const firstStarted = yield* Deferred.make<void>();
543+ const secondStarted = yield* Deferred.make<void>();
544+ const sendRelease = yield* Deferred.make<void>();
545+ const turnId = asTurnId("turn-analytics-stop-all-deferred");
546+ yield* provider.startSession(threadId, {
547+ provider: CODEX_DRIVER,
548+ providerInstanceId: codexInstanceId,
549+ threadId,
550+ runtimeMode: "full-access",
551+ });
552+ codex.sendTurn
553+ .mockImplementationOnce(() =>
554+ Effect.gen(function* () {
555+ yield* Deferred.succeed(firstStarted, undefined);
556+ yield* Deferred.await(sendRelease);
557+ return { threadId, turnId };
558+ }),
559+ )
560+ .mockImplementationOnce(() =>
561+ Effect.gen(function* () {
562+ yield* Deferred.succeed(secondStarted, undefined);
563+ yield* Deferred.await(sendRelease);
564+ return { threadId, turnId: asTurnId("turn-analytics-stop-all-other") };
565+ }),
566+ );
567+
568+ const firstSend = yield* provider
569+ .sendTurn({ threadId, input: "first", attachments: [] })
570+ .pipe(Effect.forkChild);
571+ yield* Deferred.await(firstStarted);
572+ const secondSend = yield* provider
573+ .sendTurn({ threadId, input: "second", attachments: [] })
574+ .pipe(Effect.forkChild);
575+ yield* Deferred.await(secondStarted);
576+
577+ const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe(
578+ Stream.runDrain,
579+ Effect.forkChild,
580+ );
581+ yield* Effect.yieldNow;
582+ codex.emit({
583+ type: "turn.started",
584+ eventId: asEventId("evt-turn-analytics-stop-all-deferred-start"),
585+ provider: CODEX_DRIVER,
586+ createdAt: "2026-01-01T00:00:00.000Z",
587+ threadId,
588+ turnId,
589+ payload: { model: "native-stop-all" },
590+ });
591+ codex.emit({
592+ type: "turn.completed",
593+ eventId: asEventId("evt-turn-analytics-stop-all-deferred-complete"),
594+ provider: CODEX_DRIVER,
595+ createdAt: "2026-01-01T00:00:00.000Z",
596+ threadId,
597+ turnId,
598+ payload: {
599+ state: "completed",
600+ tokenUsage: {
601+ usageStatus: "complete",
602+ usageScope: "main_agent",
603+ inputTokens: 1_200,
604+ outputTokens: 300,
605+ hasSubagents: false,
606+ },
607+ },
608+ });
609+ yield* Fiber.join(runtimeEvents);
610+ assert.equal(recordedAnalytics.eventsByName("provider.turn.completed").length, 0);
611+
612+ const closeExit = yield* Scope.close(scope, Exit.void).pipe(Effect.exit);
613+ const completed = recordedAnalytics.eventsByName("provider.turn.completed");
614+ assert.equal(Exit.isSuccess(closeExit), true);
615+ assert.equal(completed.length, 1);
616+ assert.equal(completed[0]?.properties?.model, "native-stop-all");
617+ assert.equal(completed[0]?.properties?.inputTokens, 1_200);
618+ assert.equal(completed[0]?.properties?.outputTokens, 300);
619+ yield* Fiber.interrupt(firstSend);
620+ yield* Fiber.interrupt(secondSend);
621+ assert.equal(recordedAnalytics.eventsByName("provider.turn.completed").length, 1);
622+ }),
623+);
624+
443625 it.effect("ProviderServiceLive rejects new sessions for disabled providers", () =>
444626 Effect.gen(function* () {
445627 const codex = makeFakeCodexAdapter();
@@ -2282,7 +2464,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { · 1836 unmodified lines
22822464 createdAt: "2026-01-01T00:00:00.000Z",
22832465 threadId: session.threadId,
22842466 turnId: asTurnId("turn-1"),
2285− status: "completed",
2467+ payload: { state: "completed" },
22862468 };
22872469
22882470 fanout.codex.emit(completedEvent);
@@ -2349,7 +2531,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { · 60 unmodified lines
23492531 createdAt: "2026-01-01T00:00:00.000Z",
23502532 threadId: session.threadId,
23512533 turnId: asTurnId("turn-1"),
2352− status: "completed",
2534+ payload: { state: "completed" },
23532535 });
23542536
23552537 yield* Fiber.join(consumer);
@@ -2415,7 +2597,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { · 59 unmodified lines
24152597 createdAt: "2026-01-01T00:00:00.000Z",
24162598 threadId: session.threadId,
24172599 turnId: asTurnId("turn-1"),
2418− status: "completed",
2600+ payload: { state: "completed" },
24192601 },
24202602 ];
24212603
@@ -2659,6 +2841,1057 @@ citations.layer("ProviderServiceLive assistant citations", (it) => { · 237 unmodified lines
26592841 );
26602842 });
26612843
2844+const recordedTurnAnalytics = makeRecordingAnalytics();
2845+const secondaryCodexInstanceId = ProviderInstanceId.make("codex_work");
2846+const primaryAnalyticsCodex = makeFakeCodexAdapter();
2847+const secondaryAnalyticsCodex = makeFakeCodexAdapter();
2848+const turnAnalytics = makeProviderServiceLayer({
2849+ analyticsLayer: recordedTurnAnalytics.layer,
2850+ registry: makeStaticInstanceRegistry([
2851+ [codexInstanceId, primaryAnalyticsCodex.adapter],
2852+ [secondaryCodexInstanceId, secondaryAnalyticsCodex.adapter],
2853+ ]),
2854+});
2855+
2856+turnAnalytics.layer("ProviderServiceLive turn analytics", (it) => {
2857+ it.effect("records one completed-turn event with the allowed properties", () =>
2858+ Effect.gen(function* () {
2859+ recordedTurnAnalytics.reset();
2860+ const provider = yield* ProviderService.ProviderService;
2861+ const threadId = asThreadId("thread-turn-analytics-complete");
2862+ yield* provider.startSession(threadId, {
2863+ provider: CODEX_DRIVER,
2864+ providerInstanceId: codexInstanceId,
2865+ threadId,
2866+ runtimeMode: "full-access",
2867+ });
2868+ const turn = yield* provider.sendTurn({
2869+ threadId,
2870+ input: "measure this turn",
2871+ attachments: [],
2872+ interactionMode: "plan",
2873+ modelSelection: createModelSelection(codexInstanceId, "gpt-5.6-sol", [
2874+ { id: "reasoningEffort", value: "high" },
2875+ ]),
2876+ });
2877+ yield* advanceTestClock(40);
2878+
2879+ const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe(
2880+ Stream.runDrain,
2881+ Effect.forkChild,
2882+ );
2883+ yield* Effect.yieldNow;
2884+ const completedEvent: LegacyProviderRuntimeEvent = {
2885+ type: "turn.completed",
2886+ eventId: asEventId("evt-turn-analytics-complete"),
2887+ provider: CODEX_DRIVER,
2888+ createdAt: "2026-01-01T00:00:00.000Z",
2889+ threadId,
2890+ turnId: turn.turnId,
2891+ payload: {
2892+ state: "completed",
2893+ tokenUsage: {
2894+ usageStatus: "complete",
2895+ usageScope: "main_agent",
2896+ inputTokens: 1_200,
2897+ cachedInputTokens: 800,
2898+ cacheCreationTokens: 100,
2899+ outputTokens: 300,
2900+ reasoningTokens: 120,
2901+ hasSubagents: false,
2902+ },
2903+ },
2904+ };
2905+ primaryAnalyticsCodex.emit(completedEvent);
2906+ primaryAnalyticsCodex.emit({
2907+ ...completedEvent,
2908+ eventId: asEventId("evt-turn-analytics-complete-duplicate"),
2909+ });
2910+ yield* Fiber.join(runtimeEvents);
2911+
2912+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
2913+ assert.equal(completed.length, 1);
2914+ assert.deepEqual(completed[0]?.properties, {
2915+ provider: CODEX_DRIVER,
2916+ model: "gpt-5.6-sol",
2917+ effort: "high",
2918+ interactionMode: "plan",
2919+ runtimeMode: "full-access",
2920+ mixedModels: false,
2921+ durationMs: 40,
2922+ terminalStatus: "completed",
2923+ usageStatus: "complete",
2924+ usageScope: "main_agent",
2925+ hasSubagents: false,
2926+ inputTokens: 1_200,
2927+ cachedInputTokens: 800,
2928+ cacheCreationTokens: 100,
2929+ outputTokens: 300,
2930+ reasoningTokens: 120,
2931+ });
2932+ }),
2933+ );
2934+
2935+ it.effect("does not report a generic model variant as reasoning effort", () =>
2936+ Effect.gen(function* () {
2937+ recordedTurnAnalytics.reset();
2938+ const provider = yield* ProviderService.ProviderService;
2939+ const threadId = asThreadId("thread-turn-analytics-generic-variant");
2940+ yield* provider.startSession(threadId, {
2941+ provider: CODEX_DRIVER,
2942+ providerInstanceId: codexInstanceId,
2943+ threadId,
2944+ runtimeMode: "full-access",
2945+ });
2946+ const turn = yield* provider.sendTurn({
2947+ threadId,
2948+ input: "use the provider preset",
2949+ attachments: [],
2950+ modelSelection: createModelSelection(codexInstanceId, "provider/model", [
2951+ { id: "variant", value: "high" },
2952+ ]),
2953+ });
2954+
2955+ const runtimeEvent = yield* Stream.take(provider.streamEvents, 1).pipe(
2956+ Stream.runDrain,
2957+ Effect.forkChild,
2958+ );
2959+ yield* Effect.yieldNow;
2960+ primaryAnalyticsCodex.emit({
2961+ type: "turn.completed",
2962+ eventId: asEventId("evt-turn-analytics-generic-variant"),
2963+ provider: CODEX_DRIVER,
2964+ createdAt: "2026-01-01T00:00:00.000Z",
2965+ threadId,
2966+ turnId: turn.turnId,
2967+ payload: { state: "completed" },
2968+ });
2969+ yield* Fiber.join(runtimeEvent);
2970+
2971+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
2972+ assert.equal(completed.length, 1);
2973+ assert.notProperty(completed[0]?.properties ?? {}, "effort");
2974+ }),
2975+ );
2976+
2977+ it.effect("ignores model metadata bound to another provider instance", () =>
2978+ Effect.gen(function* () {
2979+ recordedTurnAnalytics.reset();
2980+ const provider = yield* ProviderService.ProviderService;
2981+ const threadId = asThreadId("thread-turn-analytics-mismatched-model-instance");
2982+ yield* provider.startSession(threadId, {
2983+ provider: CODEX_DRIVER,
2984+ providerInstanceId: codexInstanceId,
2985+ threadId,
2986+ runtimeMode: "full-access",
2987+ });
2988+ const turn = yield* provider.sendTurn({
2989+ threadId,
2990+ input: "ignore this mismatched selection",
2991+ attachments: [],
2992+ modelSelection: createModelSelection(secondaryCodexInstanceId, "wrong-model", [
2993+ { id: "reasoningEffort", value: "high" },
2994+ ]),
2995+ });
2996+
2997+ const runtimeEvent = yield* Stream.take(provider.streamEvents, 1).pipe(
2998+ Stream.runDrain,
2999+ Effect.forkChild,
3000+ );
3001+ yield* Effect.yieldNow;
3002+ primaryAnalyticsCodex.emit({
3003+ type: "turn.completed",
3004+ eventId: asEventId("evt-turn-analytics-mismatched-model-instance"),
3005+ provider: CODEX_DRIVER,
3006+ createdAt: "2026-01-01T00:00:00.000Z",
3007+ threadId,
3008+ turnId: turn.turnId,
3009+ payload: { state: "completed" },
3010+ });
3011+ yield* Fiber.join(runtimeEvent);
3012+
3013+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3014+ assert.equal(completed.length, 1);
3015+ assert.notProperty(completed[0]?.properties ?? {}, "model");
3016+ assert.notProperty(completed[0]?.properties ?? {}, "effort");
3017+ }),
3018+ );
3019+
3020+ it.effect("keeps overlapping request metadata with out-of-order adapter responses", () =>
3021+ Effect.gen(function* () {
3022+ recordedTurnAnalytics.reset();
3023+ const provider = yield* ProviderService.ProviderService;
3024+ const threadId = asThreadId("thread-turn-analytics-overlap");
3025+ yield* provider.startSession(threadId, {
3026+ provider: CODEX_DRIVER,
3027+ providerInstanceId: codexInstanceId,
3028+ threadId,
3029+ runtimeMode: "full-access",
3030+ });
3031+
3032+ const firstStarted = yield* Deferred.make<void>();
3033+ const secondStarted = yield* Deferred.make<void>();
3034+ const firstRelease = yield* Deferred.make<void>();
3035+ const secondRelease = yield* Deferred.make<void>();
3036+ const initialStartsObserved = yield* Deferred.make<void>();
3037+ let initialStartCount = 0;
3038+ const firstTurnId = asTurnId("turn-analytics-overlap-first");
3039+ const secondTurnId = asTurnId("turn-analytics-overlap-second");
3040+ primaryAnalyticsCodex.sendTurn
3041+ .mockImplementationOnce((input) =>
3042+ Effect.gen(function* () {
3043+ yield* Deferred.succeed(firstStarted, undefined);
3044+ yield* Deferred.await(firstRelease);
3045+ return { threadId: input.threadId, turnId: firstTurnId };
3046+ }),
3047+ )
3048+ .mockImplementationOnce((input) =>
3049+ Effect.gen(function* () {
3050+ yield* Deferred.succeed(secondStarted, undefined);
3051+ yield* Deferred.await(secondRelease);
3052+ return { threadId: input.threadId, turnId: secondTurnId };
3053+ }),
3054+ );
3055+
3056+ const runtimeEvents = yield* Stream.take(provider.streamEvents, 5).pipe(
3057+ Stream.tap((event) => {
3058+ if (event.type !== "turn.started" || initialStartCount >= 2) return Effect.void;
3059+ initialStartCount += 1;
3060+ return initialStartCount === 2
3061+ ? Deferred.succeed(initialStartsObserved, undefined).pipe(Effect.asVoid)
3062+ : Effect.void;
3063+ }),
3064+ Stream.runDrain,
3065+ Effect.forkChild,
3066+ );
3067+ const firstSend = yield* provider
3068+ .sendTurn({
3069+ threadId,
3070+ input: "first",
3071+ attachments: [],
3072+ interactionMode: "default",
3073+ modelSelection: createModelSelection(codexInstanceId, "requested-first"),
3074+ })
3075+ .pipe(Effect.forkChild);
3076+ yield* Deferred.await(firstStarted);
3077+ const secondSend = yield* provider
3078+ .sendTurn({
3079+ threadId,
3080+ input: "second",
3081+ attachments: [],
3082+ interactionMode: "plan",
3083+ modelSelection: createModelSelection(codexInstanceId, "requested-second"),
3084+ })
3085+ .pipe(Effect.forkChild);
3086+ yield* Deferred.await(secondStarted);
3087+
3088+ for (const [turnId, suffix] of [
3089+ [firstTurnId, "first"],
3090+ [secondTurnId, "second"],
3091+ ] as const) {
3092+ primaryAnalyticsCodex.emit({
3093+ type: "turn.started",
3094+ eventId: asEventId(`evt-turn-analytics-overlap-start-${suffix}`),
3095+ provider: CODEX_DRIVER,
3096+ createdAt: "2026-01-01T00:00:00.000Z",
3097+ threadId,
3098+ turnId,
3099+ payload: { model: `native-${suffix}`, effort: `native-effort-${suffix}` },
3100+ });
3101+ }
3102+ yield* Deferred.await(initialStartsObserved);
3103+ yield* Deferred.succeed(secondRelease, undefined);
3104+ yield* Fiber.join(secondSend);
3105+ primaryAnalyticsCodex.emit({
3106+ type: "turn.started",
3107+ eventId: asEventId("evt-turn-analytics-overlap-start-second-duplicate"),
3108+ provider: CODEX_DRIVER,
3109+ createdAt: "2026-01-01T00:00:00.000Z",
3110+ threadId,
3111+ turnId: secondTurnId,
3112+ payload: { model: "native-second", effort: "native-effort-second" },
3113+ });
3114+ yield* Deferred.succeed(firstRelease, undefined);
3115+ yield* Fiber.join(firstSend);
3116+ for (const [turnId, suffix] of [
3117+ [secondTurnId, "second"],
3118+ [firstTurnId, "first"],
3119+ ] as const) {
3120+ primaryAnalyticsCodex.emit({
3121+ type: "turn.completed",
3122+ eventId: asEventId(`evt-turn-analytics-overlap-complete-${suffix}`),
3123+ provider: CODEX_DRIVER,
3124+ createdAt: "2026-01-01T00:00:00.000Z",
3125+ threadId,
3126+ turnId,
3127+ payload: { state: "completed" },
3128+ });
3129+ }
3130+ yield* Fiber.join(runtimeEvents);
3131+
3132+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3133+ assert.equal(completed.length, 2);
3134+ assert.deepInclude(completed[0]?.properties ?? {}, {
3135+ model: "native-second",
3136+ effort: "native-effort-second",
3137+ interactionMode: "plan",
3138+ });
3139+ assert.deepInclude(completed[1]?.properties ?? {}, {
3140+ model: "native-first",
3141+ effort: "native-effort-first",
3142+ interactionMode: "default",
3143+ });
3144+ }),
3145+ );
3146+
3147+ it.effect("waits for the adapter response when a turn completes before sendTurn returns", () =>
3148+ Effect.gen(function* () {
3149+ recordedTurnAnalytics.reset();
3150+ const provider = yield* ProviderService.ProviderService;
3151+ const threadId = asThreadId("thread-turn-analytics-fast-completion");
3152+ const turnId = asTurnId("turn-analytics-fast-completion");
3153+ const returnRelease = yield* Deferred.make<void>();
3154+ yield* provider.startSession(threadId, {
3155+ provider: CODEX_DRIVER,
3156+ providerInstanceId: codexInstanceId,
3157+ threadId,
3158+ runtimeMode: "full-access",
3159+ });
3160+ yield* Effect.yieldNow;
3161+ primaryAnalyticsCodex.sendTurn.mockImplementationOnce((input) =>
3162+ Effect.gen(function* () {
3163+ primaryAnalyticsCodex.emit({
3164+ type: "turn.started",
3165+ eventId: asEventId("evt-turn-analytics-fast-start"),
3166+ provider: CODEX_DRIVER,
3167+ createdAt: "2026-01-01T00:00:00.000Z",
3168+ threadId,
3169+ turnId,
3170+ payload: { model: "native-fast", effort: "high" },
3171+ });
3172+ primaryAnalyticsCodex.emit({
3173+ type: "turn.completed",
3174+ eventId: asEventId("evt-turn-analytics-fast-complete"),
3175+ provider: CODEX_DRIVER,
3176+ createdAt: "2026-01-01T00:00:00.000Z",
3177+ threadId,
3178+ turnId,
3179+ payload: { state: "completed" },
3180+ });
3181+ yield* Deferred.await(returnRelease);
3182+ return { threadId: input.threadId, turnId };
3183+ }),
3184+ );
3185+
3186+ const terminalReceipt = yield* provider.streamEvents.pipe(
3187+ Stream.filter((event) => event.type === "turn.completed"),
3188+ Stream.runHead,
3189+ Effect.forkChild,
3190+ );
3191+ yield* Effect.yieldNow;
3192+ const sendFiber = yield* provider
3193+ .sendTurn({
3194+ threadId,
3195+ input: "finish immediately",
3196+ attachments: [],
3197+ interactionMode: "plan",
3198+ modelSelection: createModelSelection(codexInstanceId, "requested-fast"),
3199+ })
3200+ .pipe(Effect.forkChild);
3201+ const terminal = yield* Fiber.join(terminalReceipt);
3202+ assert.equal(terminal._tag, "Some");
3203+ assert.equal(sendFiber.pollUnsafe(), undefined);
3204+ assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0);
3205+
3206+ yield* Deferred.succeed(returnRelease, undefined);
3207+ yield* Fiber.join(sendFiber);
3208+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3209+ assert.equal(completed.length, 1);
3210+ assert.deepInclude(completed[0]?.properties ?? {}, {
3211+ model: "native-fast",
3212+ effort: "high",
3213+ interactionMode: "plan",
3214+ });
3215+ }),
3216+ );
3217+
3218+ it.effect("does not give a synthetic turn the metadata of an in-flight send", () =>
3219+ Effect.gen(function* () {
3220+ recordedTurnAnalytics.reset();
3221+ const provider = yield* ProviderService.ProviderService;
3222+ const threadId = asThreadId("thread-turn-analytics-synthetic-start");
3223+ const syntheticTurnId = asTurnId("turn-analytics-synthetic");
3224+ const realTurnId = asTurnId("turn-analytics-real");
3225+ yield* provider.startSession(threadId, {
3226+ provider: CODEX_DRIVER,
3227+ providerInstanceId: codexInstanceId,
3228+ threadId,
3229+ runtimeMode: "full-access",
3230+ });
3231+ yield* Effect.yieldNow;
3232+ // Claude closes a leftover synthetic turn while it prepares the real
3233+ // turn, so both events arrive before sendTurn returns the real turn ID.
3234+ primaryAnalyticsCodex.sendTurn.mockImplementationOnce((input) =>
3235+ Effect.gen(function* () {
3236+ primaryAnalyticsCodex.emit({
3237+ type: "turn.started",
3238+ eventId: asEventId("evt-turn-analytics-synthetic-start"),
3239+ provider: CODEX_DRIVER,
3240+ createdAt: "2026-01-01T00:00:00.000Z",
3241+ threadId,
3242+ turnId: syntheticTurnId,
3243+ payload: {},
3244+ });
3245+ primaryAnalyticsCodex.emit({
3246+ type: "turn.completed",
3247+ eventId: asEventId("evt-turn-analytics-synthetic-complete"),
3248+ provider: CODEX_DRIVER,
3249+ createdAt: "2026-01-01T00:00:00.000Z",
3250+ threadId,
3251+ turnId: syntheticTurnId,
3252+ payload: { state: "completed" },
3253+ });
3254+ primaryAnalyticsCodex.emit({
3255+ type: "turn.started",
3256+ eventId: asEventId("evt-turn-analytics-real-start"),
3257+ provider: CODEX_DRIVER,
3258+ createdAt: "2026-01-01T00:00:00.000Z",
3259+ threadId,
3260+ turnId: realTurnId,
3261+ payload: { model: "native-real" },
3262+ });
3263+ yield* Effect.yieldNow;
3264+ return { threadId: input.threadId, turnId: realTurnId };
3265+ }),
3266+ );
3267+
3268+ yield* provider.sendTurn({
3269+ threadId,
3270+ input: "start the real turn",
3271+ attachments: [],
3272+ interactionMode: "plan",
3273+ modelSelection: createModelSelection(codexInstanceId, "requested-real"),
3274+ });
3275+ const realCompletion = yield* provider.streamEvents.pipe(
3276+ Stream.filter((event) => event.type === "turn.completed" && event.turnId === realTurnId),
3277+ Stream.runHead,
3278+ Effect.forkChild,
3279+ );
3280+ yield* Effect.yieldNow;
3281+ primaryAnalyticsCodex.emit({
3282+ type: "turn.completed",
3283+ eventId: asEventId("evt-turn-analytics-real-complete"),
3284+ provider: CODEX_DRIVER,
3285+ createdAt: "2026-01-01T00:00:00.000Z",
3286+ threadId,
3287+ turnId: realTurnId,
3288+ payload: { state: "completed" },
3289+ });
3290+ yield* Fiber.join(realCompletion);
3291+
3292+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3293+ assert.equal(completed.length, 2);
3294+ assert.equal(completed[0]?.properties?.interactionMode, undefined);
3295+ assert.equal(completed[0]?.properties?.model, undefined);
3296+ assert.deepInclude(completed[1]?.properties ?? {}, {
3297+ model: "native-real",
3298+ interactionMode: "plan",
3299+ });
3300+ }),
3301+ );
3302+
3303+ it.effect("defers overlapping terminal analytics until exact request association", () =>
3304+ Effect.gen(function* () {
3305+ recordedTurnAnalytics.reset();
3306+ const provider = yield* ProviderService.ProviderService;
3307+ const threadId = asThreadId("thread-turn-analytics-overlap-fast-completion");
3308+ const firstStarted = yield* Deferred.make<void>();
3309+ const secondStarted = yield* Deferred.make<void>();
3310+ const firstRelease = yield* Deferred.make<void>();
3311+ const secondRelease = yield* Deferred.make<void>();
3312+ const firstTurnId = asTurnId("turn-analytics-overlap-fast-first");
3313+ const secondTurnId = asTurnId("turn-analytics-overlap-fast-second");
3314+ yield* provider.startSession(threadId, {
3315+ provider: CODEX_DRIVER,
3316+ providerInstanceId: codexInstanceId,
3317+ threadId,
3318+ runtimeMode: "full-access",
3319+ });
3320+ primaryAnalyticsCodex.sendTurn
3321+ .mockImplementationOnce(() =>
3322+ Effect.gen(function* () {
3323+ yield* Deferred.succeed(firstStarted, undefined);
3324+ yield* Deferred.await(firstRelease);
3325+ return { threadId, turnId: firstTurnId };
3326+ }),
3327+ )
3328+ .mockImplementationOnce(() =>
3329+ Effect.gen(function* () {
3330+ yield* Deferred.succeed(secondStarted, undefined);
3331+ yield* Deferred.await(secondRelease);
3332+ return { threadId, turnId: secondTurnId };
3333+ }),
3334+ );
3335+
3336+ const runtimeEvents = yield* Stream.take(provider.streamEvents, 4).pipe(
3337+ Stream.runDrain,
3338+ Effect.forkChild,
3339+ );
3340+ const firstSend = yield* provider
3341+ .sendTurn({
3342+ threadId,
3343+ input: "first fast completion",
3344+ attachments: [],
3345+ interactionMode: "default",
3346+ modelSelection: createModelSelection(codexInstanceId, "requested-first"),
3347+ })
3348+ .pipe(Effect.forkChild);
3349+ yield* Deferred.await(firstStarted);
3350+ yield* advanceTestClock(10);
3351+ const secondSend = yield* provider
3352+ .sendTurn({
3353+ threadId,
3354+ input: "second fast completion",
3355+ attachments: [],
3356+ interactionMode: "plan",
3357+ modelSelection: createModelSelection(codexInstanceId, "requested-second"),
3358+ })
3359+ .pipe(Effect.forkChild);
3360+ yield* Deferred.await(secondStarted);
3361+ yield* advanceTestClock(20);
3362+
3363+ for (const [turnId, suffix] of [
3364+ [firstTurnId, "first"],
3365+ [secondTurnId, "second"],
3366+ ] as const) {
3367+ primaryAnalyticsCodex.emit({
3368+ type: "turn.started",
3369+ eventId: asEventId(`evt-turn-analytics-overlap-fast-start-${suffix}`),
3370+ provider: CODEX_DRIVER,
3371+ createdAt: "2026-01-01T00:00:00.000Z",
3372+ threadId,
3373+ turnId,
3374+ payload: { model: `native-${suffix}`, effort: `native-effort-${suffix}` },
3375+ });
3376+ primaryAnalyticsCodex.emit({
3377+ type: "turn.completed",
3378+ eventId: asEventId(`evt-turn-analytics-overlap-fast-complete-${suffix}`),
3379+ provider: CODEX_DRIVER,
3380+ createdAt: "2026-01-01T00:00:00.000Z",
3381+ threadId,
3382+ turnId,
3383+ payload: { state: "completed" },
3384+ });
3385+ }
3386+ yield* Fiber.join(runtimeEvents);
3387+ assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0);
3388+ yield* advanceTestClock(40);
3389+
3390+ yield* Deferred.succeed(secondRelease, undefined);
3391+ yield* Fiber.join(secondSend);
3392+ let completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3393+ assert.equal(completed.length, 1);
3394+ assert.deepInclude(completed[0]?.properties ?? {}, {
3395+ model: "native-second",
3396+ effort: "native-effort-second",
3397+ interactionMode: "plan",
3398+ durationMs: 20,
3399+ });
3400+
3401+ yield* Deferred.succeed(firstRelease, undefined);
3402+ yield* Fiber.join(firstSend);
3403+ completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3404+ assert.equal(completed.length, 2);
3405+ assert.deepInclude(completed[1]?.properties ?? {}, {
3406+ model: "native-first",
3407+ effort: "native-effort-first",
3408+ interactionMode: "default",
3409+ durationMs: 30,
3410+ });
3411+ }),
3412+ );
3413+
3414+ it.effect("cleans pending metadata when a send is canceled", () =>
3415+ Effect.gen(function* () {
3416+ recordedTurnAnalytics.reset();
3417+ const provider = yield* ProviderService.ProviderService;
3418+ const threadId = asThreadId("thread-turn-analytics-canceled-send");
3419+ const canceledStarted = yield* Deferred.make<void>();
3420+ const canceledRelease = yield* Deferred.make<void>();
3421+ const nextReturnRelease = yield* Deferred.make<void>();
3422+ const nextTurnId = asTurnId("turn-analytics-after-canceled-send");
3423+ yield* provider.startSession(threadId, {
3424+ provider: CODEX_DRIVER,
3425+ providerInstanceId: codexInstanceId,
3426+ threadId,
3427+ runtimeMode: "full-access",
3428+ });
3429+ primaryAnalyticsCodex.sendTurn
3430+ .mockImplementationOnce(() =>
3431+ Effect.gen(function* () {
3432+ yield* Deferred.succeed(canceledStarted, undefined);
3433+ yield* Deferred.await(canceledRelease);
3434+ return { threadId, turnId: asTurnId("turn-analytics-canceled") };
3435+ }),
3436+ )
3437+ .mockImplementationOnce(() =>
3438+ Effect.gen(function* () {
3439+ primaryAnalyticsCodex.emit({
3440+ type: "turn.started",
3441+ eventId: asEventId("evt-turn-analytics-after-canceled-start"),
3442+ provider: CODEX_DRIVER,
3443+ createdAt: "2026-01-01T00:00:00.000Z",
3444+ threadId,
3445+ turnId: nextTurnId,
3446+ payload: { model: "native-next", effort: "high" },
3447+ });
3448+ primaryAnalyticsCodex.emit({
3449+ type: "turn.completed",
3450+ eventId: asEventId("evt-turn-analytics-after-canceled-complete"),
3451+ provider: CODEX_DRIVER,
3452+ createdAt: "2026-01-01T00:00:00.000Z",
3453+ threadId,
3454+ turnId: nextTurnId,
3455+ payload: { state: "completed" },
3456+ });
3457+ yield* Deferred.await(nextReturnRelease);
3458+ return { threadId, turnId: nextTurnId };
3459+ }),
3460+ );
3461+
3462+ const canceledSend = yield* provider
3463+ .sendTurn({
3464+ threadId,
3465+ input: "cancel this request",
3466+ attachments: [],
3467+ interactionMode: "default",
3468+ })
3469+ .pipe(Effect.forkChild);
3470+ yield* Deferred.await(canceledStarted);
3471+ yield* Fiber.interrupt(canceledSend);
3472+
3473+ const terminalReceipt = yield* provider.streamEvents.pipe(
3474+ Stream.filter((event) => event.type === "turn.completed"),
3475+ Stream.runHead,
3476+ Effect.forkChild,
3477+ );
3478+ yield* Effect.yieldNow;
3479+ const nextSend = yield* provider
3480+ .sendTurn({
3481+ threadId,
3482+ input: "measure the next request",
3483+ attachments: [],
3484+ interactionMode: "plan",
3485+ })
3486+ .pipe(Effect.forkChild);
3487+ const terminal = yield* Fiber.join(terminalReceipt);
3488+ assert.equal(terminal._tag, "Some");
3489+ assert.equal(nextSend.pollUnsafe(), undefined);
3490+ // The canceled request must not hold the completion. The live request
3491+ // still does, until its adapter response links it to the turn.
3492+ assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0);
3493+ yield* Deferred.succeed(nextReturnRelease, undefined);
3494+ const nextTurn = yield* Fiber.join(nextSend);
3495+ assert.equal(nextTurn.turnId, nextTurnId);
3496+
3497+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3498+ assert.equal(completed.length, 1);
3499+ assert.deepInclude(completed[0]?.properties ?? {}, {
3500+ model: "native-next",
3501+ effort: "high",
3502+ interactionMode: "plan",
3503+ });
3504+ }),
3505+ );
3506+
3507+ it.effect("bounds deferred completions and drains them after sends are canceled", () =>
3508+ Effect.gen(function* () {
3509+ recordedTurnAnalytics.reset();
3510+ const provider = yield* ProviderService.ProviderService;
3511+ const threadId = asThreadId("thread-turn-analytics-bounded-deferred");
3512+ const allStarted = yield* Deferred.make<void>();
3513+ const sendRelease = yield* Deferred.make<void>();
3514+ const turnIds = Array.from({ length: 9 }, (_, index) =>
3515+ asTurnId(`turn-analytics-bounded-deferred-${index + 1}`),
3516+ );
3517+ let startedCount = 0;
3518+ yield* provider.startSession(threadId, {
3519+ provider: CODEX_DRIVER,
3520+ providerInstanceId: codexInstanceId,
3521+ threadId,
3522+ runtimeMode: "full-access",
3523+ });
3524+ for (const turnId of turnIds) {
3525+ primaryAnalyticsCodex.sendTurn.mockImplementationOnce((input) =>
3526+ Effect.gen(function* () {
3527+ startedCount += 1;
3528+ if (startedCount === turnIds.length) {
3529+ yield* Deferred.succeed(allStarted, undefined);
3530+ }
3531+ yield* Deferred.await(sendRelease);
3532+ return { threadId: input.threadId, turnId };
3533+ }),
3534+ );
3535+ }
3536+
3537+ const runtimeEvents = yield* Stream.take(provider.streamEvents, turnIds.length * 2).pipe(
3538+ Stream.runDrain,
3539+ Effect.forkChild,
3540+ );
3541+ yield* Effect.yieldNow;
3542+ const sends = [];
3543+ for (let index = 0; index < turnIds.length; index += 1) {
3544+ sends.push(
3545+ yield* provider
3546+ .sendTurn({
3547+ threadId,
3548+ input: `bounded deferred ${index + 1}`,
3549+ attachments: [],
3550+ interactionMode: index % 2 === 0 ? "default" : "plan",
3551+ })
3552+ .pipe(Effect.forkChild),
3553+ );
3554+ }
3555+ yield* Deferred.await(allStarted);
3556+
3557+ for (const [index, turnId] of turnIds.entries()) {
3558+ primaryAnalyticsCodex.emit({
3559+ type: "turn.started",
3560+ eventId: asEventId(`evt-turn-analytics-bounded-deferred-start-${index + 1}`),
3561+ provider: CODEX_DRIVER,
3562+ createdAt: "2026-01-01T00:00:00.000Z",
3563+ threadId,
3564+ turnId,
3565+ payload: { model: `native-bounded-${index + 1}` },
3566+ });
3567+ primaryAnalyticsCodex.emit({
3568+ type: "turn.completed",
3569+ eventId: asEventId(`evt-turn-analytics-bounded-deferred-complete-${index + 1}`),
3570+ provider: CODEX_DRIVER,
3571+ createdAt: "2026-01-01T00:00:00.000Z",
3572+ threadId,
3573+ turnId,
3574+ payload: { state: "completed" },
3575+ });
3576+ }
3577+ yield* Fiber.join(runtimeEvents);
3578+ assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 1);
3579+
3580+ for (const send of sends) {
3581+ yield* Fiber.interrupt(send);
3582+ }
3583+ assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 9);
3584+ }),
3585+ );
3586+
3587+ it.effect("flushes a deferred completion when the session stops", () =>
3588+ Effect.gen(function* () {
3589+ recordedTurnAnalytics.reset();
3590+ const provider = yield* ProviderService.ProviderService;
3591+ const threadId = asThreadId("thread-turn-analytics-stop-deferred");
3592+ const firstStarted = yield* Deferred.make<void>();
3593+ const secondStarted = yield* Deferred.make<void>();
3594+ const sendRelease = yield* Deferred.make<void>();
3595+ const turnId = asTurnId("turn-analytics-stop-deferred");
3596+ yield* provider.startSession(threadId, {
3597+ provider: CODEX_DRIVER,
3598+ providerInstanceId: codexInstanceId,
3599+ threadId,
3600+ runtimeMode: "full-access",
3601+ });
3602+ primaryAnalyticsCodex.sendTurn
3603+ .mockImplementationOnce(() =>
3604+ Effect.gen(function* () {
3605+ yield* Deferred.succeed(firstStarted, undefined);
3606+ yield* Deferred.await(sendRelease);
3607+ return { threadId, turnId };
3608+ }),
3609+ )
3610+ .mockImplementationOnce(() =>
3611+ Effect.gen(function* () {
3612+ yield* Deferred.succeed(secondStarted, undefined);
3613+ yield* Deferred.await(sendRelease);
3614+ return { threadId, turnId: asTurnId("turn-analytics-stop-other") };
3615+ }),
3616+ );
3617+
3618+ const firstSend = yield* provider
3619+ .sendTurn({ threadId, input: "first", attachments: [] })
3620+ .pipe(Effect.forkChild);
3621+ yield* Deferred.await(firstStarted);
3622+ const secondSend = yield* provider
3623+ .sendTurn({ threadId, input: "second", attachments: [] })
3624+ .pipe(Effect.forkChild);
3625+ yield* Deferred.await(secondStarted);
3626+
3627+ const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe(
3628+ Stream.runDrain,
3629+ Effect.forkChild,
3630+ );
3631+ yield* Effect.yieldNow;
3632+ primaryAnalyticsCodex.emit({
3633+ type: "turn.started",
3634+ eventId: asEventId("evt-turn-analytics-stop-deferred-start"),
3635+ provider: CODEX_DRIVER,
3636+ createdAt: "2026-01-01T00:00:00.000Z",
3637+ threadId,
3638+ turnId,
3639+ payload: { model: "native-stop" },
3640+ });
3641+ primaryAnalyticsCodex.emit({
3642+ type: "turn.completed",
3643+ eventId: asEventId("evt-turn-analytics-stop-deferred-complete"),
3644+ provider: CODEX_DRIVER,
3645+ createdAt: "2026-01-01T00:00:00.000Z",
3646+ threadId,
3647+ turnId,
3648+ payload: { state: "completed" },
3649+ });
3650+ yield* Fiber.join(runtimeEvents);
3651+ assert.equal(recordedTurnAnalytics.eventsByName("provider.turn.completed").length, 0);
3652+
3653+ yield* provider.stopSession({ threadId });
3654+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3655+ assert.equal(completed.length, 1);
3656+ assert.equal(completed[0]?.properties?.model, "native-stop");
3657+ yield* Fiber.interrupt(firstSend);
3658+ yield* Fiber.interrupt(secondSend);
3659+ }),
3660+ );
3661+
3662+ it.effect("keeps the first metadata when steering reuses a rerouted turn", () =>
3663+ Effect.gen(function* () {
3664+ recordedTurnAnalytics.reset();
3665+ const provider = yield* ProviderService.ProviderService;
3666+ const threadId = asThreadId("thread-turn-analytics-steering");
3667+ yield* provider.startSession(threadId, {
3668+ provider: CODEX_DRIVER,
3669+ providerInstanceId: codexInstanceId,
3670+ threadId,
3671+ runtimeMode: "auto",
3672+ });
3673+ const firstTurn = yield* provider.sendTurn({
3674+ threadId,
3675+ input: "start",
3676+ attachments: [],
3677+ interactionMode: "default",
3678+ modelSelection: createModelSelection(codexInstanceId, "gpt-5.6-sol", [
3679+ { id: "reasoningEffort", value: "high" },
3680+ ]),
3681+ });
3682+ yield* advanceTestClock(10);
3683+
3684+ const reroutedEvent = yield* Stream.take(provider.streamEvents, 1).pipe(
3685+ Stream.runDrain,
3686+ Effect.forkChild,
3687+ );
3688+ yield* Effect.yieldNow;
3689+ primaryAnalyticsCodex.emit({
3690+ type: "model.rerouted",
3691+ eventId: asEventId("evt-turn-analytics-rerouted"),
3692+ provider: CODEX_DRIVER,
3693+ createdAt: "2026-01-01T00:00:00.000Z",
3694+ threadId,
3695+ turnId: firstTurn.turnId,
3696+ payload: {
3697+ fromModel: "gpt-5.6-sol",
3698+ toModel: "gpt-5.6-terra",
3699+ reason: "capacity",
3700+ },
3701+ });
3702+ yield* Fiber.join(reroutedEvent);
3703+ yield* advanceTestClock(15);
3704+
3705+ const steeredTurn = yield* provider.sendTurn({
3706+ threadId,
3707+ input: "steer",
3708+ attachments: [],
3709+ interactionMode: "plan",
3710+ modelSelection: createModelSelection(codexInstanceId, "gpt-5.6-terra", [
3711+ { id: "reasoningEffort", value: "low" },
3712+ ]),
3713+ });
3714+ assert.equal(steeredTurn.turnId, firstTurn.turnId);
3715+ yield* advanceTestClock(20);
3716+
3717+ const completedEvent = yield* Stream.take(provider.streamEvents, 1).pipe(
3718+ Stream.runDrain,
3719+ Effect.forkChild,
3720+ );
3721+ yield* Effect.yieldNow;
3722+ primaryAnalyticsCodex.emit({
3723+ type: "turn.completed",
3724+ eventId: asEventId("evt-turn-analytics-steered-complete"),
3725+ provider: CODEX_DRIVER,
3726+ createdAt: "2026-01-01T00:00:00.000Z",
3727+ threadId,
3728+ turnId: firstTurn.turnId,
3729+ payload: {
3730+ state: "completed",
3731+ tokenUsage: {
3732+ usageStatus: "complete",
3733+ usageScope: "main_agent",
3734+ inputTokens: 500,
3735+ outputTokens: 100,
3736+ hasSubagents: false,
3737+ },
3738+ },
3739+ });
3740+ yield* Fiber.join(completedEvent);
3741+
3742+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3743+ assert.equal(completed.length, 1);
3744+ assert.equal(completed[0]?.properties?.model, "gpt-5.6-sol");
3745+ assert.equal(completed[0]?.properties?.effort, "high");
3746+ assert.equal(completed[0]?.properties?.interactionMode, "default");
3747+ assert.equal(completed[0]?.properties?.mixedModels, true);
3748+ assert.equal(completed[0]?.properties?.durationMs, 45);
3749+ }),
3750+ );
3751+
3752+ it.effect("bounds active metadata while preserving recent delayed completions", () =>
3753+ Effect.gen(function* () {
3754+ recordedTurnAnalytics.reset();
3755+ const provider = yield* ProviderService.ProviderService;
3756+ const threadId = asThreadId("thread-turn-analytics-bounded-active");
3757+ const runtimeEvents = yield* Stream.take(provider.streamEvents, 12).pipe(
3758+ Stream.runDrain,
3759+ Effect.forkChild,
3760+ );
3761+ yield* Effect.yieldNow;
3762+
3763+ for (let index = 1; index <= 10; index += 1) {
3764+ primaryAnalyticsCodex.emit({
3765+ type: "turn.started",
3766+ eventId: asEventId(`evt-turn-analytics-bounded-start-${index}`),
3767+ provider: CODEX_DRIVER,
3768+ createdAt: "2026-01-01T00:00:00.000Z",
3769+ threadId,
3770+ turnId: asTurnId(`turn-analytics-bounded-${index}`),
3771+ payload: { model: `model-${index}` },
3772+ });
3773+ }
3774+ for (const index of [3, 1]) {
3775+ primaryAnalyticsCodex.emit({
3776+ type: "turn.completed",
3777+ eventId: asEventId(`evt-turn-analytics-bounded-complete-${index}`),
3778+ provider: CODEX_DRIVER,
3779+ createdAt: "2026-01-01T00:00:00.000Z",
3780+ threadId,
3781+ turnId: asTurnId(`turn-analytics-bounded-${index}`),
3782+ payload: { state: "completed" },
3783+ });
3784+ }
3785+ yield* Fiber.join(runtimeEvents);
3786+
3787+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3788+ assert.equal(completed.length, 2);
3789+ assert.equal(completed[0]?.properties?.model, "model-3");
3790+ assert.notProperty(completed[1]?.properties ?? {}, "model");
3791+ }),
3792+ );
3793+
3794+ it.effect("separates provider instances and omits unavailable counts", () =>
3795+ Effect.gen(function* () {
3796+ recordedTurnAnalytics.reset();
3797+ const provider = yield* ProviderService.ProviderService;
3798+ const threadId = asThreadId("thread-turn-analytics-instances");
3799+ const turnId = asTurnId("turn-shared-between-instances");
3800+ const runtimeEvents = yield* Stream.take(provider.streamEvents, 2).pipe(
3801+ Stream.runDrain,
3802+ Effect.forkChild,
3803+ );
3804+ yield* Effect.yieldNow;
3805+ const event: LegacyProviderRuntimeEvent = {
3806+ type: "turn.completed",
3807+ eventId: asEventId("evt-turn-analytics-primary-instance"),
3808+ provider: CODEX_DRIVER,
3809+ createdAt: "2026-01-01T00:00:00.000Z",
3810+ threadId,
3811+ turnId,
3812+ payload: {
3813+ state: "completed",
3814+ tokenUsage: {
3815+ usageStatus: "unavailable",
3816+ usageScope: "main_agent",
3817+ hasSubagents: false,
3818+ },
3819+ },
3820+ };
3821+ primaryAnalyticsCodex.emit(event);
3822+ secondaryAnalyticsCodex.emit({
3823+ ...event,
3824+ eventId: asEventId("evt-turn-analytics-secondary-instance"),
3825+ });
3826+ yield* Fiber.join(runtimeEvents);
3827+
3828+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3829+ assert.equal(completed.length, 2);
3830+ for (const entry of completed) {
3831+ assert.deepEqual(entry.properties, {
3832+ provider: CODEX_DRIVER,
3833+ terminalStatus: "completed",
3834+ usageStatus: "unavailable",
3835+ usageScope: "main_agent",
3836+ hasSubagents: false,
3837+ });
3838+ }
3839+ }),
3840+ );
3841+
3842+ it.effect("records known token counts for an interrupted turn", () =>
3843+ Effect.gen(function* () {
3844+ recordedTurnAnalytics.reset();
3845+ const provider = yield* ProviderService.ProviderService;
3846+ const threadId = asThreadId("thread-turn-analytics-interrupted");
3847+ yield* provider.startSession(threadId, {
3848+ provider: CODEX_DRIVER,
3849+ providerInstanceId: codexInstanceId,
3850+ threadId,
3851+ runtimeMode: "approval-required",
3852+ });
3853+ const turn = yield* provider.sendTurn({
3854+ threadId,
3855+ input: "stop after some work",
3856+ attachments: [],
3857+ });
3858+
3859+ const runtimeEvent = yield* Stream.take(provider.streamEvents, 1).pipe(
3860+ Stream.runDrain,
3861+ Effect.forkChild,
3862+ );
3863+ yield* Effect.yieldNow;
3864+ primaryAnalyticsCodex.emit({
3865+ type: "turn.aborted",
3866+ eventId: asEventId("evt-turn-analytics-interrupted"),
3867+ provider: CODEX_DRIVER,
3868+ createdAt: "2026-01-01T00:00:00.000Z",
3869+ threadId,
3870+ turnId: turn.turnId,
3871+ payload: {
3872+ reason: "Interrupted by user",
3873+ tokenUsage: {
3874+ usageStatus: "partial",
3875+ usageScope: "main_agent",
3876+ inputTokens: 120,
3877+ outputTokens: 30,
3878+ hasSubagents: true,
3879+ },
3880+ },
3881+ });
3882+ yield* Fiber.join(runtimeEvent);
3883+
3884+ const completed = recordedTurnAnalytics.eventsByName("provider.turn.completed");
3885+ assert.equal(completed.length, 1);
3886+ assert.equal(completed[0]?.properties?.terminalStatus, "interrupted");
3887+ assert.equal(completed[0]?.properties?.usageStatus, "partial");
3888+ assert.equal(completed[0]?.properties?.inputTokens, 120);
3889+ assert.equal(completed[0]?.properties?.outputTokens, 30);
3890+ assert.equal(completed[0]?.properties?.hasSubagents, true);
3891+ }),
3892+ );
3893+});
3894+
26623895 const validation = makeProviderServiceLayer();
26633896 validation.layer("ProviderServiceLive validation", (it) => {
26643897 it.effect("rejects citation-expanded input over the provider character limit", () =>
apps/server/src/provider/Layers/ProviderService.ts+484−1
@@ -31,6 +31,7 @@ import { · 30 unmodified lines
3131 } from "@t3tools/contracts";
3232 import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations";
3333 import { causeErrorTag } from "@t3tools/shared/observability";
34+import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
3435 import * as DateTime from "effect/DateTime";
3536 import * as Deferred from "effect/Deferred";
3637 import * as Effect from "effect/Effect";
@@ -96,6 +97,75 @@ export interface ProviderServiceLiveOptions { · 59 unmodified lines
9697 readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread;
9798 }
9899
100+interface TurnAnalyticsMetadata {
101+ readonly requestId: number;
102+ readonly provider: ProviderDriverKind;
103+ readonly startedAtMs: number;
104+ readonly mixedModels: boolean;
105+ readonly model?: string;
106+ readonly effort?: string;
107+ readonly interactionMode?: string;
108+ readonly runtimeMode?: string;
109+}
110+
111+interface ActiveTurnAnalytics {
112+ readonly metadata: TurnAnalyticsMetadata;
113+ readonly requestAssociated: boolean;
114+}
115+
116+interface DeferredTurnAnalyticsCompletion {
117+ readonly completionKey: string;
118+ readonly completedAtMs: number;
119+ readonly terminalProperties: Readonly<Record<string, unknown>>;
120+}
121+
122+interface TurnAnalyticsSessionState {
123+ readonly pendingByRequestId: Map<number, TurnAnalyticsMetadata>;
124+ readonly activeByTurnId: Map<string, ActiveTurnAnalytics>;
125+ readonly deferredCompletionsByTurnId: Map<string, DeferredTurnAnalyticsCompletion>;
126+}
127+
128+interface TurnAnalyticsState {
129+ readonly sessions: Map<string, TurnAnalyticsSessionState>;
130+ readonly completedKeys: Set<string>;
131+ readonly completedOrder: Array<string>;
132+}
133+
134+const MAX_COMPLETED_TURN_ANALYTICS_KEYS = 512;
135+const MAX_ACTIVE_TURN_ANALYTICS_PER_SESSION = 8;
136+
137+function setActiveTurnAnalytics(
138+ session: TurnAnalyticsSessionState,
139+ turnId: string,
140+ active: ActiveTurnAnalytics,
141+): void {
142+ session.activeByTurnId.set(turnId, active);
143+ while (session.activeByTurnId.size > MAX_ACTIVE_TURN_ANALYTICS_PER_SESSION) {
144+ const oldestTurnId = session.activeByTurnId.keys().next().value;
145+ if (oldestTurnId === undefined) return;
146+ session.activeByTurnId.delete(oldestTurnId);
147+ }
148+}
149+
150+function turnAnalyticsSessionKey(instanceId: ProviderInstanceId, threadId: ThreadId): string {
151+ return `${String(instanceId)}\u0000${String(threadId)}`;
152+}
153+
154+function turnAnalyticsCompletionKey(
155+ instanceId: ProviderInstanceId,
156+ threadId: ThreadId,
157+ turnId: string,
158+): string {
159+ return `${turnAnalyticsSessionKey(instanceId, threadId)}\u0000${turnId}`;
160+}
161+
162+function turnEffort(modelSelection: ProviderSendTurnInput["modelSelection"]): string | undefined {
163+ return (
164+ getModelSelectionStringOptionValue(modelSelection, "reasoningEffort") ??
165+ getModelSelectionStringOptionValue(modelSelection, "effort")
166+ );
167+}
168+
99169 type ProviderServiceMethod<Name extends keyof ProviderService.ProviderService["Service"]> =
100170 ProviderService.ProviderService["Service"][Name];
101171
@@ -258,7 +328,366 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( · 156 unmodified lines
258328 yield* Deferred.succeed(pending.completion, terminal);
259329 return true;
260330 });
331+ const turnAnalytics = yield* Ref.make<TurnAnalyticsState>({
332+ sessions: new Map(),
333+ completedKeys: new Set(),
334+ completedOrder: [],
335+ });
336+ let turnAnalyticsRequestId = 0;
261337 const nowIso = Effect.map(DateTime.now, DateTime.formatIso);
338+
339+ const finishTurnAnalytics = (
340+ state: TurnAnalyticsState,
341+ input: {
342+ readonly sessionKey: string;
343+ readonly turnId: string;
344+ readonly completion: DeferredTurnAnalyticsCompletion;
345+ },
346+ ): Readonly<Record<string, unknown>> | undefined => {
347+ if (state.completedKeys.has(input.completion.completionKey)) return undefined;
348+ state.completedKeys.add(input.completion.completionKey);
349+ state.completedOrder.push(input.completion.completionKey);
350+ while (state.completedOrder.length > MAX_COMPLETED_TURN_ANALYTICS_KEYS) {
351+ const expired = state.completedOrder.shift();
352+ if (expired) state.completedKeys.delete(expired);
353+ }
354+
355+ const session = state.sessions.get(input.sessionKey);
356+ const metadata = session?.activeByTurnId.get(input.turnId)?.metadata;
357+ session?.activeByTurnId.delete(input.turnId);
358+ session?.deferredCompletionsByTurnId.delete(input.turnId);
359+ if (
360+ session &&
361+ session.activeByTurnId.size === 0 &&
362+ session.pendingByRequestId.size === 0 &&
363+ session.deferredCompletionsByTurnId.size === 0
364+ ) {
365+ state.sessions.delete(input.sessionKey);
366+ }
367+
368+ return {
369+ ...input.completion.terminalProperties,
370+ ...(metadata?.model ? { model: metadata.model } : {}),
371+ ...(metadata?.effort ? { effort: metadata.effort } : {}),
372+ ...(metadata?.interactionMode ? { interactionMode: metadata.interactionMode } : {}),
373+ ...(metadata?.runtimeMode ? { runtimeMode: metadata.runtimeMode } : {}),
374+ ...(metadata ? { mixedModels: metadata.mixedModels } : {}),
375+ ...(metadata
376+ ? { durationMs: Math.max(0, input.completion.completedAtMs - metadata.startedAtMs) }
377+ : {}),
378+ };
379+ };
380+
381+ const recordCompletedTurnProperties = (
382+ properties: ReadonlyArray<Readonly<Record<string, unknown>>>,
383+ ) =>
384+ Effect.forEach(properties, (entry) => analytics.record("provider.turn.completed", entry), {
385+ discard: true,
386+ });
387+
388+ const clearTurnAnalyticsSession = (providerInstanceId: ProviderInstanceId, threadId: ThreadId) =>
389+ Effect.gen(function* () {
390+ const properties = yield* Ref.modify(turnAnalytics, (state) => {
391+ const sessionKey = turnAnalyticsSessionKey(providerInstanceId, threadId);
392+ const session = state.sessions.get(sessionKey);
393+ const completed: Array<Readonly<Record<string, unknown>>> = [];
394+ if (session) {
395+ for (const [turnId, completion] of session.deferredCompletionsByTurnId) {
396+ const entry = finishTurnAnalytics(state, { sessionKey, turnId, completion });
397+ if (entry) completed.push(entry);
398+ }
399+ }
400+ state.sessions.delete(sessionKey);
401+ return [completed, state] as const;
402+ });
403+ yield* recordCompletedTurnProperties(properties);
404+ });
405+
406+ const beginTurnAnalytics = Effect.fn("beginTurnAnalytics")(function* (input: {
407+ readonly providerInstanceId: ProviderInstanceId;
408+ readonly provider: ProviderDriverKind;
409+ readonly threadId: ThreadId;
410+ readonly modelSelection: ProviderSendTurnInput["modelSelection"];
411+ readonly interactionMode: ProviderSendTurnInput["interactionMode"];
412+ readonly runtimeMode: string | undefined;
413+ }) {
414+ const startedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
415+ turnAnalyticsRequestId += 1;
416+ const requestId = turnAnalyticsRequestId;
417+ const effort = turnEffort(input.modelSelection);
418+ return yield* Ref.modify(turnAnalytics, (state) => {
419+ const key = turnAnalyticsSessionKey(input.providerInstanceId, input.threadId);
420+ const session = state.sessions.get(key) ?? {
421+ pendingByRequestId: new Map(),
422+ activeByTurnId: new Map(),
423+ deferredCompletionsByTurnId: new Map(),
424+ };
425+ const metadata: TurnAnalyticsMetadata = {
426+ provider: input.provider,
427+ startedAtMs,
428+ mixedModels: false,
429+ requestId,
430+ ...(input.modelSelection?.model ? { model: input.modelSelection.model } : {}),
431+ ...(effort ? { effort } : {}),
432+ ...(input.interactionMode ? { interactionMode: input.interactionMode } : {}),
433+ ...(input.runtimeMode ? { runtimeMode: input.runtimeMode } : {}),
434+ };
435+ session.pendingByRequestId.set(requestId, metadata);
436+ state.sessions.set(key, session);
437+ return [metadata, state] as const;
438+ });
439+ });
440+
441+ const clearPendingTurnAnalytics = (input: {
442+ readonly providerInstanceId: ProviderInstanceId;
443+ readonly threadId: ThreadId;
444+ readonly requestId: number;
445+ }) =>
446+ Effect.gen(function* () {
447+ const properties = yield* Ref.modify(turnAnalytics, (state) => {
448+ const sessionKey = turnAnalyticsSessionKey(input.providerInstanceId, input.threadId);
449+ const session = state.sessions.get(sessionKey);
450+ if (!session)
451+ return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const;
452+ session.pendingByRequestId.delete(input.requestId);
453+ const completed: Array<Readonly<Record<string, unknown>>> = [];
454+ if (session.pendingByRequestId.size === 0) {
455+ for (const [turnId, completion] of session.deferredCompletionsByTurnId) {
456+ const entry = finishTurnAnalytics(state, { sessionKey, turnId, completion });
457+ if (entry) completed.push(entry);
458+ }
459+ }
460+ if (
461+ session.activeByTurnId.size === 0 &&
462+ session.pendingByRequestId.size === 0 &&
463+ session.deferredCompletionsByTurnId.size === 0
464+ ) {
465+ state.sessions.delete(sessionKey);
466+ }
467+ return [completed, state] as const;
468+ });
469+ yield* recordCompletedTurnProperties(properties);
470+ });
471+
472+ const associateTurnAnalytics = (input: {
473+ readonly providerInstanceId: ProviderInstanceId;
474+ readonly threadId: ThreadId;
475+ readonly turnId: string;
476+ readonly metadata: TurnAnalyticsMetadata;
477+ }) =>
478+ Effect.gen(function* () {
479+ const properties = yield* Ref.modify(turnAnalytics, (state) => {
480+ const completionKey = turnAnalyticsCompletionKey(
481+ input.providerInstanceId,
482+ input.threadId,
483+ input.turnId,
484+ );
485+ const sessionKey = turnAnalyticsSessionKey(input.providerInstanceId, input.threadId);
486+ const session = state.sessions.get(sessionKey);
487+ if (!session || state.completedKeys.has(completionKey)) {
488+ if (session) {
489+ session.pendingByRequestId.delete(input.metadata.requestId);
490+ if (
491+ session.activeByTurnId.size === 0 &&
492+ session.pendingByRequestId.size === 0 &&
493+ session.deferredCompletionsByTurnId.size === 0
494+ ) {
495+ state.sessions.delete(sessionKey);
496+ }
497+ }
498+ return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const;
499+ }
500+ const existing = session.activeByTurnId.get(input.turnId);
501+ const existingMetadata = existing?.metadata;
502+ const base = existing?.requestAssociated ? existing.metadata : input.metadata;
503+ setActiveTurnAnalytics(session, input.turnId, {
504+ requestAssociated: true,
505+ metadata: {
506+ ...base,
507+ ...(existingMetadata?.model
508+ ? { model: existingMetadata.model }
509+ : input.metadata.model
510+ ? { model: input.metadata.model }
511+ : {}),
512+ ...(existingMetadata?.effort
513+ ? { effort: existingMetadata.effort }
514+ : input.metadata.effort
515+ ? { effort: input.metadata.effort }
516+ : {}),
517+ ...(base?.interactionMode
518+ ? {}
519+ : input.metadata.interactionMode
520+ ? { interactionMode: input.metadata.interactionMode }
521+ : {}),
522+ ...(base?.runtimeMode
523+ ? {}
524+ : input.metadata.runtimeMode
525+ ? { runtimeMode: input.metadata.runtimeMode }
526+ : {}),
527+ mixedModels: existingMetadata?.mixedModels ?? input.metadata.mixedModels,
528+ },
529+ });
530+ session.pendingByRequestId.delete(input.metadata.requestId);
531+ const completion = session.deferredCompletionsByTurnId.get(input.turnId);
532+ const completed = completion
533+ ? finishTurnAnalytics(state, {
534+ sessionKey,
535+ turnId: input.turnId,
536+ completion,
537+ })
538+ : undefined;
539+ return [completed ? [completed] : [], state] as const;
540+ });
541+ yield* recordCompletedTurnProperties(properties);
542+ });
543+
544+ const observeTurnStartedForAnalytics = Effect.fn("observeTurnStartedForAnalytics")(function* (
545+ source: { readonly instanceId: ProviderInstanceId; readonly provider: ProviderDriverKind },
546+ event: Extract<ProviderRuntimeEvent, { readonly type: "turn.started" }>,
547+ ) {
548+ if (!event.turnId) return;
549+ const observedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
550+ yield* Ref.update(turnAnalytics, (state) => {
551+ const completionKey = turnAnalyticsCompletionKey(
552+ source.instanceId,
553+ event.threadId,
554+ String(event.turnId),
555+ );
556+ if (state.completedKeys.has(completionKey)) return state;
557+ const sessionKey = turnAnalyticsSessionKey(source.instanceId, event.threadId);
558+ const session = state.sessions.get(sessionKey) ?? {
559+ pendingByRequestId: new Map(),
560+ activeByTurnId: new Map(),
561+ deferredCompletionsByTurnId: new Map(),
562+ };
563+ // A start never binds send metadata on its own. Claude can start a
564+ // synthetic turn for leftover agent output while sendTurn is still
565+ // preparing the real turn, so only the adapter's sendTurn response
566+ // links a request to its turn. Completions that land before that
567+ // response wait in deferredCompletionsByTurnId.
568+ const current = session.activeByTurnId.get(String(event.turnId));
569+ const metadata: TurnAnalyticsMetadata = {
570+ ...(current?.metadata ?? {
571+ requestId: ++turnAnalyticsRequestId,
572+ provider: source.provider,
573+ startedAtMs: observedAtMs,
574+ mixedModels: false,
575+ }),
576+ ...(event.payload.model ? { model: event.payload.model } : {}),
577+ ...(event.payload.effort ? { effort: event.payload.effort } : {}),
578+ };
579+ setActiveTurnAnalytics(session, String(event.turnId), {
580+ metadata,
581+ requestAssociated: current?.requestAssociated ?? false,
582+ });
583+ state.sessions.set(sessionKey, session);
584+ return state;
585+ });
586+ });
587+
588+ const observeModelReroutedForAnalytics = (
589+ source: { readonly instanceId: ProviderInstanceId },
590+ event: Extract<ProviderRuntimeEvent, { readonly type: "model.rerouted" }>,
591+ ) =>
592+ Ref.update(turnAnalytics, (state) => {
593+ const session = state.sessions.get(
594+ turnAnalyticsSessionKey(source.instanceId, event.threadId),
595+ );
596+ if (!session) return state;
597+ if (event.turnId) {
598+ const current = session.activeByTurnId.get(String(event.turnId));
599+ if (current) {
600+ session.activeByTurnId.set(String(event.turnId), {
601+ ...current,
602+ metadata: { ...current.metadata, mixedModels: true },
603+ });
604+ }
605+ } else {
606+ for (const [turnId, current] of session.activeByTurnId) {
607+ session.activeByTurnId.set(turnId, {
608+ ...current,
609+ metadata: { ...current.metadata, mixedModels: true },
610+ });
611+ }
612+ }
613+ return state;
614+ });
615+
616+ const recordTurnCompletedAnalytics = Effect.fn("recordTurnCompletedAnalytics")(function* (
617+ source: { readonly instanceId: ProviderInstanceId; readonly provider: ProviderDriverKind },
618+ event: Extract<ProviderRuntimeEvent, { readonly type: "turn.completed" | "turn.aborted" }>,
619+ ) {
620+ if (!event.turnId) return;
621+ const completedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
622+ const tokenUsage = event.payload.tokenUsage;
623+ const completion: DeferredTurnAnalyticsCompletion = {
624+ completionKey: turnAnalyticsCompletionKey(
625+ source.instanceId,
626+ event.threadId,
627+ String(event.turnId),
628+ ),
629+ completedAtMs,
630+ terminalProperties: {
631+ provider: source.provider,
632+ terminalStatus:
633+ event.type === "turn.completed"
634+ ? event.payload.state
635+ : event.payload.reason.toLowerCase().includes("interrupt")
636+ ? "interrupted"
637+ : "cancelled",
638+ usageStatus: tokenUsage?.usageStatus ?? "unavailable",
639+ usageScope: tokenUsage?.usageScope ?? "main_agent",
640+ ...(tokenUsage ? { hasSubagents: tokenUsage.hasSubagents } : {}),
641+ ...(tokenUsage?.inputTokens !== undefined ? { inputTokens: tokenUsage.inputTokens } : {}),
642+ ...(tokenUsage?.cachedInputTokens !== undefined
643+ ? { cachedInputTokens: tokenUsage.cachedInputTokens }
644+ : {}),
645+ ...(tokenUsage?.cacheCreationTokens !== undefined
646+ ? { cacheCreationTokens: tokenUsage.cacheCreationTokens }
647+ : {}),
648+ ...(tokenUsage?.outputTokens !== undefined
649+ ? { outputTokens: tokenUsage.outputTokens }
650+ : {}),
651+ ...(tokenUsage?.reasoningTokens !== undefined
652+ ? { reasoningTokens: tokenUsage.reasoningTokens }
653+ : {}),
654+ },
655+ };
656+ const properties = yield* Ref.modify(turnAnalytics, (state) => {
657+ if (state.completedKeys.has(completion.completionKey)) {
658+ return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const;
659+ }
660+ const turnId = String(event.turnId);
661+ const sessionKey = turnAnalyticsSessionKey(source.instanceId, event.threadId);
662+ const session = state.sessions.get(sessionKey);
663+ if (session?.deferredCompletionsByTurnId.has(turnId)) {
664+ return [[] as ReadonlyArray<Readonly<Record<string, unknown>>>, state] as const;
665+ }
666+ const active = session?.activeByTurnId.get(turnId);
667+ const needsAssociation =
668+ (session?.pendingByRequestId.size ?? 0) > 0 && active?.requestAssociated !== true;
669+ if (!session || !needsAssociation) {
670+ const completed = finishTurnAnalytics(state, { sessionKey, turnId, completion });
671+ return [completed ? [completed] : [], state] as const;
672+ }
673+
674+ session.deferredCompletionsByTurnId.set(turnId, completion);
675+ const completed: Array<Readonly<Record<string, unknown>>> = [];
676+ while (session.deferredCompletionsByTurnId.size > MAX_ACTIVE_TURN_ANALYTICS_PER_SESSION) {
677+ const oldest = session.deferredCompletionsByTurnId.entries().next().value;
678+ if (!oldest) break;
679+ const [oldestTurnId, oldestCompletion] = oldest;
680+ const entry = finishTurnAnalytics(state, {
681+ sessionKey,
682+ turnId: oldestTurnId,
683+ completion: oldestCompletion,
684+ });
685+ if (entry) completed.push(entry);
686+ }
687+ return [completed, state] as const;
688+ });
689+ yield* recordCompletedTurnProperties(properties);
690+ });
262691 /**
263692 * Attach the `t3-code` MCP server to the session that is about to start.
264693 *
@@ -437,6 +866,18 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( · 172 unmodified lines
437866 provider: canonicalEvent.provider,
438867 eventType: canonicalEvent.type,
439868 });
869+ if (canonicalEvent.type === "turn.started") {
870+ yield* observeTurnStartedForAnalytics(source, canonicalEvent);
871+ } else if (canonicalEvent.type === "model.rerouted") {
872+ yield* observeModelReroutedForAnalytics(source, canonicalEvent);
873+ } else if (
874+ canonicalEvent.type === "turn.completed" ||
875+ canonicalEvent.type === "turn.aborted"
876+ ) {
877+ yield* recordTurnCompletedAnalytics(source, canonicalEvent);
878+ } else if (canonicalEvent.type === "session.exited") {
879+ yield* clearTurnAnalyticsSession(source.instanceId, canonicalEvent.threadId);
880+ }
440881 if (
441882 isCompactedEvent(canonicalEvent) &&
442883 timedOutNativeCompactions.delete(canonicalEvent.threadId)
@@ -790,6 +1231,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( · 347 unmodified lines
7901231 "provider.cwd.effective": effectiveCwd ?? "",
7911232 });
7921233 const adapter = yield* registry.getByInstance(resolvedInstanceId);
1234+ yield* clearTurnAnalyticsSession(resolvedInstanceId, threadId);
7931235 yield* prepareMcpSession(threadId, resolvedInstanceId);
7941236 const session = yield* adapter
7951237 .startSession({
@@ -953,7 +1395,35 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( · 157 unmodified lines
9531395 // rather than issuing a new one: sessions that go a long time between
9541396 // browser tool calls used to lose the toolkit outright.
9551397 yield* McpSessionRegistry.touchActiveMcpThread(input.threadId);
956− const turn = yield* routed.adapter.sendTurn(input);
1398+ const analyticsModelSelection =
1399+ input.modelSelection?.instanceId === routed.instanceId ? input.modelSelection : undefined;
1400+ const turn = yield* Effect.acquireUseRelease(
1401+ beginTurnAnalytics({
1402+ providerInstanceId: routed.instanceId,
1403+ provider: routed.adapter.provider,
1404+ threadId: input.threadId,
1405+ modelSelection: analyticsModelSelection,
1406+ interactionMode: input.interactionMode,
1407+ runtimeMode: routed.runtimeMode,
1408+ }),
1409+ (turnMetadata) =>
1410+ Effect.gen(function* () {
1411+ const turn = yield* routed.adapter.sendTurn(input);
1412+ yield* associateTurnAnalytics({
1413+ providerInstanceId: routed.instanceId,
1414+ threadId: input.threadId,
1415+ turnId: String(turn.turnId),
1416+ metadata: turnMetadata,
1417+ });
1418+ return turn;
1419+ }),
1420+ (turnMetadata) =>
1421+ clearPendingTurnAnalytics({
1422+ providerInstanceId: routed.instanceId,
1423+ threadId: input.threadId,
1424+ requestId: turnMetadata.requestId,
1425+ }),
1426+ );
9571427 yield* directory.upsert({
9581428 threadId: input.threadId,
9591429 provider: routed.adapter.provider,
@@ -1254,6 +1724,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( · 294 unmodified lines
12541724 yield* settleCompaction(input.threadId, pendingCompaction, "turn.aborted");
12551725 }
12561726 timedOutNativeCompactions.delete(input.threadId);
1727+ yield* clearTurnAnalyticsSession(routed.instanceId, input.threadId);
12571728 yield* clearMcpSession(input.threadId);
12581729 yield* directory.upsert({
12591730 threadId: input.threadId,
@@ -1471,6 +1942,18 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( · 211 unmodified lines
14711942 );
14721943
14731944 const runStopAll = Effect.fn("runStopAll")(function* () {
1945+ const properties = yield* Ref.modify(turnAnalytics, (state) => {
1946+ const completed: Array<Readonly<Record<string, unknown>>> = [];
1947+ for (const [sessionKey, session] of state.sessions) {
1948+ for (const [turnId, completion] of session.deferredCompletionsByTurnId) {
1949+ const entry = finishTurnAnalytics(state, { sessionKey, turnId, completion });
1950+ if (entry) completed.push(entry);
1951+ }
1952+ }
1953+ state.sessions.clear();
1954+ return [completed, state] as const;
1955+ });
1956+ yield* recordCompletedTurnProperties(properties);
14741957 const threadIds = yield* directory.listThreadIds();
14751958 const currentAdapters = yield* getAdapterEntries;
14761959 const activeSessions = yield* Effect.forEach(currentAdapters, ([instanceId, adapter]) =>
apps/server/src/telemetry/AnalyticsService.test.ts+43−0
@@ -148,4 +148,47 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { · 147 unmodified lines
148148 );
149149 }),
150150 );
151+
152+ it.effect("does not send batch requests when telemetry is disabled", () =>
153+ Effect.gen(function* () {
154+ const capturedPaths: Array<string> = [];
155+ const serverConfigLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {
156+ prefix: "t3-telemetry-disabled-",
157+ });
158+ const telemetryLayer = AnalyticsService.layer.pipe(Layer.provideMerge(serverConfigLayer));
159+ const configLayer = ConfigProvider.layer(
160+ ConfigProvider.fromUnknown({
161+ T3CODE_TELEMETRY_ENABLED: false,
162+ T3CODE_POSTHOG_KEY: "phc_test_key",
163+ T3CODE_POSTHOG_HOST: "http://localhost",
164+ }),
165+ );
166+ const batchServerLayer = HttpServer.serve(
167+ Effect.gen(function* () {
168+ const request = yield* HttpServerRequest.HttpServerRequest;
169+ capturedPaths.push(request.url);
170+ return HttpServerResponse.jsonUnsafe({});
171+ }),
172+ );
173+ const runtimeLayer = telemetryLayer.pipe(
174+ Layer.provide(configLayer),
175+ Layer.provide(
176+ Layer.mergeAll(
177+ Layer.succeed(HostProcessPlatform, "linux"),
178+ Layer.succeed(HostProcessArchitecture, "arm64"),
179+ ),
180+ ),
181+ Layer.provideMerge(NodeHttpServer.layerTest),
182+ );
183+
184+ yield* Effect.gen(function* () {
185+ yield* Layer.launch(batchServerLayer).pipe(Effect.forkScoped);
186+ const analytics = yield* AnalyticsService.AnalyticsService;
187+ yield* analytics.record("test.disabled", { index: 1 });
188+ yield* analytics.flush;
189+ }).pipe(Effect.provide(runtimeLayer));
190+
191+ assert.deepEqual(capturedPaths, []);
192+ }),
193+ );
151194 });
docs/README.md+1−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 - [Keyboard shortcuts](./user/keybindings.md)
88 - [Organizing threads](./user/thread-sidebar.md)
99 - [Review usage](./user/usage.md)
10+- [Anonymous usage data](./user/telemetry.md)
1011 - [Customize a project icon](./user/project-settings.md)
1112 - [Mobile appearance](./user/mobile-appearance.md)
1213 - [Environment themes](./user/environment-theme.md)
docs/internals/product-analytics.md+68−3
@@ -23,6 +23,54 @@ time. · 22 unmodified lines
2323 client metadata because a provider turn can continue after the requesting
2424 client disconnects.
2525
26+## Provider turn events
27+
28+`provider.turn.completed` records one anonymous event when a provider emits a
29+canonical completed or aborted turn. The event reports normalized main-agent
30+token counts when the provider supplies enough data.
31+
32+| Property | Values and meaning |
33+| --------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
34+| `provider` | Provider driver, such as `codex`, `claudeAgent`, or `opencode`. Use this as the harness breakdown. |
35+| `model` | Selected or provider-reported model when known. This is metadata, not proof that one model produced every token. |
36+| `effort` | Selected or provider-reported reasoning effort when known. |
37+| `interactionMode` | Turn interaction mode, `default` or `plan`, when known. |
38+| `runtimeMode` | Session permission mode when known. |
39+| `mixedModels` | `true` when T3 Code observed `model.rerouted` during the turn. This does not split token counts by model. |
40+| `durationMs` | Time from the observed turn start or accepted send request to the terminal event. Omitted when start metadata is not available. |
41+| `terminalStatus` | `completed`, `failed`, `interrupted`, or `cancelled`. |
42+| `usageStatus` | `complete`, `partial`, or `unavailable`. |
43+| `usageScope` | `main_agent`. Child-agent token use is not included. |
44+| `hasSubagents` | Whether the adapter observed child agents during the turn. Omitted when the adapter does not provide a normalized usage report. |
45+| `inputTokens` | Total main-agent input tokens. This includes uncached input, cache reads, and cache creation or writes. |
46+| `cachedInputTokens` | Cache-read tokens. This is a subset of `inputTokens`. |
47+| `cacheCreationTokens` | Cache creation or write tokens. This is a subset of `inputTokens`. |
48+| `outputTokens` | Total main-agent output tokens, including reasoning tokens. |
49+| `reasoningTokens` | Reasoning tokens when the provider reports them separately. This is a subset of `outputTokens`. |
50+
51+`complete` means the provider supplied full input and output totals for the
52+turn. `partial` means each included count is valid, but T3 Code did not observe
53+the full turn. This can happen after an interruption, provider failure, or
54+reconnect. `unavailable` means the provider did not supply trustworthy counts.
55+Numeric properties are omitted when their values are unknown. They are not set
56+to zero.
57+
58+Codex uses deltas from its cumulative token totals, and falls back to the
59+newest response's `last` usage when no prior total exists or the total was
60+reset. Claude uses the final result's per-turn `usage` value and does not use
61+cumulative `modelUsage`.
62+OpenCode sums unique main-session step totals. Cursor, Grok, and Antigravity
63+usage stays `unavailable` until their ACP token fields and scope are verified.
64+
65+The completed event count does not have to match `provider.turn.sent`. For
66+example, Claude can emit synthetic turns without a matching send request.
67+Duplicate terminal events for the same provider instance, thread, and turn
68+produce one analytics event.
69+
70+Collection is best effort and starts with the first server release that
71+contains this event. T3 Code does not scan provider history or backfill older
72+turns.
73+
2674 ## Recommended properties
2775
2876 Client properties appear on the three client events when the connected client
@@ -94,6 +142,20 @@ In PostHog Data management, use the event and property descriptions from this · 65 unmodified lines
94142 document. Mark the recommended properties as verified. Keep `clientType`
95143 visible with its legacy description so old reports remain understandable.
96144
145+Create a second saved dashboard named `Provider token efficiency`. Start each
146+token insight with these filters:
147+
148+- `usageStatus = complete`
149+- `hasSubagents = false`
150+- `mixedModels = false`
151+
152+Break down input tokens, output tokens, and their ratio by `provider`. Compare
153+the same `model`, `effort`, `interactionMode`, and `terminalStatus`. Without
154+these filters, model choice, reasoning level, plan mode, failures, and child
155+agents can look like harness differences. Keep separate views for median turn
156+usage and aggregate `sum(outputTokens) / sum(inputTokens)`. Do not average each
157+turn's ratio because small-input turns can dominate that result.
158+
97159 ## Collection and release boundary
98160
99161 Client values are best effort. Invalid values are ignored and never reject a
@@ -101,9 +163,12 @@ connection. Browser clients use user-agent data for broad OS, browser, phone, · 1 unmodified line
101163 and tablet groups. They do not infer CPU architecture or an exact OS version
102164 from `navigator.platform`.
103165
104−This change does not collect URLs, tokens, prompts, IP addresses, or
105−user-assigned device names. It measures authenticated product use. It does not
106−measure a person who visits the hosted app without connecting to a server.
166+This change does not collect URLs, authentication tokens, prompts, IP
167+addresses, or user-assigned device names. The token event does not include
168+thread IDs, turn IDs, provider instance IDs, event IDs, request IDs, raw
169+provider payloads, or child-agent output. It does not scan provider history on
170+disk or start external work. It measures authenticated product use. It does
171+not measure a person who visits the hosted app without connecting to a server.
107172
108173 The new fields start with the first client and server release that contains
109174 this metadata path. Historical events cannot reliably identify the client OS,
docs/user/telemetry.md+15−0
@@ -0,0 +1,15 @@
1+# Anonymous usage data
2+
3+The T3 server sends anonymous product events to PostHog. These events include
4+the provider, model, reasoning effort, permission mode, turn result, duration,
5+and normalized main-agent token totals when the provider reports them.
6+
7+T3 Code does not send prompts, responses, file contents, raw provider events,
8+thread IDs, turn IDs, provider instance IDs, authentication tokens, or child
9+agent output. Token totals can be complete, partial, or unavailable. Child
10+agent token use is not included.
11+
12+Set `T3CODE_TELEMETRY_ENABLED=false` before you start the T3 server to stop
13+product events from being recorded or sent. The server still derives an
14+anonymous identifier during startup and can create its local fallback
15+identifier file.
packages/contracts/src/providerRuntime.test.ts+46−0
@@ -6,6 +6,52 @@ import { classifyTaskAgentKind, ProviderRuntimeEvent } from "./providerRuntime.t · 5 unmodified lines
66 const decodeRuntimeEvent = Schema.decodeUnknownSync(ProviderRuntimeEvent);
77
88 describe("ProviderRuntimeEvent", () => {
9+ it("requires input and output totals for complete turn usage", () => {
10+ const completeEvent = {
11+ type: "turn.completed",
12+ eventId: "event-complete-usage",
13+ provider: "codex",
14+ createdAt: "2026-02-28T00:00:00.000Z",
15+ threadId: "thread-1",
16+ turnId: "turn-1",
17+ payload: {
18+ state: "completed",
19+ tokenUsage: {
20+ usageStatus: "complete",
21+ usageScope: "main_agent",
22+ hasSubagents: false,
23+ },
24+ },
25+ };
26+
27+ expect(() => decodeRuntimeEvent(completeEvent)).toThrow();
28+ expect(
29+ decodeRuntimeEvent({
30+ ...completeEvent,
31+ payload: {
32+ ...completeEvent.payload,
33+ tokenUsage: {
34+ ...completeEvent.payload.tokenUsage,
35+ inputTokens: 10,
36+ outputTokens: 2,
37+ },
38+ },
39+ }).type,
40+ ).toBe("turn.completed");
41+ expect(
42+ decodeRuntimeEvent({
43+ ...completeEvent,
44+ payload: {
45+ ...completeEvent.payload,
46+ tokenUsage: {
47+ ...completeEvent.payload.tokenUsage,
48+ usageStatus: "partial",
49+ },
50+ },
51+ }).type,
52+ ).toBe("turn.completed");
53+ });
54+
955 it("accepts fork-provided driver kinds as branded slugs", () => {
1056 const parsed = decodeRuntimeEvent({
1157 type: "session.started",
packages/contracts/src/providerRuntime.ts+31−0
@@ -367,6 +367,35 @@ const TurnStartedPayload = Schema.Struct({ · 366 unmodified lines
367367 });
368368 export type TurnStartedPayload = typeof TurnStartedPayload.Type;
369369
370+/**
371+ * Normalized main-agent usage for one turn. Input includes cache reads and
372+ * writes. Output includes reasoning, and reasoningTokens is an optional subset.
373+ * Complete means the provider supplied full input and output totals. Partial
374+ * means every included count is valid, but the full turn total is not known.
375+ */
376+const TurnTokenUsageCommonFields = {
377+ usageScope: Schema.Literal("main_agent"),
378+ cachedInputTokens: Schema.optional(NonNegativeInt),
379+ cacheCreationTokens: Schema.optional(NonNegativeInt),
380+ reasoningTokens: Schema.optional(NonNegativeInt),
381+ hasSubagents: Schema.Boolean,
382+};
383+export const TurnTokenUsage = Schema.Union([
384+ Schema.Struct({
385+ ...TurnTokenUsageCommonFields,
386+ usageStatus: Schema.Literal("complete"),
387+ inputTokens: NonNegativeInt,
388+ outputTokens: NonNegativeInt,
389+ }),
390+ Schema.Struct({
391+ ...TurnTokenUsageCommonFields,
392+ usageStatus: Schema.Literals(["partial", "unavailable"]),
393+ inputTokens: Schema.optional(NonNegativeInt),
394+ outputTokens: Schema.optional(NonNegativeInt),
395+ }),
396+]);
397+export type TurnTokenUsage = typeof TurnTokenUsage.Type;
398+
370399 const TurnCompletedPayload = Schema.Struct({
371400 state: RuntimeTurnState,
372401 stopReason: Schema.optional(Schema.NullOr(TrimmedNonEmptyStringSchema)),
@@ -374,11 +403,13 @@ const TurnCompletedPayload = Schema.Struct({ · 1 unmodified line
374403 modelUsage: Schema.optional(UnknownRecordSchema),
375404 totalCostUsd: Schema.optional(Schema.Number),
376405 errorMessage: Schema.optional(TrimmedNonEmptyStringSchema),
406+ tokenUsage: Schema.optional(TurnTokenUsage),
377407 });
378408 export type TurnCompletedPayload = typeof TurnCompletedPayload.Type;
379409
380410 const TurnAbortedPayload = Schema.Struct({
381411 reason: TrimmedNonEmptyStringSchema,
412+ tokenUsage: Schema.optional(TurnTokenUsage),
382413 });
383414 export type TurnAbortedPayload = typeof TurnAbortedPayload.Type;
384415

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t