// --------------------------------------------------------------------------- // Effect → OTEL → Axiom bridge // --------------------------------------------------------------------------- // // Both the fetch path and Durable Object path install a Worker-safe // WebTracerProvider in their own isolate. We deliberately avoid global fetch // instrumentation libraries here: they proxy Cloudflare-native functions and // can break `this` binding inside Worker-only clients. // // We install a `WebTracerProvider` once per isolate as the global provider // (lazy on first layer provide, not at module load — `env` from // `cloudflare:workers` is reliably populated at request time but we keep the // lazy gate as a defensive cheap no-op). Once installed, the provider lives for // the entire isolate lifetime, so deferred MCP SDK callbacks — which fire after // the request Effect has resolved — still hit a live `SimpleSpanProcessor` + // exporter. // // Previously the WebSdk layer was scoped per-request: when the outer // `Effect.runPromise(...)` resolved, the layer's scope closed and // `processor.shutdown()` ran. Engine / runtime spans created from deferred SDK // callbacks (which captured the old runtime + tracer) then silently failed to // export, even though they showed up in `Effect.currentSpan` traces during // execution. // --------------------------------------------------------------------------- // Subpath imports — the barrel `@effect/opentelemetry` re-exports `NodeSdk`, // which eagerly imports `@opentelemetry/sdk-trace-node` and its // `context-async-hooks` dep. Under vitest-pool-workers that crashes module // load (no `async_hooks` in workerd). Production bundles tree-shake the // unused NodeSdk; vitest does not. import * as Resource from "@effect/opentelemetry/Resource"; import * as OtelTracer from "@effect/opentelemetry/Tracer"; import { trace } from "@opentelemetry/api"; // Force the browser platform entry — the package's conditional export would // otherwise resolve to the Node build, which uses `https.request` / `node:http`. // Under workerd + unenv's nodejs_compat, `https.request` isn't implemented // (surfaces as `[unenv] https.request is not implemented yet!` at export // time) and every DO span fails to ship. The browser build uses `fetch()`, // which workerd does support. import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http/build/esm/platform/browser/index.js"; import { resourceFromAttributes } from "@opentelemetry/resources"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; import { WebTracerProvider } from "@opentelemetry/sdk-trace-web"; import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions"; import { env } from "cloudflare:workers"; import { Effect, Layer } from "effect"; import { SpanHeaderRedactionLive } from "./header-redaction"; import { CountingSpanExporter, CountingSpanProcessor, OTEL_MAX_SPAN_QUEUE_SIZE, recordForceFlush, } from "./memory-metrics"; const SERVICE_NAME = "executor-cloud"; // `service.version` is the Cloudflare Worker version id (from the // `version_metadata` binding) so any span links back to the exact deploy in a // step-change investigation; "dev" is the documented default for hosts without // the binding (local dev, older test workers). `executor.commit_sha` rides // along when CI passed it (`wrangler deploy --var GIT_COMMIT_SHA:...`). const serviceVersion = (): string => env.CF_VERSION_METADATA?.id ?? "dev"; // One id per isolate: distinguishes "many isolates each paying a cold cost" // from "one isolate is slow", and makes per-isolate cache behavior (JWKS, // module caches) measurable from Axiom. The Aug 2026 latency investigation // stalled for lack of exactly this attribute. // // Generated LAZILY on first use, not at module scope: workerd forbids random // generation (and I/O) in global scope and Cloudflare's upload validation // rejects the whole deploy for it (error 10021). First use is inside // `installTracerProvider()` / the telemetry layer build, which both run in a // request handler, so the id is still one-per-isolate. let isolateInstanceId: string | null = null; let isolateStartedAt: number | null = null; const resourceAttributes = (): Record => { isolateInstanceId ??= crypto.randomUUID(); isolateStartedAt ??= Date.now(); return { "service.instance.id": isolateInstanceId, "executor.isolate_started_at": new Date(isolateStartedAt).toISOString(), ...(env.GIT_COMMIT_SHA === undefined ? {} : { "executor.commit_sha": env.GIT_COMMIT_SHA }), }; }; // Module-scope: one provider per isolate, never shut down. The provider holds // the SimpleSpanProcessor + OTLP exporter, so any tracer reference captured by // deferred work keeps finding a live exporter after the request Effect resolves. let provider: WebTracerProvider | null = null; const ensureGlobalTracerProvider = (): boolean => { if (provider) return true; if (!env.AXIOM_TOKEN) return false; provider = new WebTracerProvider({ resource: resourceFromAttributes({ [ATTR_SERVICE_NAME]: SERVICE_NAME, [ATTR_SERVICE_VERSION]: serviceVersion(), ...resourceAttributes(), }), spanProcessors: (() => { let countingProcessor: CountingSpanProcessor; const exporter = new CountingSpanExporter( new OTLPTraceExporter({ url: env.AXIOM_TRACES_URL ?? "https://api.axiom.co/v1/traces", headers: { Authorization: `Bearer ${env.AXIOM_TOKEN}`, "X-Axiom-Dataset": env.AXIOM_DATASET ?? "executor-cloud", }, }), (spans) => countingProcessor.recordExportAttempt(spans), ); // Batch, not Simple: SimpleSpanProcessor issues one synchronous fetch // per span END — a busy MCP request emits dozens of spans, and the // serialized export fetches add seconds of latency inside the request // (enough to flip pause/resume timing in e2e). Batch buffers and // ships on a timer; `ctx.waitUntil(flushTracerProvider())` at the end // of each request still drains the buffer before the isolate exits. countingProcessor = new CountingSpanProcessor( new BatchSpanProcessor(exporter, { scheduledDelayMillis: 1_000, maxExportBatchSize: 512, maxQueueSize: OTEL_MAX_SPAN_QUEUE_SIZE, }), OTEL_MAX_SPAN_QUEUE_SIZE, ); return [countingProcessor]; })(), }); // Skip `provider.register()` — its StackContextManager / W3C propagator // setup wires the global OTel context API, but Effect's tracer goes // through `OtelTracer.layerGlobal` which only needs the global provider, // not the OTel context machinery. trace.setGlobalTracerProvider(provider); return true; }; // Worker-entry use: install lazily at the start of the fetch handler so the // outer `http.server` span can be opened with a real provider, and flush via // `ctx.waitUntil(flushTracerProvider())` so SimpleSpanProcessor exports // survive request termination. export const installTracerProvider = (): boolean => ensureGlobalTracerProvider(); export const flushTracerProvider = async (): Promise => { if (!provider) return; const startedAt = Date.now(); // oxlint-disable-next-line executor/no-try-catch-or-throw -- observability boundary; preserve forceFlush failure behavior while counting it try { await provider.forceFlush(); recordForceFlush(Date.now() - startedAt, false); } catch (error) { recordForceFlush(Date.now() - startedAt, true); // oxlint-disable-next-line executor/no-try-catch-or-throw -- preserve original rejection for waitUntil diagnostics throw error; } }; const makeTelemetryLive = (): Layer.Layer => Layer.mergeAll( // Redaction applies even when the exporter is not installed: Effect still // builds spans (and their header attributes) in-memory, and any future // consumer of those spans must never observe an unredacted credential. SpanHeaderRedactionLive, Layer.unwrap( Effect.sync(() => ensureGlobalTracerProvider() ? OtelTracer.layerGlobal.pipe( Layer.provide( Resource.layer({ serviceName: SERVICE_NAME, serviceVersion: serviceVersion(), attributes: resourceAttributes(), }), ), ) : Layer.empty, ), ), ); export const WorkerTelemetryLive: Layer.Layer = makeTelemetryLive(); export const DoTelemetryLive: Layer.Layer = makeTelemetryLive();