// Cloud: the telemetry contract, end to end. A tool call that hits an// upstream error wall must be visible in the EXPORTED spans — not just// handled gracefully for the caller. This is the regression class where the// product silently goes dark to operators: `ToolResult.fail` rides the// Effect success channel (a healthy-looking span), and an attribute stamped// on the wrong span simply never arrives in the trace store, which looks// identical to health. So the assertion runs against the OTLP store the dev// stack actually exported to (the suite's motel — the same exporter layer// that ships prod spans to Axiom), driving the whole production topology:// HTTP API → execution engine → sandbox → OpenAPI invoke → a real upstream// returning 502 → span batch → OTLP export.//// Pins two regressions found live in prod (2026-06-12): http.status_code was// annotated inside the inner `OpenApi.invoke` span so the `plugin.openapi.// invoke` span queries target carried it on 0 of ~19.5k spans; and failed// tool calls were indistinguishable from successes on `executor.tool.execute`.import { randomBytes } from "node:crypto";import { createServer } from "node:http"; import { expect } from "@effect/vitest";import { Effect } from "effect";import { composePluginApi } from "@executor-js/api/server";import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; import { scenario } from "../src/scenario";import { Api, Target, Telemetry } from "../src/services"; const api = composePluginApi([openApiHttpPlugin()] as const); /** Two operations: /ok answers 200, /fail answers 502 — the success and * expected-upstream-failure outcome classes the telemetry must separate. */const upstreamSpec = (baseUrl: string): string => JSON.stringify({ openapi: "3.0.3", info: { title: "Telemetry Upstream", version: "1.0.0" }, servers: [{ url: baseUrl }], paths: { "/ok": { get: { operationId: "ok", summary: "Succeeds", tags: ["probe"], responses: { "200": { description: "" } }, }, }, "/fail": { get: { operationId: "fail", summary: "Always 502", tags: ["probe"], responses: { "200": { description: "" } }, }, }, }, }); /** A real upstream on 127.0.0.1: /ok → 200 JSON, anything else → 502 JSON. */const serveUpstream = Effect.acquireRelease( Effect.callback<{ readonly baseUrl: string; readonly close: () => void }>((resume) => { const server = createServer((request, response) => { const ok = request.url?.startsWith("/ok") ?? false; response.writeHead(ok ? 200 : 502, { "content-type": "application/json" }); response.end(ok ? '{"fine":true}' : '{"error":{"message":"bad gateway"}}'); }); server.listen(0, "127.0.0.1", () => { const address = server.address(); const port = typeof address === "object" && address ? address.port : 0; resume( Effect.succeed({ baseUrl: `http://127.0.0.1:${port}`, close: () => { server.close(); server.closeAllConnections(); }, }), ); }); }), (server) => Effect.sync(server.close),); scenario( "Telemetry · a failing tool call is visible in the exported spans", { timeout: 180_000 }, Effect.scoped( Effect.gen(function* () { const target = yield* Target; const { client: apiClient } = yield* Api; const telemetry = yield* Telemetry; const identity = yield* target.newIdentity(); const client = yield* apiClient(api, identity); const upstream = yield* serveUpstream; // Identifier-safe slug: it becomes a property path in the sandbox code. const slug = IntegrationSlug.make(`telscn${randomBytes(4).toString("hex")}`); yield* client.openapi.addSpec({ payload: { spec: { kind: "blob", value: upstreamSpec(upstream.baseUrl) }, slug, baseUrl: upstream.baseUrl, authenticationTemplate: [ { slug: "apiKey", type: "apiKey", headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] }, }, ], }, }); yield* client.connections.create({ payload: { owner: "org", name: ConnectionName.make("main"), integration: slug, template: AuthTemplateSlug.make("apiKey"), value: "telemetry-scenario-token", }, }); const tools = yield* client.tools.list({ query: {} }); const addressOf = (op: string) => { const tool = tools.find( (entry) => String(entry.integration) === String(slug) && String(entry.address).endsWith(`.${op}`), ); expect(tool, `the ${op} tool is in the catalog`).toBeDefined(); return String(tool!.address); }; const failAddress = addressOf("fail"); const okAddress = addressOf("ok"); // Drive both outcome classes through the full production path. The // failing call still completes for the caller — that is exactly why // the exported span is the only place an operator can see it. for (const address of [okAddress, failAddress]) { const execution = yield* client.executions.execute({ payload: { code: `return await ${address}({});` }, }); expect(execution.status, `the ${address} execution completes`).toBe("completed"); } // The failure: outcome attributes on the tool span... const failSpan = yield* telemetry.expectSpan({ operation: "executor.tool.execute", attributes: { "mcp.tool.name": failAddress }, }); expect(failSpan.span.tags, "a failed tool call is marked on the exported span").toMatchObject( { "executor.tool.outcome": "fail", "executor.tool.error_code": "upstream_http_error", "executor.tool.error_status": "502", }, ); expect( failSpan.span.tags["executor.tenant"], "the span carries tenant attribution (no trace-join needed to ask 'whose error?')", ).toBeTruthy(); // ...and the upstream status on the HTTP span queries actually target. const invokeSpan = yield* telemetry.expectSpan({ operation: "plugin.openapi.invoke", attributes: { "plugin.openapi.base_url": upstream.baseUrl, "http.status_code": "502" }, }); expect( invokeSpan.span.tags["plugin.openapi.method"], "the invoke span names the method", ).toBe("GET"); // The success is distinguishable from the failure. const okSpan = yield* telemetry.expectSpan({ operation: "executor.tool.execute", attributes: { "mcp.tool.name": okAddress }, }); expect(okSpan.span.tags["executor.tool.outcome"], "a successful call is marked ok").toBe( "ok", ); }), ),);
// Cloud: the telemetry contract, end to end. A tool call that hits an
// upstream error wall must be visible in the EXPORTED spans — not just
// handled gracefully for the caller. This is the regression class where the
// product silently goes dark to operators: `ToolResult.fail` rides the
// Effect success channel (a healthy-looking span), and an attribute stamped
// on the wrong span simply never arrives in the trace store, which looks
// identical to health. So the assertion runs against the OTLP store the dev
// stack actually exported to (the suite's motel — the same exporter layer
// that ships prod spans to Axiom), driving the whole production topology:
// HTTP API → execution engine → sandbox → OpenAPI invoke → a real upstream
// returning 502 → span batch → OTLP export.
//
// Pins two regressions found live in prod (2026-06-12): http.status_code was
// annotated inside the inner `OpenApi.invoke` span so the `plugin.openapi.
// invoke` span queries target carried it on 0 of ~19.5k spans; and failed
// tool calls were indistinguishable from successes on `executor.tool.execute`.
import { randomBytes } from "node:crypto";
import { createServer } from "node:http";
import { expect } from "@effect/vitest";
import { Effect } from "effect";
import { composePluginApi } from "@executor-js/api/server";
import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
import { scenario } from "../src/scenario";
import { Api, Target, Telemetry } from "../src/services";
const api = composePluginApi([openApiHttpPlugin()] as const);
/** Two operations: /ok answers 200, /fail answers 502 — the success and
* expected-upstream-failure outcome classes the telemetry must separate. */
const upstreamSpec = (baseUrl: string): string =>
JSON.stringify({
openapi: "3.0.3",
info: { title: "Telemetry Upstream", version: "1.0.0" },
servers: [{ url: baseUrl }],
paths: {
"/ok": {
get: {
operationId: "ok",
summary: "Succeeds",
tags: ["probe"],
responses: { "200": { description: "" } },
},
},
"/fail": {
get: {
operationId: "fail",
summary: "Always 502",
tags: ["probe"],
responses: { "200": { description: "" } },
},
},
},
});
/** A real upstream on 127.0.0.1: /ok → 200 JSON, anything else → 502 JSON. */
const serveUpstream = Effect.acquireRelease(
Effect.callback<{ readonly baseUrl: string; readonly close: () => void }>((resume) => {
const server = createServer((request, response) => {
const ok = request.url?.startsWith("/ok") ?? false;
response.writeHead(ok ? 200 : 502, { "content-type": "application/json" });
response.end(ok ? '{"fine":true}' : '{"error":{"message":"bad gateway"}}');
});
server.listen(0, "127.0.0.1", () => {
const address = server.address();
const port = typeof address === "object" && address ? address.port : 0;
resume(
Effect.succeed({
baseUrl: `http://127.0.0.1:${port}`,
close: () => {
server.close();
server.closeAllConnections();
},
}),
);
});
}),
(server) => Effect.sync(server.close),
);
scenario(
"Telemetry · a failing tool call is visible in the exported spans",
{ timeout: 180_000 },
Effect.scoped(
Effect.gen(function* () {
const target = yield* Target;
const { client: apiClient } = yield* Api;
const telemetry = yield* Telemetry;
const identity = yield* target.newIdentity();
const client = yield* apiClient(api, identity);
const upstream = yield* serveUpstream;
// Identifier-safe slug: it becomes a property path in the sandbox code.
const slug = IntegrationSlug.make(`telscn${randomBytes(4).toString("hex")}`);
yield* client.openapi.addSpec({
payload: {
spec: { kind: "blob", value: upstreamSpec(upstream.baseUrl) },
slug,
baseUrl: upstream.baseUrl,
authenticationTemplate: [
{
slug: "apiKey",
type: "apiKey",
headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
},
],
},
});
yield* client.connections.create({
payload: {
owner: "org",
name: ConnectionName.make("main"),
integration: slug,
template: AuthTemplateSlug.make("apiKey"),
value: "telemetry-scenario-token",
},
});
const tools = yield* client.tools.list({ query: {} });
const addressOf = (op: string) => {
const tool = tools.find(
(entry) =>
String(entry.integration) === String(slug) && String(entry.address).endsWith(`.${op}`),
);
expect(tool, `the ${op} tool is in the catalog`).toBeDefined();
return String(tool!.address);
};
const failAddress = addressOf("fail");
const okAddress = addressOf("ok");
// Drive both outcome classes through the full production path. The
// failing call still completes for the caller — that is exactly why
// the exported span is the only place an operator can see it.
for (const address of [okAddress, failAddress]) {
const execution = yield* client.executions.execute({
payload: { code: `return await ${address}({});` },
});
expect(execution.status, `the ${address} execution completes`).toBe("completed");
}
// The failure: outcome attributes on the tool span...
const failSpan = yield* telemetry.expectSpan({
operation: "executor.tool.execute",
attributes: { "mcp.tool.name": failAddress },
});
expect(failSpan.span.tags, "a failed tool call is marked on the exported span").toMatchObject(
{
"executor.tool.outcome": "fail",
"executor.tool.error_code": "upstream_http_error",
"executor.tool.error_status": "502",
},
);
expect(
failSpan.span.tags["executor.tenant"],
"the span carries tenant attribution (no trace-join needed to ask 'whose error?')",
).toBeTruthy();
// ...and the upstream status on the HTTP span queries actually target.
const invokeSpan = yield* telemetry.expectSpan({
operation: "plugin.openapi.invoke",
attributes: { "plugin.openapi.base_url": upstream.baseUrl, "http.status_code": "502" },
});
expect(
invokeSpan.span.tags["plugin.openapi.method"],
"the invoke span names the method",
).toBe("GET");
// The success is distinguishable from the failure.
const okSpan = yield* telemetry.expectSpan({
operation: "executor.tool.execute",
attributes: { "mcp.tool.name": okAddress },
});
expect(okSpan.span.tags["executor.tool.outcome"], "a successful call is marked ok").toBe(
"ok",
);
}),
),
);