History

fix(mcp): report completed connected tool (#1964)

* fix(mcp): report completed connected tool * Test queue timeout with a controlled clock * Verify connected tool provenance over MCP * Exclude upstream tool failures from completion metadata ---------

Dhairyashil Shinde committed 2 months ago

Commit 89b0f8d1 parent d64639b8 files · +191 −14Browse files

Unified · Split

8 files
  1. .changeset/completed-mcp-tool-name.md +5 −0
  2. e2e/scenarios/tool-call-contract.test.ts +96 −6
  3. packages/core/execution/src/engine.test.ts +29 −0
  4. packages/core/execution/src/engine.ts +21 −5
  5. packages/core/execution/src/tool-invoker.test.ts +19 −0
  6. packages/core/execution/src/tool-invoker.ts +10 −1
  7. packages/hosts/mcp/src/tool-server.test.ts +9 −2
  8. packages/kernel/core/src/types.ts +2 −0
.changeset/completed-mcp-tool-name.md+5−0
@@ -0,0 +1,5 @@
1+---
2+"@executor-js/execution": patch
3+---
4+
5+Completed MCP execute results now include `toolName` when a script successfully uses exactly one connected tool. Executions that use distinct tools remain unlabeled, and internal call provenance is not exposed in the MCP response.
e2e/scenarios/tool-call-contract.test.ts+96−6
@@ -21,7 +21,7 @@ import { randomBytes } from "node:crypto"; · 20 unmodified lines
2121 import { createServer } from "node:http";
2222
2323 import { expect } from "@effect/vitest";
24−import { Effect } from "effect";
24+import { Effect, Schema } from "effect";
2525 import { composePluginApi } from "@executor-js/api/server";
2626 import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
2727 import {
@@ -38,6 +38,13 @@ import type { McpSession } from "../src/surfaces/mcp"; · 10 unmodified lines
3838
3939 const api = composePluginApi([openApiHttpPlugin()] as const);
4040
41+const completion = Schema.Struct({
42+ structuredContent: Schema.Struct({
43+ status: Schema.String,
44+ toolName: Schema.optional(Schema.String),
45+ }),
46+});
47+
4148 const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`;
4249
4350 type UpstreamHandle = {
@@ -114,14 +121,14 @@ const executeApproved = (session: McpSession, code: string) => · 70 unmodified lines
114121 guard += 1;
115122 }
116123 expect(result.ok, `execute completed (got: ${result.text.slice(0, 400)})`).toBe(true);
117− return result.text;
124+ return result;
118125 });
119126
120127 /** Invoke a dynamic tool by full address and parse the envelope it returns. */
121128 const invokeEnvelope = (session: McpSession, address: string, args: unknown = {}) =>
122129 Effect.map(
123130 executeApproved(session, invokeByAddressCode(address, args)),
124− (text) => JSON.parse(text) as ToolEnvelope,
131+ (result) => JSON.parse(result.text) as ToolEnvelope,
125132 );
126133
127134 // ---------------------------------------------------------------------------
@@ -136,6 +143,20 @@ const widgetsSpec = (baseUrl: string): string => · 8 unmodified lines
136143 info: { title: "Widgets API", version: "1.0.0" },
137144 servers: [{ url: baseUrl }],
138145 paths: {
146+ "/unavailable": {
147+ get: {
148+ operationId: "unavailableWidgets",
149+ summary: "Unavailable widgets",
150+ responses: { "200": { description: "widgets" } },
151+ },
152+ },
153+ "/widgets/count": {
154+ get: {
155+ operationId: "countWidgets",
156+ summary: "Count widgets",
157+ responses: { "200": { description: "widget count" } },
158+ },
159+ },
139160 "/widgets": {
140161 get: {
141162 operationId: "listWidgets",
@@ -184,7 +205,7 @@ scenario( · 42 unmodified lines
184205 },
185206 });
186207 const created = JSON.parse(
187− yield* executeApproved(session, createConnectionCode(slug)),
208+ (yield* executeApproved(session, createConnectionCode(slug))).text,
188209 ) as ToolEnvelope;
189210 expect(created.ok, `the no-auth connection was created: ${JSON.stringify(created)}`).toBe(
190211 true,
@@ -199,7 +220,8 @@ scenario( · 8 unmodified lines
199220 const path = address!.replace(/^tools\./, "");
200221
201222 // 1. A well-addressed call executes and carries the upstream's payload.
202− const success = yield* invokeEnvelope(session, address!);
223+ const successfulCall = yield* executeApproved(session, invokeByAddressCode(address!, {}));
224+ const success = JSON.parse(successfulCall.text) as ToolEnvelope;
203225 expect(
204226 success.ok,
205227 `the call succeeded (got: ${JSON.stringify(success.error ?? {}).slice(0, 400)})`,
@@ -207,7 +229,73 @@ scenario( · 1 unmodified line
207229 expect(JSON.stringify(success.data), "the upstream's payload comes back").toContain(
208230 "anvil",
209231 );
232+ const structured = (successfulCall.raw as { readonly structuredContent?: unknown })
233+ .structuredContent;
234+ expect(structured, "the completed MCP result includes structured content").toMatchObject({
235+ status: "completed",
236+ toolName: path,
237+ });
238+ expect(structured, "the internal tool-call trace is not exposed").not.toHaveProperty(
239+ "toolPaths",
240+ );
210241 expect(upstream.requests(), "the upstream served exactly one call").toBe(1);
242+ const discovery = yield* executeApproved(
243+ session,
244+ 'return await tools.search({ query: "widgets" });',
245+ );
246+ const discoveryResult = yield* Schema.decodeUnknownEffect(completion)(discovery.raw);
247+ expect(discoveryResult.structuredContent.toolName).toBeUndefined();
248+ expect(discovery.raw).not.toHaveProperty("structuredContent.toolPaths");
249+
250+ const failed = yield* executeApproved(
251+ session,
252+ invokeByAddressCode(address!.replace(/listWidgets$/, "missingWidget"), {}),
253+ );
254+ const failedResult = yield* Schema.decodeUnknownEffect(completion)(failed.raw);
255+ expect(failedResult.structuredContent.toolName).toBeUndefined();
256+ expect(failed.raw).not.toHaveProperty("structuredContent.toolPaths");
257+ const unavailableAddress = yield* Schema.decodeUnknownEffect(Schema.String)(
258+ tools
259+ .filter((tool) => String(tool.integration) === slug)
260+ .map((tool) => String(tool.address))
261+ .find((candidate) => candidate.endsWith("unavailableWidgets")),
262+ );
263+ const upstreamFailure = yield* executeApproved(
264+ session,
265+ invokeByAddressCode(unavailableAddress, {}),
266+ );
267+ const upstreamFailureResult = yield* Schema.decodeUnknownEffect(completion)(
268+ upstreamFailure.raw,
269+ );
270+ expect(upstreamFailure.text).toContain('"ok":false');
271+ expect(upstreamFailureResult.structuredContent.toolName).toBeUndefined();
272+ expect(upstreamFailure.raw).not.toHaveProperty("structuredContent.toolPaths");
273+
274+ const repeated = yield* executeApproved(
275+ session,
276+ `
277+ await tools[${JSON.stringify(path)}]({});
278+ return await tools[${JSON.stringify(path)}]({});
279+ `,
280+ );
281+ const repeatedResult = yield* Schema.decodeUnknownEffect(completion)(repeated.raw);
282+ expect(repeatedResult.structuredContent.toolName).toBe(path);
283+ expect(repeated.raw).not.toHaveProperty("structuredContent.toolPaths");
284+ const anotherAddress = tools
285+ .filter((tool) => String(tool.integration) === slug)
286+ .map((tool) => String(tool.address))
287+ .find((candidate) => candidate.endsWith("countWidgets"));
288+ const anotherPath = yield* Schema.decodeUnknownEffect(Schema.String)(anotherAddress);
289+ const multiple = yield* executeApproved(
290+ session,
291+ `
292+ await tools[${JSON.stringify(path)}]({});
293+ return await tools[${JSON.stringify(anotherPath.replace(/^tools\./, ""))}]({});
294+ `,
295+ );
296+ const multipleResult = yield* Schema.decodeUnknownEffect(completion)(multiple.raw);
297+ expect(multipleResult.structuredContent.toolName).toBeUndefined();
298+ expect(multiple.raw).not.toHaveProperty("structuredContent.toolPaths");
211299
212300 // 2a. A wrong TOOL name on a live connection: tool_not_found, and the
213301 // suggestions name the connection's real tools so the agent can
@@ -257,7 +345,9 @@ scenario( · 43 unmodified lines
257345 "the defect mask never surfaces for a missing connection",
258346 ).not.toContain("Internal tool error");
259347
260− expect(upstream.requests(), "no misaddressed call ever reached the upstream").toBe(1);
348+ expect(upstream.requests(), "only the five successful calls reached the upstream").toBe(
349+ 5,
350+ );
261351 }),
262352 // Selfhost shares one workspace identity — leaked resources fail other
263353 // scenarios' zero-state assertions.
packages/core/execution/src/engine.test.ts+29−0
@@ -309,6 +309,35 @@ describe("formatExecuteResult output identity", () => { · 308 unmodified lines
309309 expect(formatted.isError).toBe(false);
310310 });
311311
312+ it("returns the sole distinct connected tool name without exposing the call trace", () => {
313+ const result = {
314+ result: { issues: [] },
315+ logs: [],
316+ toolPaths: ["linear.org.work.issues.list", "linear.org.work.issues.list"],
317+ } as ExecuteResult & { readonly toolPaths: readonly string[] };
318+
319+ const formatted = formatExecuteResult(result);
320+
321+ expect(formatted.structured).toEqual({
322+ status: "completed",
323+ result: { issues: [] },
324+ toolName: "linear.org.work.issues.list",
325+ logs: [],
326+ });
327+ });
328+
329+ it("omits a tool name when distinct connected tools were used", () => {
330+ const result = {
331+ result: { issues: [], projects: [] },
332+ logs: [],
333+ toolPaths: ["linear.org.work.issues.list", "linear.org.work.projects.list"],
334+ } as ExecuteResult & { readonly toolPaths: readonly string[] };
335+
336+ const formatted = formatExecuteResult(result);
337+
338+ expect(formatted.structured).not.toHaveProperty("toolName");
339+ });
340+
312341 it("truncates a long preview with the exact suffix and untouched structured value", () => {
313342 const value = { data: "é🎉".repeat(12_000) };
314343 const pretty = JSON.stringify(value, null, 2);
packages/core/execution/src/engine.ts+21−5
@@ -138,6 +138,11 @@ const truncate = (value: string, max: number): string => · 137 unmodified lines
138138 ? `${value.slice(0, max)}\n... [truncated ${value.length - max} chars]`
139139 : value;
140140
141+const soleConnectedToolName = (toolPaths: readonly string[] | undefined): string | undefined => {
142+ const names = [...new Set(toolPaths ?? [])];
143+ return names.length === 1 ? names[0] : undefined;
144+};
145+
141146 export const formatExecuteResult = (
142147 result: ExecuteResult,
143148 ): {
@@ -183,11 +188,13 @@ export const formatExecuteResult = ( · 39 unmodified lines
183188 ? `(no return value; ${emittedNote})`
184189 : "(no result)";
185190 const parts = [resultPart, ...(logText ? [`\nLogs:\n${logText}`] : [])];
191+ const toolName = soleConnectedToolName(result.toolPaths);
186192 return {
187193 text: parts.join("\n"),
188194 structured: {
189195 status: "completed",
190196 result: result.result ?? null,
197+ ...(toolName ? { toolName } : {}),
191198 ...emittedField,
192199 logs: result.logs ?? [],
193200 },
@@ -318,8 +325,9 @@ const makeFullInvoker = ( · 124 unmodified lines
318325 executor: Executor,
319326 invokeOptions: InvokeOptions,
320327 toolDiscoveryProvider: ToolDiscoveryProvider,
328+ onConnectedToolCall?: (path: string) => void,
321329 ): SandboxToolInvoker => {
322− const base = makeExecutorToolInvoker(executor, { invokeOptions });
330+ const base = makeExecutorToolInvoker(executor, { invokeOptions, onConnectedToolCall });
323331 return {
324332 invoke: ({ path, args }) => {
325333 if (path === "search") {
@@ -694,13 +702,18 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu · 368 unmodified lines
694702 return yield* Deferred.await(responseDeferred);
695703 });
696704
705+ const toolPaths: string[] = [];
697706 const invoker = makeFullInvoker(
698707 executor,
699708 { onElicitation: elicitationHandler },
700709 toolDiscoveryProvider,
710+ (path) => toolPaths.push(path),
701711 );
702712 fiber = yield* Effect.forkDetach(
703− codeExecutor.execute(code, invoker).pipe(Effect.withSpan("executor.code.exec")),
713+ codeExecutor.execute(code, invoker).pipe(
714+ Effect.map((result) => (toolPaths.length === 0 ? result : { ...result, toolPaths })),
715+ Effect.withSpan("executor.code.exec"),
716+ ),
704717 );
705718 liveSandboxFibers.add(fiber);
706719
@@ -825,16 +838,19 @@ export const createExecutionEngine = <E extends Cause.YieldableError = CodeExecu · 118 unmodified lines
825838 "mcp.execute.mode": "inline",
826839 "mcp.execute.code_length": code.length,
827840 });
841+ const toolPaths: string[] = [];
828842 const invoker = makeFullInvoker(
829843 executor,
830844 {
831845 onElicitation: options.onElicitation,
832846 },
833847 toolDiscoveryProvider,
848+ (path) => toolPaths.push(path),
849+ );
850+ const result = yield* codeExecutor.execute(code, invoker).pipe(
851+ Effect.map((result) => (toolPaths.length === 0 ? result : { ...result, toolPaths })),
852+ Effect.withSpan("executor.code.exec"),
834853 );
835− const result = yield* codeExecutor
836− .execute(code, invoker)
837− .pipe(Effect.withSpan("executor.code.exec"));
838854 yield* annotateExecuteOutcome(result);
839855 return result;
840856 });
packages/core/execution/src/tool-invoker.test.ts+19−0
@@ -674,6 +674,25 @@ describe("tool discovery", () => { · 673 unmodified lines
674674 }),
675675 );
676676
677+ it.effect("records only the connected tool resolved after discovery", () =>
678+ Effect.gen(function* () {
679+ const executor = yield* makeSearchExecutor();
680+ const engine = createExecutionEngine({ executor, codeExecutor });
681+
682+ const execution = yield* engine.execute(
683+ [
684+ 'const search = await tools.search({ query: "repository details", namespace: "github", limit: 1 });',
685+ 'const result = await tools[search.items[0].path]({ owner: "executor", repo: "executor" });',
686+ "return result;",
687+ ].join("\n"),
688+ { onElicitation: acceptAll },
689+ );
690+
691+ expect(execution.error).toBeUndefined();
692+ expect(execution.toolPaths).toEqual(["github.org.main.getRepositoryDetails"]);
693+ }),
694+ );
695+
677696 it.effect("lets execution hosts provide custom tool discovery", () =>
678697 Effect.gen(function* () {
679698 const executor = yield* makeSearchExecutor();
packages/core/execution/src/tool-invoker.ts+10−1
@@ -306,7 +306,10 @@ const extractNamespace = (path: string): string => { · 305 unmodified lines
306306 */
307307 export const makeExecutorToolInvoker = (
308308 executor: Executor,
309− options: { readonly invokeOptions: InvokeOptions },
309+ options: {
310+ readonly invokeOptions: InvokeOptions;
311+ readonly onConnectedToolCall?: (path: string) => void;
312+ },
310313 ): SandboxToolInvoker => ({
311314 invoke: Effect.fn("mcp.tool.dispatch")(function* ({ path, args }) {
312315 yield* Effect.annotateCurrentSpan({
@@ -372,6 +375,12 @@ export const makeExecutorToolInvoker = ( · 59 unmodified lines
372375 // outcome annotation the dispatch span reads as healthy even when the
373376 // caller hit an upstream error or auth wall.
374377 yield* annotateToolResultOutcome(result);
378+ const connectedToolPath = parseToolAddress(String(address))
379+ ? addressToPath(String(address))
380+ : undefined;
381+ if (connectedToolPath && (!isToolResult(result) || result.ok)) {
382+ options.onConnectedToolCall?.(connectedToolPath);
383+ }
375384 if (isToolResult(result)) {
376385 return result;
377386 }
packages/hosts/mcp/src/tool-server.test.ts+9−2
@@ -1039,12 +1039,15 @@ describe("MCP host server — native form-only elicitation", () => { · 1038 unmodified lines
10391039 // ---------------------------------------------------------------------------
10401040
10411041 describe("MCP host server — client without elicitation (pause/resume)", () => {
1042− it("completed execution returns result directly", async () => {
1042+ it("completed execution returns result and connected-tool metadata directly", async () => {
10431043 const engine = makeStubEngine({
10441044 executeWithPause: () =>
10451045 Effect.succeed({
10461046 status: "completed",
1047− result: { result: "done" },
1047+ result: {
1048+ result: "done",
1049+ toolPaths: ["linear.org.work.issues.list"],
1050+ },
10481051 }),
10491052 });
10501053
@@ -1054,6 +1057,10 @@ describe("MCP host server — client without elicitation (pause/resume)", () => · 3 unmodified lines
10541057 arguments: { code: "ok" },
10551058 });
10561059 expect(result.content).toEqual([{ type: "text", text: "done" }]);
1060+ expect(result.structuredContent).toMatchObject({
1061+ status: "completed",
1062+ toolName: "linear.org.work.issues.list",
1063+ });
10571064 expect(result.isError).toBeFalsy();
10581065 });
10591066 });
packages/kernel/core/src/types.ts+2−0
@@ -46,6 +46,8 @@ export type ExecuteResult = { · 45 unmodified lines
4646 /** Enumerable failure class for telemetry; never carries message content. */
4747 errorKind?: ExecuteErrorKind;
4848 logs?: string[];
49+ /** Successful connected-tool paths observed during this execution. */
50+ toolPaths?: readonly string[];
4951 };
5052
5153 /**