Serve runtime-observed output shapes for schemaless tools (#1759)

* Serve runtime-observed output shapes for schemaless tools * Mark observed types inline in the rendered signature

Rhys Sullivan committed 2 months ago

Commit 30cfb4d1 parent 2c75d7a12 files · +777 −27Browse files

Unified · Split

12 files

apps/desktop/build/entitlements.mac.plist +0 −21
apps/desktop/build/icon.png +0 −0
e2e/scenarios/shape-memory.test.ts +178 −0
packages/core/execution/src/skills.ts +1 −0
packages/core/execution/src/tool-invoker.test.ts +27 −0
packages/core/execution/src/tool-invoker.ts +19 −3
packages/core/sdk/src/executor.test.ts +57 −0
packages/core/sdk/src/executor.ts +47 −4
packages/core/sdk/src/shape-inference.test.ts +136 −0
packages/core/sdk/src/shape-inference.ts +219 −0
packages/core/sdk/src/shape-memory.ts +90 −0
packages/core/sdk/src/types.ts +4 −0

apps/desktop/build/entitlements.mac.plist+0−21

@@ -1,21 +0,0 @@
1		−<?xml version="1.0" encoding="UTF-8"?>
2		−<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3		−<plist version="1.0">
4		−<dict>
5		− <!-- Electron's V8 / Chromium needs JIT pages writable+executable -->
6		− <key>com.apple.security.cs.allow-jit</key>
7		− <true/>
8		− <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
9		− <true/>
10		− <!-- Electron reads NODE_OPTIONS / NODE_PATH at startup; the bundled executor
11		− reads EXECUTOR_* env vars. Hardened runtime would strip them
12		− without this entitlement. -->
13		− <key>com.apple.security.cs.allow-dyld-environment-variables</key>
14		− <true/>
15		− <!-- The Bun-compiled executor binary inside Resources/executor/
16		− loads dylibs Apple's library validator doesn't recognize as part
17		− of the Electron bundle's signing chain. -->
18		− <key>com.apple.security.cs.disable-library-validation</key>
19		− <true/>
20		−</dict>
21		−</plist>

apps/desktop/build/icon.png+0−0

Binary file not shown.

e2e/scenarios/shape-memory.test.ts+178−0

@@ -0,0 +1,178 @@
	1	+// Cross-target: muscle memory — runtime-observed output shapes. Most OpenAPI
	2	+// operations declare no response schema, so `tools.describe.tool()` used to
	3	+// render `data: unknown` forever and the model had to guess response shapes.
	4	+// This journey proves the warm path end to end through public surfaces only:
	5	+// a schemaless tool describes as `unknown`, one real invocation against a live
	6	+// upstream teaches the shape, and the very next describe serves a real
	7	+// TypeScript type marked as observed.
	8	+import { randomBytes } from "node:crypto";
	9	+import { createServer } from "node:http";
	10	+
	11	+import { expect } from "@effect/vitest";
	12	+import { Effect } from "effect";
	13	+import { composePluginApi } from "@executor-js/api/server";
	14	+import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
	15	+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
	16	+
	17	+import { scenario } from "../src/scenario";
	18	+import { Api, Target } from "../src/services";
	19	+
	20	+const api = composePluginApi([openApiHttpPlugin()] as const);
	21	+
	22	+/** One GET operation whose 200 declares no response schema — the shape the
	23	+ * model would otherwise have to guess. */
	24	+const issuesSpec = JSON.stringify({
	25	+ openapi: "3.0.3",
	26	+ info: { title: "Issues API", version: "1.0.0" },
	27	+ paths: {
	28	+ "/issues": {
	29	+ get: {
	30	+ operationId: "listIssues",
	31	+ summary: "List issues",
	32	+ responses: { "200": { description: "issues" } },
	33	+ },
	34	+ },
	35	+ },
	36	+});
	37	+
	38	+/** A live upstream for the single invocation that teaches the shape. */
	39	+const serveIssuesFixture = Effect.acquireRelease(
	40	+ Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => {
	41	+ const server = createServer((_request, response) => {
	42	+ response.writeHead(200, { "content-type": "application/json" });
	43	+ response.end(
	44	+ JSON.stringify({
	45	+ issues: [
	46	+ { id: 1, title: "first", open: true },
	47	+ { id: 2, title: "second", open: false },
	48	+ ],
	49	+ total: 2,
	50	+ }),
	51	+ );
	52	+ });
	53	+ server.listen(0, "127.0.0.1", () => {
	54	+ const addressInfo = server.address();
	55	+ const port = typeof addressInfo === "object" && addressInfo !== null ? addressInfo.port : 0;
	56	+ resume(
	57	+ Effect.succeed({
	58	+ url: `http://127.0.0.1:${port}`,
	59	+ close: () => {
	60	+ server.close();
	61	+ server.closeAllConnections();
	62	+ },
	63	+ }),
	64	+ );
	65	+ });
	66	+ }),
	67	+ (fixture) => Effect.sync(fixture.close),
	68	+);
	69	+
	70	+const describeCode = (slug: string) => `
	71	+const details = await tools.describe.tool({ path: "${slug}.org.main.issues.listIssues" });
	72	+return {
	73	+ outputTypeScript: details.outputTypeScript ?? null,
	74	+ note: details.outputTypeScriptNote ?? null,
	75	+ error: details.error ?? null,
	76	+};
	77	+`;
	78	+
	79	+type DescribeOutcome = {
	80	+ readonly outputTypeScript: string \| null;
	81	+ readonly note: string \| null;
	82	+ readonly error: unknown;
	83	+};
	84	+
	85	+scenario(
	86	+ "Muscle memory · a schemaless tool's observed output shape reaches describe",
	87	+ {},
	88	+ Effect.scoped(
	89	+ Effect.gen(function* () {
	90	+ const target = yield* Target;
	91	+ const { client: makeApiClient } = yield* Api;
	92	+ const identity = yield* target.newIdentity();
	93	+ const client = yield* makeApiClient(api, identity);
	94	+ const slug = IntegrationSlug.make(`shape_memory_${randomBytes(4).toString("hex")}`);
	95	+ const upstream = yield* serveIssuesFixture;
	96	+
	97	+ yield* Effect.ensuring(
	98	+ Effect.gen(function* () {
	99	+ yield* client.openapi.addSpec({
	100	+ payload: {
	101	+ spec: { kind: "blob", value: issuesSpec },
	102	+ slug,
	103	+ baseUrl: upstream.url,
	104	+ authenticationTemplate: [
	105	+ {
	106	+ slug: "apiKey",
	107	+ type: "apiKey",
	108	+ headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] },
	109	+ },
	110	+ ],
	111	+ },
	112	+ });
	113	+ yield* client.connections.create({
	114	+ payload: {
	115	+ owner: "org",
	116	+ name: ConnectionName.make("main"),
	117	+ integration: slug,
	118	+ template: AuthTemplateSlug.make("apiKey"),
	119	+ value: `key_${randomBytes(8).toString("hex")}`,
	120	+ },
	121	+ });
	122	+
	123	+ const describe = Effect.gen(function* () {
	124	+ const executed = yield* client.executions.execute({
	125	+ payload: { code: describeCode(String(slug)), autoApprove: true },
	126	+ });
	127	+ expect(executed.status, executed.text).toBe("completed");
	128	+ return JSON.parse(executed.text) as DescribeOutcome;
	129	+ });
	130	+
	131	+ // Cold: no declared response schema — the model sees unknown.
	132	+ const cold = yield* describe;
	133	+ expect(cold.error, "the tool resolves").toBeNull();
	134	+ expect(cold.outputTypeScript, "cold describe has no shape").toContain("data: unknown;");
	135	+ expect(cold.note, "cold describe carries no provenance note").toBeNull();
	136	+
	137	+ // One real call against the live upstream teaches the shape.
	138	+ const invoked = yield* client.executions.execute({
	139	+ payload: {
	140	+ code: `
	141	+const result = await tools.${slug}.org.main.issues.listIssues({});
	142	+return { ok: result.ok };
	143	+`,
	144	+ autoApprove: true,
	145	+ },
	146	+ });
	147	+ expect(invoked.status, invoked.text).toBe("completed");
	148	+ expect(JSON.parse(invoked.text), "the teaching call succeeded").toEqual({ ok: true });
	149	+
	150	+ // Warm: the observed shape is served, marked as observed.
	151	+ const warm = yield* describe;
	152	+ expect(warm.outputTypeScript, "warm describe serves the observed shape").toContain(
	153	+ "issues",
	154	+ );
	155	+ expect(warm.outputTypeScript, "field types come from the live payload").toContain(
	156	+ "total",
	157	+ );
	158	+ expect(warm.outputTypeScript, "the shape no longer collapses").not.toContain(
	159	+ "data: unknown;",
	160	+ );
	161	+ expect(warm.note, "provenance is explicit").toContain("observed from 1 live response");
	162	+ }),
	163	+ Effect.gen(function* () {
	164	+ yield* client.connections
	165	+ .remove({
	166	+ params: {
	167	+ owner: "org",
	168	+ integration: slug,
	169	+ name: ConnectionName.make("main"),
	170	+ },
	171	+ })
	172	+ .pipe(Effect.ignore);
	173	+ yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore);
	174	+ }),
	175	+ );
	176	+ }),
	177	+ ),
	178	+);

packages/core/execution/src/skills.ts+1−0

@@ -58,6 +58,7 @@ const EXECUTE_SKILL_BODY = [ · 57 unmodified lines
58	58	"- The `tools` object is a lazy proxy — enumerating it (`Object.keys(tools)`, spread, `for...in`) throws. Use `tools.search()` or `tools.executor.coreTools.connections.list({})` instead.",
59	59	'- Pass an object to system tools, e.g. `tools.search({ query: "..." })`, `tools.executor.coreTools.connections.list({})`, and `tools.describe.tool({ path })`.',
60	60	'- `tools.describe.tool()` returns compact TypeScript shapes. Use `inputTypeScript`, `outputTypeScript`, and `typeScriptDefinitions`. If the path doesn\'t resolve, the result carries `error: { code: "tool_not_found", suggestions }` — use a suggestion instead of retrying the same path.',
	61	+ "- When `outputTypeScriptNote` is present, the `data` type was observed from live responses rather than declared by the provider: the listed fields are reliable, but the shape may be incomplete — prefer optional access for anything not listed.",
61	62	"- For tools that return large collections (e.g. `getStates`, `getAll`), filter results in code rather than calling per-item tools.",
62	63	"- Do not use `fetch` — all API calls go through `tools.*`.",
63	64	"- If execution pauses for interaction, resume it with the returned `resumePayload`.",

packages/core/execution/src/tool-invoker.test.ts+27−0

@@ -877,6 +877,33 @@ describe("tool discovery", () => { · 876 unmodified lines
877	877	}),
878	878	);
879	879
	880	+ it.effect("serves an observed shape with a provenance note once a schemaless tool runs", () =>
	881	+ Effect.gen(function* () {
	882	+ const executor = yield* makeSearchExecutor();
	883	+ const invoker = makeExecutorToolInvoker(executor, {
	884	+ invokeOptions: { onElicitation: acceptAll },
	885	+ });
	886	+
	887	+ // Cold: no declared output schema — data renders as unknown, no note.
	888	+ const cold = yield* describeTool(executor, "github.org.main.listRepositoryIssues");
	889	+ expect(cold.outputTypeScript).toContain("data: unknown;");
	890	+ expect(cold.outputTypeScriptNote).toBeUndefined();
	891	+
	892	+ yield* invoker.invoke({
	893	+ path: "github.org.main.listRepositoryIssues",
	894	+ args: { owner: "executor", repo: "executor" },
	895	+ });
	896	+
	897	+ // Warm: the live `[]` payload becomes the served type, marked observed
	898	+ // both inline and via the note.
	899	+ const warm = yield* describeTool(executor, "github.org.main.listRepositoryIssues");
	900	+ expect(warm.outputTypeScript).toBe(
	901	+ "{ ok: true; data: unknown[] /* observed; may be incomplete */; http?: ToolHttpMeta } \| { ok: false; error: ToolError }",
	902	+ );
	903	+ expect(warm.outputTypeScriptNote).toContain("observed from 1 live response");
	904	+ }),
	905	+ );
	906	+
880	907	it.effect("describes a return type that accepts the sandbox invocation result", () =>
881	908	Effect.gen(function* () {
882	909	const executor = yield* makeSearchExecutor();

packages/core/execution/src/tool-invoker.ts+19−3

@@ -31,8 +31,12 @@ const TOOL_HTTP_META_TYPESCRIPT = "{ status: number; headers: { [k: string]: str · 30 unmodified lines
31	31	const TOOL_FILE_TYPESCRIPT =
32	32	'{ _tag: "ToolFile"; name?: string; mimeType: string; encoding: "base64"; data: string; byteLength: number; }';
33	33
34		−const wrapOutputTypeScript = (outputTypeScript?: string): string =>
35		− `{ ok: true; data: ${outputTypeScript ?? "unknown"}; http?: ToolHttpMeta } \| { ok: false; error: ToolError }`;
	34	+const wrapOutputTypeScript = (outputTypeScript?: string, marker?: string): string =>
	35	+ `{ ok: true; data: ${outputTypeScript ?? "unknown"}${marker ?? ""}; http?: ToolHttpMeta } \| { ok: false; error: ToolError }`;
	36	+
	37	+/** Inline provenance for observed types — a model that copies only the type
	38	+ * string still sees the hint, since the compact render drops descriptions. */
	39	+const OBSERVED_TYPE_MARKER = " /* observed; may be incomplete */";
36	40
37	41	const withToolResultDefinitions = (
38	42	definitions?: Record<string, string>,
@@ -76,6 +80,7 @@ type DescribedTool = { · 37 unmodified lines
76	80	readonly description?: string;
77	81	readonly inputTypeScript?: string;
78	82	readonly outputTypeScript?: string;
	83	+ readonly outputTypeScriptNote?: string;
79	84	readonly typeScriptDefinitions?: Record<string, string>;
80	85	/** Set when the path resolves to no tool — mirrors invoke's tool_not_found. */
81	86	readonly error?: {
@@ -865,7 +870,18 @@ export const describeTool = Effect.fn("executor.tools.describe")(function* ( · 783 unmodified lines
865	870	name: schema.name ?? path,
866	871	description: schema.description,
867	872	inputTypeScript: schema.inputTypeScript,
868		− outputTypeScript: wrapOutputTypeScript(schema.outputTypeScript),
	873	+ outputTypeScript: wrapOutputTypeScript(
	874	+ schema.outputTypeScript,
	875	+ schema.outputSchemaSource === "observed" ? OBSERVED_TYPE_MARKER : undefined,
	876	+ ),
	877	+ // The compact TS render drops the schema's provenance description, so an
	878	+ // observed (runtime-inferred) shape gets an explicit note: the model
	879	+ // should treat the fields as reliable but not exhaustive.
	880	+ ...(schema.outputSchemaSource === "observed"
	881	+ ? {
	882	+ outputTypeScriptNote: `data type observed from ${schema.outputSchemaObservations ?? 1} live response(s), not declared by the provider; fields may be incomplete.`,
	883	+ }
	884	+ : {}),
869	885	typeScriptDefinitions: withToolResultDefinitions(schema.typeScriptDefinitions),
870	886	};
871	887	return described;

packages/core/sdk/src/executor.test.ts+57−0

@@ -673,3 +673,60 @@ describe("createExecutor", () => { · 672 unmodified lines
673	673	}),
674	674	);
675	675	});
	676	+
	677	+describe("muscle memory (observed output shapes)", () => {
	678	+ const provisioned = Effect.fn(function* () {
	679	+ const executor = yield* makeTestExecutor({
	680	+ plugins: [demoPlugin] as const,
	681	+ coreTools: { webBaseUrl: "http://localhost:3000" },
	682	+ });
	683	+ yield* executor.demo.seed();
	684	+ yield* executor.execute(ToolAddress.make("executor.coreTools.connections.create"), {
	685	+ owner: "org",
	686	+ name: String(CONN),
	687	+ integration: String(INTEG),
	688	+ template: String(TEMPLATE),
	689	+ identityLabel: "Demo",
	690	+ from: { provider: "memory", id: "secret-token" },
	691	+ });
	692	+ return executor;
	693	+ });
	694	+
	695	+ it.effect("serves an observed output shape once a schemaless tool has run", () =>
	696	+ Effect.gen(function* () {
	697	+ const executor = yield* provisioned();
	698	+
	699	+ // Cold: `run` declares no output schema, nothing observed yet.
	700	+ const cold = yield* executor.tools.schema(addr("run"));
	701	+ expect(cold?.outputSchema).toBeUndefined();
	702	+ expect(cold?.outputTypeScript).toBeUndefined();
	703	+
	704	+ yield* executor.execute(addr("run"), {});
	705	+
	706	+ // Warm: the live payload `{ ran: "run" }` becomes the served shape,
	707	+ // with provenance marked on the schema.
	708	+ const warm = yield* executor.tools.schema(addr("run"));
	709	+ expect(warm?.outputSchema).toMatchObject({
	710	+ type: "object",
	711	+ properties: { ran: { type: "string" } },
	712	+ required: ["ran"],
	713	+ description: "Observed from 1 live response; fields may be incomplete.",
	714	+ });
	715	+ expect(warm?.outputTypeScript).toContain("ran");
	716	+ expect(warm?.outputTypeScript).not.toBe("unknown");
	717	+ }),
	718	+ );
	719	+
	720	+ it.effect("never overrides a declared output schema with observations", () =>
	721	+ Effect.gen(function* () {
	722	+ const executor = yield* provisioned();
	723	+
	724	+ // `inspect` declares `outputSchema: { $ref: "#/$defs/Owner" }`; running
	725	+ // it observes `{ ran: "inspect" }`, which must not displace the
	726	+ // declared schema.
	727	+ yield* executor.execute(addr("inspect"), { pet: { lives: 9 } });
	728	+ const schema = yield* executor.tools.schema(addr("inspect"));
	729	+ expect(schema?.outputSchema).toEqual({ $ref: "#/$defs/Owner" });
	730	+ }),
	731	+ );
	732	+});

packages/core/sdk/src/executor.ts+47−4

@@ -176,7 +176,8 @@ import { · 175 unmodified lines
176	176	type EnterpriseManagedRollout,
177	177	} from "./oauth-ema";
178	178	import { connectionIdentifier } from "./connection-name-identifier";
179		−import { annotateToolResultOutcome } from "./tool-result";
	179	+import { annotateToolResultOutcome, isToolResult } from "./tool-result";
	180	+import { makeShapeMemory, observedShapeToJsonSchema, SHAPE_MEMORY_PLUGIN_ID } from "./shape-memory";
180	181	import { isUnauthorizedToolFailure } from "./auth-tool-failure";
181	182
182	183	const PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE = 90;
@@ -1664,6 +1665,13 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 1481 unmodified lines
1664	1665	const blobs = config.blobs ?? makeFumaBlobStore(fuma);
1665	1666	const transaction = <A, E>(effect: Effect.Effect<A, E>) => fuma.transaction(effect);
1666	1667
	1668	+ // Runtime-observed output shapes ("muscle memory"): learned on the
	1669	+ // execute success path, served by tools.schema when a tool declares no
	1670	+ // output schema. Backed by plugin_storage under a reserved system id.
	1671	+ const shapeMemory = makeShapeMemory(
	1672	+ makePluginStorageFacade({ core, pluginId: SHAPE_MEMORY_PLUGIN_ID, owner: ownerBinding }),
	1673	+ );
	1674	+
1667	1675	// Populated once, never mutated after startup.
1668	1676	const staticTools = new Map<string, StaticTools>();
1669	1677	const runtimes = new Map<string, PluginRuntime>();
@@ -4058,6 +4066,21 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 2388 unmodified lines
4058	4066	? projected.outputSchema
4059	4067	: tool.outputSchema;
4060	4068
	4069	+ // Muscle memory: when neither the catalog row nor the plugin's
	4070	+ // projection declares an output schema, serve the shape observed from
	4071	+ // live responses instead of letting the type collapse to `unknown`.
	4072	+ // The schema's description marks it as observed.
	4073	+ const observed =
	4074	+ outputSchema === undefined
	4075	+ ? yield* shapeMemory.recall(String(address), parsed.owner)
	4076	+ : null;
	4077	+ const effectiveOutputSchema =
	4078	+ outputSchema !== undefined
	4079	+ ? outputSchema
	4080	+ : observed !== null
	4081	+ ? observedShapeToJsonSchema(observed)
	4082	+ : undefined;
	4083	+
4061	4084	const definitionRows = yield* core.findMany("definition", {
4062	4085	where: (b: AnyCb) =>
4063	4086	b.and(
@@ -4069,12 +4092,12 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 5 unmodified lines
4069	4092	const defs = new Map<string, unknown>();
4070	4093	for (const def of definitionRows) defs.set(def.name, decodeJsonColumn(def.schema));
4071	4094
4072		− const referenced = collectReferencedDefinitions([inputSchema, outputSchema], defs);
	4095	+ const referenced = collectReferencedDefinitions([inputSchema, effectiveOutputSchema], defs);
4073	4096	const preview = yield* Effect.tryPromise({
4074	4097	try: () =>
4075	4098	buildToolTypeScriptPreview({
4076	4099	inputSchema,
4077		− outputSchema,
	4100	+ outputSchema: effectiveOutputSchema,
4078	4101	defs,
4079	4102	}),
4080	4103	catch: (cause) =>
@@ -4087,7 +4110,13 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 6 unmodified lines
4087	4110	name: tool.name,
4088	4111	description: tool.description,
4089	4112	inputSchema,
4090		− outputSchema,
	4113	+ outputSchema: effectiveOutputSchema,
	4114	+ ...(observed !== null
	4115	+ ? {
	4116	+ outputSchemaSource: "observed" as const,
	4117	+ outputSchemaObservations: observed.observations,
	4118	+ }
	4119	+ : {}),
4091	4120	schemaDefinitions:
4092	4121	Object.keys(referenced).length > 0
4093	4122	? (referenced as Record<string, unknown>)
@@ -4762,6 +4791,20 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 668 unmodified lines
4762	4791	// "tool ran fine" from "user hit an upstream error / auth wall"
4763	4792	// without parsing response bodies.
4764	4793	Effect.tap(annotateToolResultOutcome),
	4794	+ // Muscle memory: fold successful dynamic-tool payloads into the
	4795	+ // observed output shape. Static tools are hand-typed already;
	4796	+ // failures teach nothing about the success shape. Runs inline — not
	4797	+ // forked — so the write survives Workers request teardown; `observe`
	4798	+ // never fails, is size-bounded, and stops writing once the shape
	4799	+ // stabilizes, so steady-state cost is one cache lookup.
	4800	+ Effect.tap((result) => {
	4801	+ if (staticTools.has(String(address))) return Effect.void;
	4802	+ const parsed = parseToolAddress(String(address));
	4803	+ if (!parsed) return Effect.void;
	4804	+ const data = isToolResult(result) ? (result.ok ? result.data : undefined) : result;
	4805	+ if (data === undefined) return Effect.void;
	4806	+ return shapeMemory.observe(String(address), parsed.owner, data);
	4807	+ }),
4765	4808	Effect.withSpan("executor.tool.execute", {
4766	4809	attributes: {
4767	4810	"mcp.tool.name": String(address),

packages/core/sdk/src/shape-inference.test.ts+136−0

@@ -0,0 +1,136 @@
	1	+import { describe, expect, it } from "@effect/vitest";
	2	+
	3	+import { inferShape, mergeShapes, observeShape, type InferredShape } from "./shape-inference";
	4	+
	5	+describe("inferShape", () => {
	6	+ it("infers primitives without recording values", () => {
	7	+ expect(inferShape("secret token")).toEqual({ type: "string" });
	8	+ expect(inferShape(42)).toEqual({ type: "number" });
	9	+ expect(inferShape(true)).toEqual({ type: "boolean" });
	10	+ expect(inferShape(null)).toEqual({ type: "null" });
	11	+ expect(inferShape(undefined)).toEqual({ type: "null" });
	12	+ });
	13	+
	14	+ it("infers object structure with all keys required", () => {
	15	+ expect(inferShape({ id: "abc", count: 3 })).toEqual({
	16	+ type: "object",
	17	+ properties: { id: { type: "string" }, count: { type: "number" } },
	18	+ required: ["count", "id"],
	19	+ });
	20	+ });
	21	+
	22	+ it("merges sampled array elements into one item shape", () => {
	23	+ expect(inferShape([{ id: 1 }, { id: 2, label: "x" }])).toEqual({
	24	+ type: "array",
	25	+ items: {
	26	+ type: "object",
	27	+ properties: { id: { type: "number" }, label: { type: "string" } },
	28	+ required: ["id"],
	29	+ },
	30	+ });
	31	+ });
	32	+
	33	+ it("keeps an empty array itemless", () => {
	34	+ expect(inferShape([])).toEqual({ type: "array" });
	35	+ });
	36	+
	37	+ it("collapses wide objects to a map so data-bearing keys never persist", () => {
	38	+ const byEmail = Object.fromEntries(
	39	+ Array.from({ length: 40 }, (_, i) => [`user${i}@example.com`, { active: true }]),
	40	+ );
	41	+ const shape = inferShape(byEmail);
	42	+ expect(shape.properties).toBeUndefined();
	43	+ expect(shape.additionalProperties).toEqual({
	44	+ type: "object",
	45	+ properties: { active: { type: "boolean" } },
	46	+ required: ["active"],
	47	+ });
	48	+ });
	49	+
	50	+ it("degrades to unknown past the depth bound", () => {
	51	+ let value: unknown = "leaf";
	52	+ for (let i = 0; i < 10; i++) value = { child: value };
	53	+ const json = JSON.stringify(inferShape(value));
	54	+ expect(json).toContain("{}");
	55	+ });
	56	+});
	57	+
	58	+describe("mergeShapes", () => {
	59	+ it("makes fields missing from one observation optional", () => {
	60	+ const merged = mergeShapes(inferShape({ id: "a", email: "x@y.z" }), inferShape({ id: "b" }));
	61	+ expect(merged).toEqual({
	62	+ type: "object",
	63	+ properties: { id: { type: "string" }, email: { type: "string" } },
	64	+ required: ["id"],
	65	+ });
	66	+ });
	67	+
	68	+ it("unions differing primitive types", () => {
	69	+ expect(mergeShapes({ type: "string" }, { type: "number" })).toEqual({
	70	+ anyOf: [{ type: "string" }, { type: "number" }],
	71	+ });
	72	+ });
	73	+
	74	+ it("merges same-typed union branches instead of duplicating them", () => {
	75	+ const union = mergeShapes({ type: "string" }, { type: "null" });
	76	+ const widened = mergeShapes(union, inferShape({ id: 1 }));
	77	+ const again = mergeShapes(widened, inferShape({ id: 2, extra: true }));
	78	+ expect(again.anyOf).toHaveLength(3);
	79	+ const objectBranch = again.anyOf?.find((branch) => branch.type === "object");
	80	+ expect(objectBranch?.required).toEqual(["id"]);
	81	+ });
	82	+
	83	+ it("degrades to unknown when the union grows past its cap", () => {
	84	+ const wide = [
	85	+ { type: "string" } as const,
	86	+ { type: "number" } as const,
	87	+ { type: "boolean" } as const,
	88	+ { type: "null" } as const,
	89	+ { type: "array" } as const,
	90	+ ].reduce<InferredShape>((left, right) => mergeShapes(left, right), { type: "string" });
	91	+ expect(wide).toEqual({});
	92	+ });
	93	+
	94	+ it("merges array item shapes across calls", () => {
	95	+ const merged = mergeShapes(inferShape([{ id: 1 }]), inferShape([{ id: 2, done: false }]));
	96	+ expect(merged).toEqual({
	97	+ type: "array",
	98	+ items: {
	99	+ type: "object",
	100	+ properties: { id: { type: "number" }, done: { type: "boolean" } },
	101	+ required: ["id"],
	102	+ },
	103	+ });
	104	+ });
	105	+});
	106	+
	107	+describe("observeShape", () => {
	108	+ it("starts a record from the first observation", () => {
	109	+ const record = observeShape(null, { ok: true }, 1000);
	110	+ expect(record.observations).toBe(1);
	111	+ expect(record.updatedAt).toBe(1000);
	112	+ expect(record.schema.type).toBe("object");
	113	+ });
	114	+
	115	+ it("accumulates observations by merging", () => {
	116	+ const first = observeShape(null, { id: "a", email: "x@y.z" }, 1000);
	117	+ const second = observeShape(first, { id: "b" }, 2000);
	118	+ expect(second.observations).toBe(2);
	119	+ expect(second.updatedAt).toBe(2000);
	120	+ expect(second.schema.required).toEqual(["id"]);
	121	+ expect(Object.keys(second.schema.properties ?? {}).sort()).toEqual(["email", "id"]);
	122	+ });
	123	+
	124	+ it("stays under the serialized size bound for adversarial payloads", () => {
	125	+ const wide = Object.fromEntries(
	126	+ Array.from({ length: 24 }, (_, i) => [
	127	+ `field_with_a_rather_long_name_${i}`,
	128	+ Object.fromEntries(
	129	+ Array.from({ length: 24 }, (_, j) => [`nested_property_name_${j}`, { deep: { x: 1 } }]),
	130	+ ),
	131	+ ]),
	132	+ );
	133	+ const record = observeShape(null, wide, 1000);
	134	+ expect(JSON.stringify(record.schema).length).toBeLessThanOrEqual(16_000);
	135	+ });
	136	+});

packages/core/sdk/src/shape-inference.ts+219−0

@@ -0,0 +1,219 @@
	1	+/**
	2	+ * Runtime output-shape inference — the "muscle memory" half of code mode.
	3	+ *
	4	+ * Most tools declare no output schema, so `tools.describe.tool()` renders
	5	+ * `data: unknown` and the model guesses response shapes. This module infers a
	6	+ * lightweight shape from real tool results at dispatch time: field names and
	7	+ * broad types only, never values. Shapes merge across calls, so unions and
	8	+ * optional fields converge instead of thrashing.
	9	+ *
	10	+ * The output is a small JSON Schema subset (`type`, `properties`, `required`,
	11	+ * `items`, `additionalProperties`, `anyOf`) so it can be stored as plain JSON
	12	+ * and rendered through the same schema → TypeScript compiler that declared
	13	+ * schemas use.
	14	+ *
	15	+ * Every dimension is bounded — depth, object width, array sampling, union
	16	+ * width, serialized size — because inference runs on the hot dispatch path
	17	+ * against arbitrary upstream payloads.
	18	+ */
	19	+
	20	+export type InferredShape = {
	21	+ readonly type?: "null" \| "boolean" \| "number" \| "string" \| "object" \| "array";
	22	+ readonly properties?: Readonly<Record<string, InferredShape>>;
	23	+ readonly required?: readonly string[];
	24	+ readonly items?: InferredShape;
	25	+ readonly additionalProperties?: InferredShape;
	26	+ readonly anyOf?: readonly InferredShape[];
	27	+};
	28	+
	29	+/** `{}` — matches anything; the "we know nothing beyond this point" shape. */
	30	+const UNKNOWN: InferredShape = {};
	31	+
	32	+const MAX_DEPTH = 6;
	33	+/** Array elements sampled per call; later calls keep widening the merge. */
	34	+const MAX_ARRAY_SAMPLE = 5;
	35	+/**
	36	+ * An object with more own keys than this is treated as a map keyed by data
	37	+ * (ids, emails, dates) rather than a struct. Collapsing to
	38	+ * `additionalProperties` keeps data-bearing keys out of the shape — field
	39	+ * names of a struct are API surface, but map keys are values.
	40	+ */
	41	+const MAX_OBJECT_KEYS = 24;
	42	+/** Union width cap; beyond this the shape degrades to unknown. */
	43	+const MAX_ANYOF = 4;
	44	+
	45	+const isUnknown = (shape: InferredShape): boolean =>
	46	+ shape.type === undefined && shape.anyOf === undefined;
	47	+
	48	+/** Infer the shape of one observed value. Reads structure only, never values. */
	49	+export const inferShape = (value: unknown, depth = 0): InferredShape => {
	50	+ if (value === null \|\| value === undefined) return { type: "null" };
	51	+ if (typeof value === "boolean") return { type: "boolean" };
	52	+ if (typeof value === "number") return { type: "number" };
	53	+ if (typeof value === "string") return { type: "string" };
	54	+ if (depth >= MAX_DEPTH) return UNKNOWN;
	55	+
	56	+ if (Array.isArray(value)) {
	57	+ if (value.length === 0) return { type: "array" };
	58	+ const sampled = value
	59	+ .slice(0, MAX_ARRAY_SAMPLE)
	60	+ .map((item) => inferShape(item, depth + 1))
	61	+ .reduce((left, right) => mergeShapes(left, right));
	62	+ return { type: "array", items: sampled };
	63	+ }
	64	+
	65	+ if (typeof value === "object") {
	66	+ const entries = Object.entries(value as Record<string, unknown>);
	67	+ if (entries.length > MAX_OBJECT_KEYS) {
	68	+ const merged = entries
	69	+ .slice(0, MAX_ARRAY_SAMPLE)
	70	+ .map(([, item]) => inferShape(item, depth + 1))
	71	+ .reduce((left, right) => mergeShapes(left, right));
	72	+ return { type: "object", additionalProperties: merged };
	73	+ }
	74	+ const properties: Record<string, InferredShape> = {};
	75	+ for (const [key, item] of entries) {
	76	+ properties[key] = inferShape(item, depth + 1);
	77	+ }
	78	+ return { type: "object", properties, required: entries.map(([key]) => key).sort() };
	79	+ }
	80	+
	81	+ // function / symbol / bigint — nothing useful to say structurally.
	82	+ return UNKNOWN;
	83	+};
	84	+
	85	+const mergeObjectShapes = (left: InferredShape, right: InferredShape): InferredShape => {
	86	+ // A map-shaped observation absorbs struct-shaped ones: once keys look like
	87	+ // data, later struct keys are data too.
	88	+ if (left.additionalProperties !== undefined \|\| right.additionalProperties !== undefined) {
	89	+ const values = [
	90	+ left.additionalProperties,
	91	+ right.additionalProperties,
	92	+ ...Object.values(left.properties ?? {}),
	93	+ ...Object.values(right.properties ?? {}),
	94	+ ].filter((shape): shape is InferredShape => shape !== undefined);
	95	+ return {
	96	+ type: "object",
	97	+ additionalProperties:
	98	+ values.length === 0 ? UNKNOWN : values.reduce((a, b) => mergeShapes(a, b)),
	99	+ };
	100	+ }
	101	+
	102	+ const leftProps = left.properties ?? {};
	103	+ const rightProps = right.properties ?? {};
	104	+ const keys = [...new Set([...Object.keys(leftProps), ...Object.keys(rightProps)])].sort();
	105	+ if (keys.length > MAX_OBJECT_KEYS) {
	106	+ const values = keys
	107	+ .slice(0, MAX_ARRAY_SAMPLE)
	108	+ .map((key) => leftProps[key] ?? rightProps[key])
	109	+ .filter((shape): shape is InferredShape => shape !== undefined);
	110	+ return {
	111	+ type: "object",
	112	+ additionalProperties:
	113	+ values.length === 0 ? UNKNOWN : values.reduce((a, b) => mergeShapes(a, b)),
	114	+ };
	115	+ }
	116	+
	117	+ const properties: Record<string, InferredShape> = {};
	118	+ for (const key of keys) {
	119	+ const a = leftProps[key];
	120	+ const b = rightProps[key];
	121	+ properties[key] = a !== undefined && b !== undefined ? mergeShapes(a, b) : (a ?? b ?? UNKNOWN);
	122	+ }
	123	+ const leftRequired = new Set(left.required ?? []);
	124	+ const required = (right.required ?? []).filter((key) => leftRequired.has(key)).sort();
	125	+ return { type: "object", properties, required };
	126	+};
	127	+
	128	+/**
	129	+ * Merge two observed shapes into the narrowest shape matching both.
	130	+ * Same-typed shapes merge structurally; differently-typed shapes union into
	131	+ * `anyOf`, degrading to unknown past `MAX_ANYOF` branches.
	132	+ */
	133	+export const mergeShapes = (left: InferredShape, right: InferredShape): InferredShape => {
	134	+ if (isUnknown(left) \|\| isUnknown(right)) return UNKNOWN;
	135	+
	136	+ if (left.anyOf !== undefined \|\| right.anyOf !== undefined) {
	137	+ const branches = [...(left.anyOf ?? [left]), ...(right.anyOf ?? [right])];
	138	+ return branches.reduce((merged, branch) => addUnionBranch(merged, branch), {
	139	+ anyOf: [],
	140	+ } as InferredShape);
	141	+ }
	142	+
	143	+ if (left.type !== right.type) return addUnionBranch({ anyOf: [left] }, right);
	144	+
	145	+ if (left.type === "object") return mergeObjectShapes(left, right);
	146	+ if (left.type === "array") {
	147	+ if (left.items === undefined) return right;
	148	+ if (right.items === undefined) return left;
	149	+ return { type: "array", items: mergeShapes(left.items, right.items) };
	150	+ }
	151	+ return left;
	152	+};
	153	+
	154	+const addUnionBranch = (union: InferredShape, branch: InferredShape): InferredShape => {
	155	+ if (isUnknown(union) \|\| isUnknown(branch)) return UNKNOWN;
	156	+ const branches = [...(union.anyOf ?? [union])];
	157	+ const index = branches.findIndex((existing) => existing.type === branch.type);
	158	+ const next =
	159	+ index === -1
	160	+ ? [...branches, branch]
	161	+ : branches.map((existing, i) => (i === index ? mergeShapes(existing, branch) : existing));
	162	+ if (next.length > MAX_ANYOF) return UNKNOWN;
	163	+ return next.length === 1 ? (next[0] ?? UNKNOWN) : { anyOf: next };
	164	+};
	165	+
	166	+/**
	167	+ * One tool's accumulated muscle memory: the merged shape plus enough
	168	+ * bookkeeping to judge freshness. Stored as plain JSON.
	169	+ */
	170	+export type ObservedShape = {
	171	+ readonly schema: InferredShape;
	172	+ readonly observations: number;
	173	+ readonly updatedAt: number;
	174	+};
	175	+
	176	+/** Serialized-size ceiling per tool; a shape past this degrades to unknown
	177	+ * children rather than growing without bound. */
	178	+const MAX_SHAPE_JSON_CHARS = 16_000;
	179	+
	180	+const shrink = (shape: InferredShape, depth: number): InferredShape => {
	181	+ if (depth <= 0) return UNKNOWN;
	182	+ if (shape.anyOf) return { anyOf: shape.anyOf.map((branch) => shrink(branch, depth - 1)) };
	183	+ if (shape.type === "array" && shape.items) {
	184	+ return { type: "array", items: shrink(shape.items, depth - 1) };
	185	+ }
	186	+ if (shape.type === "object" && shape.additionalProperties) {
	187	+ return { type: "object", additionalProperties: shrink(shape.additionalProperties, depth - 1) };
	188	+ }
	189	+ if (shape.type === "object" && shape.properties) {
	190	+ const properties: Record<string, InferredShape> = {};
	191	+ for (const [key, child] of Object.entries(shape.properties)) {
	192	+ properties[key] = shrink(child, depth - 1);
	193	+ }
	194	+ return { ...shape, properties };
	195	+ }
	196	+ return shape;
	197	+};
	198	+
	199	+/** Fold a new observation into an existing record, keeping the result bounded. */
	200	+export const observeShape = (
	201	+ previous: ObservedShape \| null,
	202	+ value: unknown,
	203	+ now: number,
	204	+): ObservedShape => {
	205	+ const observed = inferShape(value);
	206	+ let schema = previous === null ? observed : mergeShapes(previous.schema, observed);
	207	+ for (
	208	+ let depth = MAX_DEPTH;
	209	+ depth > 0 && JSON.stringify(schema).length > MAX_SHAPE_JSON_CHARS;
	210	+ depth--
	211	+ ) {
	212	+ schema = shrink(schema, depth);
	213	+ }
	214	+ return {
	215	+ schema,
	216	+ observations: (previous?.observations ?? 0) + 1,
	217	+ updatedAt: now,
	218	+ };
	219	+};

packages/core/sdk/src/shape-memory.ts+90−0

@@ -0,0 +1,90 @@
	1	+/**
	2	+ * Muscle memory for tool outputs — the persistence half of runtime shape
	3	+ * inference (see shape-inference.ts for the algorithm).
	4	+ *
	5	+ * Observed shapes live in the already-migrated `plugin_storage` table under a
	6	+ * reserved system plugin id: owner-scoped, tenant-partitioned, and untouched
	7	+ * by tool-catalog refresh (which deletes and recreates `tool` rows, so the
	8	+ * tool row itself is not a viable home). An in-memory read-through cache
	9	+ * keeps the hot path off the database: within one executor instance a tool's
	10	+ * shape is loaded at most once, and a write happens only when a new
	11	+ * observation actually changes the merged shape — after a few calls a stable
	12	+ * API stops producing writes entirely.
	13	+ *
	14	+ * `observe` never fails and is intended to be forked off the dispatch path;
	15	+ * `recall` degrades to "no memory" on any storage failure.
	16	+ */
	17	+
	18	+import { Clock, Effect } from "effect";
	19	+
	20	+import type { Owner } from "./ids";
	21	+import type { PluginStorageFacade } from "./plugin-storage";
	22	+import { observeShape, type ObservedShape } from "./shape-inference";
	23	+
	24	+/** Reserved system namespace inside `plugin_storage`; not a real plugin. */
	25	+export const SHAPE_MEMORY_PLUGIN_ID = "executor.shape-memory";
	26	+const COLLECTION = "observed-output-shapes";
	27	+
	28	+export type ShapeMemory = {
	29	+ /**
	30	+ * Fold one successful tool payload into the tool's remembered shape.
	31	+ * Structure only — values never leave this call. Never fails.
	32	+ */
	33	+ readonly observe: (address: string, owner: Owner, value: unknown) => Effect.Effect<void>;
	34	+ /** The remembered shape for an address, or null when nothing is known. */
	35	+ readonly recall: (address: string, owner: Owner) => Effect.Effect<ObservedShape \| null>;
	36	+};
	37	+
	38	+export const makeShapeMemory = (storage: PluginStorageFacade): ShapeMemory => {
	39	+ const cache = new Map<string, ObservedShape \| null>();
	40	+ const persisted = new Map<string, string>();
	41	+
	42	+ const cacheKey = (owner: Owner, address: string) => `${owner}:${address}`;
	43	+
	44	+ const load = (address: string, owner: Owner): Effect.Effect<ObservedShape \| null> =>
	45	+ Effect.gen(function* () {
	46	+ const key = cacheKey(owner, address);
	47	+ const hit = cache.get(key);
	48	+ if (hit !== undefined) return hit;
	49	+ const entry = yield* storage
	50	+ .getForOwner<ObservedShape>({ owner, collection: COLLECTION, key: address })
	51	+ .pipe(Effect.catch(() => Effect.succeed(null)));
	52	+ const record = entry?.data ?? null;
	53	+ cache.set(key, record);
	54	+ if (record !== null) persisted.set(key, JSON.stringify(record.schema));
	55	+ return record;
	56	+ });
	57	+
	58	+ const observe = (address: string, owner: Owner, value: unknown): Effect.Effect<void> =>
	59	+ Effect.gen(function* () {
	60	+ const key = cacheKey(owner, address);
	61	+ const prior = yield* load(address, owner);
	62	+ const now = yield* Clock.currentTimeMillis;
	63	+ const next = observeShape(prior, value, now);
	64	+ cache.set(key, next);
	65	+ // Write only when the merged shape actually changed — observation
	66	+ // counters alone are bookkeeping, not worth a row write per call.
	67	+ const schemaJson = JSON.stringify(next.schema);
	68	+ if (persisted.get(key) === schemaJson) return;
	69	+ yield* storage
	70	+ .put({ owner, collection: COLLECTION, key: address, data: next })
	71	+ .pipe(Effect.catch(() => Effect.succeed(null)));
	72	+ persisted.set(key, schemaJson);
	73	+ }).pipe(Effect.catchCause(() => Effect.void));
	74	+
	75	+ return {
	76	+ observe,
	77	+ recall: (address, owner) =>
	78	+ load(address, owner).pipe(Effect.catchCause(() => Effect.succeed(null))),
	79	+ };
	80	+};
	81	+
	82	+/**
	83	+ * Render a remembered shape as the JSON Schema served in place of a missing
	84	+ * declared output schema. The description marks provenance so a reader (and
	85	+ * the schema view) can tell an observed shape from an author-declared one.
	86	+ */
	87	+export const observedShapeToJsonSchema = (record: ObservedShape): unknown => ({
	88	+ ...record.schema,
	89	+ description: `Observed from ${record.observations} live response${record.observations === 1 ? "" : "s"}; fields may be incomplete.`,
	90	+});

packages/core/sdk/src/types.ts+4−0

@@ -21,6 +21,10 @@ export const ToolSchemaView = Schema.Struct({ · 20 unmodified lines
21	21	description: Schema.optional(Schema.String),
22	22	inputSchema: Schema.optional(Schema.Unknown),
23	23	outputSchema: Schema.optional(Schema.Unknown),
	24	+ // "observed" = runtime-inferred from live responses (muscle memory), served
	25	+ // because the plugin declared no output schema. Absent = declared as-is.
	26	+ outputSchemaSource: Schema.optional(Schema.Literals(["observed"])),
	27	+ outputSchemaObservations: Schema.optional(Schema.Number),
24	28	schemaDefinitions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
25	29	inputTypeScript: Schema.optional(Schema.String),
26	30	outputTypeScript: Schema.optional(Schema.String),