History

refactor: split catalog snapshots into fragment-based architecture

Source adapters now produce CatalogFragmentV1 instead of full CatalogSnapshotV1 objects. Fragments are assembled into snapshots at the sync boundary via createCatalogSnapshotV1FromFragments, keeping import metadata separate from catalog data. This prepares the IR pipeline for multi-fragment catalog composition.

Rhys Sullivan committed 8 months ago

Commit 8019c0b1 parent 7e913c013 files · +264 −115Browse files

Unified · Split

13 files
  1. packages/control-plane/src/ir/catalog.ts +9 −0
  2. packages/control-plane/src/runtime/local-source-artifacts.test.ts +16 −6
  3. packages/control-plane/src/runtime/local-source-artifacts.ts +13 −9
  4. packages/control-plane/src/runtime/source-adapters/google-discovery.test.ts +7 −4
  5. packages/control-plane/src/runtime/source-adapters/google-discovery.ts +10 −4
  6. packages/control-plane/src/runtime/source-adapters/graphql.ts +12 −6
  7. packages/control-plane/src/runtime/source-adapters/internal.ts +14 −21
  8. packages/control-plane/src/runtime/source-adapters/mcp.test.ts +11 −8
  9. packages/control-plane/src/runtime/source-adapters/mcp.ts +10 −4
  10. packages/control-plane/src/runtime/source-adapters/openapi.ts +10 −4
  11. packages/control-plane/src/runtime/source-catalog-snapshot.ts +127 −45
  12. packages/control-plane/src/runtime/source-catalog-support.ts +20 −2
  13. packages/control-plane/src/runtime/source-catalog-sync.ts +5 −2
packages/control-plane/src/ir/catalog.ts+9−0
@@ -1650,6 +1650,15 @@ export const createCatalogSnapshotV1 = (input: { · 1649 unmodified lines
16501650 };
16511651 };
16521652
1653+export const createCatalogSnapshotV1FromFragments = (input: {
1654+ import: ImportMetadata;
1655+ fragments: readonly CatalogFragmentV1[];
1656+}) =>
1657+ createCatalogSnapshotV1({
1658+ import: input.import,
1659+ catalog: mergeCatalogFragments(input.fragments),
1660+ });
1661+
16531662 export const mergeCatalogFragments = (fragments: readonly CatalogFragmentV1[]): CatalogV1 => {
16541663 const catalog = emptyCatalog();
16551664 const seenByCollection = new Map<string, string>();
packages/control-plane/src/runtime/local-source-artifacts.test.ts+16−6
@@ -20,8 +20,9 @@ import { · 19 unmodified lines
2020 } from "./local-source-artifacts";
2121 import type { ResolvedLocalWorkspaceContext } from "./local-config";
2222 import {
23− createGraphqlCatalogSnapshot,
24− createOpenApiCatalogSnapshot,
23+ createCatalogImportMetadata,
24+ createGraphqlCatalogFragment,
25+ createOpenApiCatalogFragment,
2526 } from "./source-catalog-snapshot";
2627
2728 const makeContext = (): ResolvedLocalWorkspaceContext => {
@@ -33,6 +34,7 @@ const makeContext = (): ResolvedLocalWorkspaceContext => { · 5 unmodified lines
3334 configDirectory: join(workspaceRoot, ".executor"),
3435 projectConfigPath: join(workspaceRoot, ".executor", "executor.jsonc"),
3536 homeConfigPath: join(workspaceRoot, ".executor-home.jsonc"),
37+ homeStateDirectory: join(workspaceRoot, ".executor-home-state"),
3638 artifactsDirectory: join(workspaceRoot, ".executor", "artifacts"),
3739 stateDirectory: join(workspaceRoot, ".executor", "state"),
3840 };
@@ -63,7 +65,7 @@ const makeSource = (): Source => ({ · 24 unmodified lines
6365
6466 const makeArtifact = () => {
6567 const source = makeSource();
66− const snapshot = createOpenApiCatalogSnapshot({
68+ const fragment = createOpenApiCatalogFragment({
6769 source,
6870 documents: [{
6971 documentKind: "openapi",
@@ -77,7 +79,11 @@ const makeArtifact = () => { · 7 unmodified lines
7779 return buildLocalSourceArtifact({
7880 source,
7981 syncResult: {
80− snapshot,
82+ fragment,
83+ importMetadata: createCatalogImportMetadata({
84+ source,
85+ adapterKey: "openapi",
86+ }),
8187 sourceHash: source.sourceHash,
8288 },
8389 });
@@ -92,7 +98,7 @@ const makeGraphqlArtifact = () => { · 8 unmodified lines
9298 defaultHeaders: null,
9399 },
94100 };
95− const snapshot = createGraphqlCatalogSnapshot({
101+ const fragment = createGraphqlCatalogFragment({
96102 source,
97103 documents: [{
98104 documentKind: "graphql_introspection",
@@ -128,7 +134,11 @@ const makeGraphqlArtifact = () => { · 29 unmodified lines
128134 return buildLocalSourceArtifact({
129135 source,
130136 syncResult: {
131− snapshot,
137+ fragment,
138+ importMetadata: createCatalogImportMetadata({
139+ source,
140+ adapterKey: "graphql",
141+ }),
132142 sourceHash: source.sourceHash,
133143 },
134144 });
packages/control-plane/src/runtime/local-source-artifacts.ts+13−9
@@ -16,10 +16,15 @@ import * as Schema from "effect/Schema"; · 15 unmodified lines
1616 import { decodeCatalogSnapshotV1 } from "../ir/catalog";
1717 import {
1818 CatalogSnapshotV1Schema,
19+ type CatalogSnapshotV1,
1920 type NativeBlob,
2021 type SourceDocument,
2122 } from "../ir/model";
22−import type { SourceCatalogSyncResult } from "./source-catalog-support";
23+import {
24+ contentHash,
25+ snapshotFromSourceCatalogSyncResult,
26+ type SourceCatalogSyncResult,
27+} from "./source-catalog-support";
2328 import type { ResolvedLocalWorkspaceContext } from "./local-config";
2429 import {
2530 LocalFileSystemError,
@@ -30,8 +35,6 @@ import { · 4 unmodified lines
3035 createSourceCatalogRevisionRecord,
3136 stableSourceCatalogId,
3237 } from "./source-definitions";
33−import { contentHash } from "./source-catalog-support";
34−
3538 const LOCAL_SOURCE_ARTIFACT_VERSION = 3 as const;
3639
3740 export const LocalSourceArtifactSchema = Schema.Struct({
@@ -155,10 +158,10 @@ const hydrateArtifactSourceDocuments = (input: { · 117 unmodified lines
155158 return nextArtifact;
156159 };
157160
158−const snapshotHash = (snapshot: SourceCatalogSyncResult["snapshot"]): string =>
161+const snapshotHash = (snapshot: CatalogSnapshotV1): string =>
159162 contentHash(JSON.stringify(snapshot));
160163
161−const importMetadataHash = (snapshot: SourceCatalogSyncResult["snapshot"]): string =>
164+const importMetadataHash = (snapshot: { import: SourceCatalogSyncResult["importMetadata"] }): string =>
162165 contentHash(JSON.stringify(snapshot.import));
163166
164167 export const buildLocalSourceArtifact = (input: {
@@ -167,13 +170,14 @@ export const buildLocalSourceArtifact = (input: { · 2 unmodified lines
167170 }): LocalSourceArtifact => {
168171 const catalogId: SourceCatalogId = stableSourceCatalogId(input.source);
169172 const generatedAt = Date.now();
170− const importHash = importMetadataHash(input.syncResult.snapshot);
171− const hash = snapshotHash(input.syncResult.snapshot);
173+ const snapshot = snapshotFromSourceCatalogSyncResult(input.syncResult);
174+ const importHash = importMetadataHash(snapshot);
175+ const hash = snapshotHash(snapshot);
172176 const revision = createSourceCatalogRevisionRecord({
173177 source: input.source,
174178 catalogId,
175179 revisionNumber: 1,
176− importMetadataJson: JSON.stringify(input.syncResult.snapshot.import),
180+ importMetadataJson: JSON.stringify(snapshot.import),
177181 importMetadataHash: importHash,
178182 snapshotHash: hash,
179183 });
@@ -184,7 +188,7 @@ export const buildLocalSourceArtifact = (input: { · 4 unmodified lines
184188 catalogId,
185189 generatedAt,
186190 revision,
187− snapshot: input.syncResult.snapshot,
191+ snapshot,
188192 };
189193 };
190194
packages/control-plane/src/runtime/source-adapters/google-discovery.test.ts+7−4
@@ -6,6 +6,7 @@ import { SourceIdSchema } from "#schema"; · 5 unmodified lines
66 import * as Effect from "effect/Effect";
77
88 import { googleDiscoverySourceAdapter } from "./google-discovery";
9+import { snapshotFromSourceCatalogSyncResult } from "../source-catalog-support";
910 import { createSourceFromPayload } from "../source-definitions";
1011
1112 const fetchLiveDiscoveryDocument = async (): Promise<string> => {
@@ -102,12 +103,14 @@ describe("google discovery source adapter", () => { · 90 unmodified lines
102103 }),
103104 }),
104105 );
106+ const snapshot = snapshotFromSourceCatalogSyncResult(syncResult);
105107
106− expect(Object.values(syncResult.snapshot.catalog.documents)[0]?.kind).toBe("google-discovery");
107− expect(Object.keys(syncResult.snapshot.catalog.resources)).not.toHaveLength(0);
108− expect(Object.keys(syncResult.snapshot.catalog.capabilities).length).toBeGreaterThan(50);
108+ expect(syncResult.fragment.version).toBe("ir.v1.fragment");
109+ expect(Object.values(snapshot.catalog.documents)[0]?.kind).toBe("google-discovery");
110+ expect(Object.keys(snapshot.catalog.resources)).not.toHaveLength(0);
111+ expect(Object.keys(snapshot.catalog.capabilities).length).toBeGreaterThan(50);
109112 expect(
110− Object.values(syncResult.snapshot.catalog.capabilities).some((capability) =>
113+ Object.values(snapshot.catalog.capabilities).some((capability) =>
111114 capability.surface.toolPath.join(".") === "google.sheets.spreadsheets.values.get"
112115 ),
113116 ).toBe(true);
packages/control-plane/src/runtime/source-adapters/google-discovery.ts+10−4
@@ -15,9 +15,11 @@ import * as Effect from "effect/Effect"; · 14 unmodified lines
1515 import * as Schema from "effect/Schema";
1616
1717 import {
18− createGoogleDiscoveryCatalogSnapshot,
18+ createCatalogImportMetadata,
19+ createGoogleDiscoveryCatalogFragment,
1920 type GoogleDiscoveryCatalogOperationInput,
2021 } from "../source-catalog-snapshot";
22+import { createSourceCatalogSyncResult } from "../source-catalog-support";
2123 import type { SourceAdapter } from "./types";
2224 import {
2325 ConnectHttpAuthSchema,
@@ -336,8 +338,8 @@ export const googleDiscoverySourceAdapter: SourceAdapter = { · 312 unmodified lines
336338 const definitions = compileGoogleDiscoveryToolDefinitions(manifest);
337339 const now = Date.now();
338340
339− return {
340− snapshot: createGoogleDiscoveryCatalogSnapshot({
341+ return createSourceCatalogSyncResult({
342+ fragment: createGoogleDiscoveryCatalogFragment({
341343 source,
342344 documents: [{
343345 documentKind: "google_discovery",
@@ -352,8 +354,12 @@ export const googleDiscoverySourceAdapter: SourceAdapter = { · 8 unmodified lines
352354 })
353355 ),
354356 }),
357+ importMetadata: createCatalogImportMetadata({
358+ source,
359+ adapterKey: "google_discovery",
360+ }),
355361 sourceHash: manifest.sourceHash,
356− };
362+ });
357363 }),
358364 getOauth2SetupConfig: ({ source }) => googleDiscoveryOauth2SetupConfig(source),
359365 normalizeOauthClientInput: (input) =>
packages/control-plane/src/runtime/source-adapters/graphql.ts+12−6
@@ -16,9 +16,11 @@ import * as Effect from "effect/Effect"; · 15 unmodified lines
1616 import * as Schema from "effect/Schema";
1717
1818 import {
19− createGraphqlCatalogSnapshot,
19+ createCatalogImportMetadata,
20+ createGraphqlCatalogFragment,
2021 type GraphqlCatalogOperationInput,
2122 } from "../source-catalog-snapshot";
23+import { createSourceCatalogSyncResult } from "../source-catalog-support";
2224 import type { SourceAdapter } from "./types";
2325 import {
2426 ConnectHttpAuthSchema,
@@ -327,8 +329,8 @@ export const graphqlSourceAdapter: SourceAdapter = { · 302 unmodified lines
327329 }),
328330 );
329331 const now = Date.now();
330− const snapshot = yield* Effect.sync(() =>
331− createGraphqlCatalogSnapshot({
332+ const fragment = yield* Effect.sync(() =>
333+ createGraphqlCatalogFragment({
332334 source,
333335 documents: [{
334336 documentKind: "graphql_introspection",
@@ -347,10 +349,14 @@ export const graphqlSourceAdapter: SourceAdapter = { · 12 unmodified lines
347349 }),
348350 );
349351
350− return {
351− snapshot,
352+ return createSourceCatalogSyncResult({
353+ fragment,
354+ importMetadata: createCatalogImportMetadata({
355+ source,
356+ adapterKey: "graphql",
357+ }),
352358 sourceHash: manifest.sourceHash,
353− };
359+ });
354360 }).pipe(
355361 Effect.withSpan("graphql.syncCatalog", {
356362 attributes: {
packages/control-plane/src/runtime/source-adapters/internal.ts+14−21
@@ -2,6 +2,8 @@ import * as Effect from "effect/Effect"; · 1 unmodified line
22 import * as Schema from "effect/Schema";
33 import type { Source } from "#schema";
44
5+import { createCatalogImportMetadata } from "../source-catalog-snapshot";
6+import { createSourceCatalogSyncResult } from "../source-catalog-support";
57 import type { SourceAdapter } from "./types";
68 import {
79 decodeBindingConfig,
@@ -99,28 +101,19 @@ export const internalSourceAdapter: SourceAdapter = { · 91 unmodified lines
99101 };
100102 }),
101103 shouldAutoProbe: () => false,
102− syncCatalog: () => Effect.succeed({
103− snapshot: {
104− version: "ir.v1.snapshot",
105− import: {
106− sourceKind: "custom",
107− adapterKey: "internal",
104+ syncCatalog: ({ source }) =>
105+ Effect.succeed(createSourceCatalogSyncResult({
106+ fragment: {
107+ version: "ir.v1.fragment",
108+ },
109+ importMetadata: {
110+ ...createCatalogImportMetadata({
111+ source,
112+ adapterKey: "internal",
113+ }),
108114 importerVersion: "ir.v1.internal",
109− importedAt: new Date().toISOString(),
110115 sourceConfigHash: "internal",
111116 },
112− catalog: {
113− version: "ir.v1",
114− documents: {},
115− resources: {},
116− scopes: {},
117− symbols: {},
118− capabilities: {},
119− executables: {},
120− responseSets: {},
121− diagnostics: {},
122− },
123− },
124− sourceHash: null,
125− }),
117+ sourceHash: null,
118+ })),
126119 };
packages/control-plane/src/runtime/source-adapters/mcp.test.ts+11−8
@@ -26,6 +26,7 @@ import { · 25 unmodified lines
2626 expandCatalogToolByPath,
2727 type LoadedSourceCatalog,
2828 } from "../source-catalog-runtime";
29+import { snapshotFromSourceCatalogSyncResult } from "../source-catalog-support";
2930 import { createSourceFromPayload } from "../source-definitions";
3031 import { mcpSourceAdapter } from "./mcp";
3132
@@ -252,13 +253,14 @@ describe("mcp source adapter", () => { · 220 unmodified lines
252253 cookies: {},
253254 bodyValues: {},
254255 expiresAt: null,
255− refreshAfter: null,
256− }),
256+ refreshAfter: null,
257+ }),
257258 });
259+ const snapshot = snapshotFromSourceCatalogSyncResult(syncResult);
258260
259− const capability = Object.values(syncResult.snapshot.catalog.capabilities)[0]!;
260− const executable = Object.values(syncResult.snapshot.catalog.executables)[0]!;
261− const document = Object.values(syncResult.snapshot.catalog.documents)[0]!;
261+ const capability = Object.values(snapshot.catalog.capabilities)[0]!;
262+ const executable = Object.values(snapshot.catalog.executables)[0]!;
263+ const document = Object.values(snapshot.catalog.documents)[0]!;
262264 const rawManifest = document.native?.[0]?.value;
263265
264266 expect(capability.surface.title).toBe("Read File");
@@ -339,14 +341,15 @@ describe("mcp source adapter", () => { · 74 unmodified lines
339341 cookies: {},
340342 bodyValues: {},
341343 expiresAt: null,
342− refreshAfter: null,
343− }),
344+ refreshAfter: null,
345+ }),
344346 });
347+ const snapshot = snapshotFromSourceCatalogSyncResult(syncResult);
345348
346349 const tool = yield* expandCatalogToolByPath({
347350 catalogs: [makeLoadedCatalog({
348351 source,
349− snapshot: syncResult.snapshot,
352+ snapshot,
350353 })],
351354 path: "mcp.demo.read_file",
352355 });
packages/control-plane/src/runtime/source-adapters/mcp.ts+10−4
@@ -19,11 +19,13 @@ import * as Effect from "effect/Effect"; · 18 unmodified lines
1919 import * as Schema from "effect/Schema";
2020
2121 import {
22− createMcpCatalogSnapshot,
22+ createCatalogImportMetadata,
23+ createMcpCatalogFragment,
2324 type McpCatalogOperationInput,
2425 } from "../source-catalog-snapshot";
2526 import {
2627 contentHash,
28+ createSourceCatalogSyncResult,
2729 type SourceCatalogSyncResult,
2830 } from "../source-catalog-support";
2931 import { namespaceFromSourceName } from "../source-names";
@@ -160,8 +162,8 @@ export const catalogSyncResultFromMcpManifest = (input: { · 130 unmodified lines
160162 const manifestJson = JSON.stringify(input.manifest);
161163 const manifestHash = contentHash(manifestJson);
162164
163− return {
164− snapshot: createMcpCatalogSnapshot({
165+ return createSourceCatalogSyncResult({
166+ fragment: createMcpCatalogFragment({
165167 source: input.source,
166168 documents: [{
167169 documentKind: "mcp_manifest",
@@ -176,8 +178,12 @@ export const catalogSyncResultFromMcpManifest = (input: { · 8 unmodified lines
176178 })
177179 ),
178180 }),
181+ importMetadata: createCatalogImportMetadata({
182+ source: input.source,
183+ adapterKey: "mcp",
184+ }),
179185 sourceHash: manifestHash,
180− };
186+ });
181187 };
182188
183189 export const mcpSourceAdapter: SourceAdapter = {
packages/control-plane/src/runtime/source-adapters/openapi.ts+10−4
@@ -20,9 +20,11 @@ import * as Effect from "effect/Effect"; · 19 unmodified lines
2020 import * as Schema from "effect/Schema";
2121
2222 import {
23− createOpenApiCatalogSnapshot,
23+ createCatalogImportMetadata,
24+ createOpenApiCatalogFragment,
2425 type OpenApiCatalogOperationInput,
2526 } from "../source-catalog-snapshot";
27+import { createSourceCatalogSyncResult } from "../source-catalog-support";
2628 import type { SourceAdapter } from "./types";
2729 import {
2830 ConnectHttpAuthSchema,
@@ -299,8 +301,8 @@ export const openApiSourceAdapter: SourceAdapter = { · 270 unmodified lines
299301 const definitions = compileOpenApiToolDefinitions(manifest);
300302 const now = Date.now();
301303
302− return {
303− snapshot: createOpenApiCatalogSnapshot({
304+ return createSourceCatalogSyncResult({
305+ fragment: createOpenApiCatalogFragment({
304306 source,
305307 documents: [{
306308 documentKind: "openapi",
@@ -314,7 +316,11 @@ export const openApiSourceAdapter: SourceAdapter = { · 7 unmodified lines
314316 refHintTable: manifest.refHintTable,
315317 })),
316318 }),
319+ importMetadata: createCatalogImportMetadata({
320+ source,
321+ adapterKey: "openapi",
322+ }),
317323 sourceHash: manifest.sourceHash,
318− };
324+ });
319325 }),
320326 };
packages/control-plane/src/runtime/source-catalog-snapshot.ts+127−45
@@ -14,8 +14,7 @@ import type { · 13 unmodified lines
1414 import type { Source } from "#schema";
1515
1616 import {
17− createCatalogSnapshotV1,
18− createEmptyCatalogV1,
17+ createCatalogSnapshotV1FromFragments,
1918 } from "../ir/catalog";
2019 import {
2120 CapabilityIdSchema,
@@ -36,8 +35,8 @@ import { · 14 unmodified lines
3635 import type {
3736 AuthRequirement,
3837 Capability,
38+ CatalogFragmentV1,
3939 CatalogSnapshotV1,
40− CatalogV1,
4140 ContentSpec,
4241 DocumentationBlock,
4342 EffectKind,
@@ -63,6 +62,18 @@ import { namespaceFromSourceName } from "./source-names"; · 19 unmodified lines
6362
6463 type JsonSchema = boolean | Record<string, unknown>;
6564
65+type CatalogFragmentBuilder = {
66+ version: "ir.v1.fragment";
67+ documents: NonNullable<CatalogFragmentV1["documents"]>;
68+ resources: NonNullable<CatalogFragmentV1["resources"]>;
69+ scopes: NonNullable<CatalogFragmentV1["scopes"]>;
70+ symbols: NonNullable<CatalogFragmentV1["symbols"]>;
71+ capabilities: NonNullable<CatalogFragmentV1["capabilities"]>;
72+ executables: NonNullable<CatalogFragmentV1["executables"]>;
73+ responseSets: NonNullable<CatalogFragmentV1["responseSets"]>;
74+ diagnostics: NonNullable<CatalogFragmentV1["diagnostics"]>;
75+};
76+
6677 export type CatalogSourceDocumentInput = {
6778 documentKind: string;
6879 documentKey: string;
@@ -262,8 +273,37 @@ const importMetadataFor = (input: { · 193 unmodified lines
262273 }),
263274 });
264275
276+export const createCatalogImportMetadata = (input: {
277+ source: Source;
278+ adapterKey: string;
279+}) => importMetadataFor(input);
280+
281+const createEmptyCatalogFragment = (): CatalogFragmentBuilder => ({
282+ version: "ir.v1.fragment",
283+ documents: {},
284+ resources: {},
285+ scopes: {},
286+ symbols: {},
287+ capabilities: {},
288+ executables: {},
289+ responseSets: {},
290+ diagnostics: {},
291+});
292+
293+const finalizeCatalogFragment = (fragment: CatalogFragmentBuilder): CatalogFragmentV1 => ({
294+ version: "ir.v1.fragment",
295+ ...(Object.keys(fragment.documents).length > 0 ? { documents: fragment.documents } : {}),
296+ ...(Object.keys(fragment.resources).length > 0 ? { resources: fragment.resources } : {}),
297+ ...(Object.keys(fragment.scopes).length > 0 ? { scopes: fragment.scopes } : {}),
298+ ...(Object.keys(fragment.symbols).length > 0 ? { symbols: fragment.symbols } : {}),
299+ ...(Object.keys(fragment.capabilities).length > 0 ? { capabilities: fragment.capabilities } : {}),
300+ ...(Object.keys(fragment.executables).length > 0 ? { executables: fragment.executables } : {}),
301+ ...(Object.keys(fragment.responseSets).length > 0 ? { responseSets: fragment.responseSets } : {}),
302+ ...(Object.keys(fragment.diagnostics).length > 0 ? { diagnostics: fragment.diagnostics } : {}),
303+});
304+
265305 const addDiagnostic = (
266− catalog: CatalogV1,
306+ catalog: CatalogFragmentBuilder,
267307 input: Omit<ImportDiagnostic, "id">,
268308 ) => {
269309 const id = DiagnosticIdSchema.make(`diag_${stableHash(input)}`);
@@ -310,7 +350,7 @@ const docsFrom = (input: { · 40 unmodified lines
310350 };
311351
312352 const createJsonSchemaImporter = (input: {
313− catalog: CatalogV1;
353+ catalog: CatalogFragmentBuilder;
314354 source: Source;
315355 resourceId: ReturnType<typeof ResourceIdSchema.make>;
316356 documentId: ReturnType<typeof DocumentIdSchema.make>;
@@ -800,7 +840,7 @@ const createJsonSchemaImporter = (input: { · 483 unmodified lines
800840 };
801841
802842 const exampleSymbolFromValue = (input: {
803− catalog: CatalogV1;
843+ catalog: CatalogFragmentBuilder;
804844 source: Source;
805845 documentId: ReturnType<typeof DocumentIdSchema.make>;
806846 pointer: string;
@@ -886,7 +926,7 @@ const preferredResponseContentTypes = ( · 79 unmodified lines
886926 };
887927
888928 const createServiceScope = (input: {
889− catalog: CatalogV1;
929+ catalog: CatalogFragmentBuilder;
890930 source: Source;
891931 documentId: ReturnType<typeof DocumentIdSchema.make>;
892932 defaults?: Scope["defaults"];
@@ -936,7 +976,7 @@ const googleDiscoveryServerSpecs = ( · 43 unmodified lines
936976 };
937977
938978 const createOperationScope = (input: {
939− catalog: CatalogV1;
979+ catalog: CatalogFragmentBuilder;
940980 source: Source;
941981 documentId: ReturnType<typeof DocumentIdSchema.make>;
942982 parentScopeId: ReturnType<typeof ScopeIdSchema.make>;
@@ -967,7 +1007,7 @@ const createOperationScope = (input: { · 24 unmodified lines
9671007 };
9681008
9691009 const responseSetFromSingleResponse = (input: {
970− catalog: CatalogV1;
1010+ catalog: CatalogFragmentBuilder;
9711011 responseId: ReturnType<typeof ResponseSymbolIdSchema.make>;
9721012 provenance: ProvenanceRef[];
9731013 traits?: ResponseSet["variants"][number]["traits"];
@@ -993,7 +1033,7 @@ const responseSetFromSingleResponse = (input: { · 19 unmodified lines
9931033 };
9941034
9951035 const responseSetFromVariants = (input: {
996− catalog: CatalogV1;
1036+ catalog: CatalogFragmentBuilder;
9971037 variants: ResponseSet["variants"];
9981038 provenance: ProvenanceRef[];
9991039 }) => {
@@ -1038,7 +1078,7 @@ const statusMatchFromHttpStatusCode = ( · 38 unmodified lines
10381078 };
10391079
10401080 const ensureOpenApiSecuritySchemeSymbol = (input: {
1041− catalog: CatalogV1;
1081+ catalog: CatalogFragmentBuilder;
10421082 source: Source;
10431083 documentId: ReturnType<typeof DocumentIdSchema.make>;
10441084 schemeName: string;
@@ -1147,7 +1187,7 @@ const ensureOpenApiSecuritySchemeSymbol = (input: { · 102 unmodified lines
11471187 };
11481188
11491189 const openApiAuthRequirementToIr = (input: {
1150− catalog: CatalogV1;
1190+ catalog: CatalogFragmentBuilder;
11511191 source: Source;
11521192 documentId: ReturnType<typeof DocumentIdSchema.make>;
11531193 authRequirement: OpenApiToolProviderData["authRequirement"] | undefined;
@@ -1199,7 +1239,7 @@ const openApiAuthRequirementToIr = (input: { · 45 unmodified lines
11991239 };
12001240
12011241 const contentSpecsFromOpenApiContents = (input: {
1202− catalog: CatalogV1;
1242+ catalog: CatalogFragmentBuilder;
12031243 source: Source;
12041244 documentId: ReturnType<typeof DocumentIdSchema.make>;
12051245 importer: JsonSchemaImporter;
@@ -1239,7 +1279,7 @@ const contentSpecsFromOpenApiContents = (input: { · 33 unmodified lines
12391279 });
12401280
12411281 const createOpenApiHeaderSymbol = (input: {
1242− catalog: CatalogV1;
1282+ catalog: CatalogFragmentBuilder;
12431283 source: Source;
12441284 documentId: ReturnType<typeof DocumentIdSchema.make>;
12451285 importer: JsonSchemaImporter;
@@ -1307,7 +1347,7 @@ const createOpenApiHeaderSymbol = (input: { · 61 unmodified lines
13071347 };
13081348
13091349 const createHttpCapabilityFromOpenApi = (input: {
1310− catalog: CatalogV1;
1350+ catalog: CatalogFragmentBuilder;
13111351 source: Source;
13121352 documentId: ReturnType<typeof DocumentIdSchema.make>;
13131353 serviceScopeId: ReturnType<typeof ScopeIdSchema.make>;
@@ -1714,7 +1754,7 @@ const createHttpCapabilityFromOpenApi = (input: { · 400 unmodified lines
17141754 };
17151755
17161756 const createHttpCapabilityFromGoogleDiscovery = (input: {
1717− catalog: CatalogV1;
1757+ catalog: CatalogFragmentBuilder;
17181758 source: Source;
17191759 documentId: ReturnType<typeof DocumentIdSchema.make>;
17201760 serviceScopeId: ReturnType<typeof ScopeIdSchema.make>;
@@ -1927,7 +1967,7 @@ const createHttpCapabilityFromGoogleDiscovery = (input: { · 206 unmodified lines
19271967 };
19281968
19291969 const createGraphqlCapability = (input: {
1930− catalog: CatalogV1;
1970+ catalog: CatalogFragmentBuilder;
19311971 source: Source;
19321972 documentId: ReturnType<typeof DocumentIdSchema.make>;
19331973 serviceScopeId: ReturnType<typeof ScopeIdSchema.make>;
@@ -2047,7 +2087,7 @@ const createGraphqlCapability = (input: { · 113 unmodified lines
20472087 };
20482088
20492089 const createMcpCapability = (input: {
2050− catalog: CatalogV1;
2090+ catalog: CatalogFragmentBuilder;
20512091 source: Source;
20522092 documentId: ReturnType<typeof DocumentIdSchema.make>;
20532093 serviceScopeId: ReturnType<typeof ScopeIdSchema.make>;
@@ -2147,19 +2187,18 @@ const createMcpCapability = (input: { · 93 unmodified lines
21472187 } satisfies Capability;
21482188 };
21492189
2150−const buildCatalogSnapshot = (input: {
2190+const buildCatalogFragment = (input: {
21512191 source: Source;
2152− adapterKey: string;
21532192 documents: readonly CatalogSourceDocumentInput[];
21542193 serviceScopeDefaults?: Scope["defaults"];
21552194 registerOperations: (context: {
2156− catalog: CatalogV1;
2195+ catalog: CatalogFragmentBuilder;
21572196 documentId: ReturnType<typeof DocumentIdSchema.make>;
21582197 serviceScopeId: ReturnType<typeof ScopeIdSchema.make>;
21592198 importer: JsonSchemaImporter;
21602199 }) => void;
2161−}): CatalogSnapshotV1 => {
2162− const catalog = createEmptyCatalogV1();
2200+}): CatalogFragmentV1 => {
2201+ const catalog = createEmptyCatalogFragment();
21632202 const documents = input.documents.length > 0
21642203 ? input.documents
21652204 : [{
@@ -2237,19 +2276,14 @@ const buildCatalogSnapshot = (input: { · 71 unmodified lines
22372276 });
22382277 importer.finalize();
22392278
2240− const snapshot = createCatalogSnapshotV1({
2241− import: importMetadataFor(input),
2242− catalog,
2243− });
2244−
2245− return snapshot;
2279+ return finalizeCatalogFragment(catalog);
22462280 };
22472281
2248−export const createOpenApiCatalogSnapshot = (input: {
2282+export const createOpenApiCatalogFragment = (input: {
22492283 source: Source;
22502284 documents: readonly CatalogSourceDocumentInput[];
22512285 operations: readonly OpenApiCatalogOperationInput[];
2252−}): CatalogSnapshotV1 => {
2286+}): CatalogFragmentV1 => {
22532287 const rootSchema = (() => {
22542288 const primaryDocumentText = input.documents[0]?.contentText;
22552289 if (!primaryDocumentText) {
@@ -2263,9 +2297,8 @@ export const createOpenApiCatalogSnapshot = (input: { · 7 unmodified lines
22632297 }
22642298 })();
22652299
2266− return buildCatalogSnapshot({
2300+ return buildCatalogFragment({
22672301 source: input.source,
2268− adapterKey: "openapi",
22692302 documents: input.documents,
22702303 serviceScopeDefaults: (() => {
22712304 const documentServers = openApiServerSpecs(
@@ -2292,14 +2325,26 @@ export const createOpenApiCatalogSnapshot = (input: { · 20 unmodified lines
22922325 });
22932326 };
22942327
2295−export const createGoogleDiscoveryCatalogSnapshot = (input: {
2328+export const createOpenApiCatalogSnapshot = (input: {
22962329 source: Source;
22972330 documents: readonly CatalogSourceDocumentInput[];
2298− operations: readonly GoogleDiscoveryCatalogOperationInput[];
2331+ operations: readonly OpenApiCatalogOperationInput[];
22992332 }): CatalogSnapshotV1 =>
2300− buildCatalogSnapshot({
2333+ createCatalogSnapshotV1FromFragments({
2334+ import: createCatalogImportMetadata({
2335+ source: input.source,
2336+ adapterKey: "openapi",
2337+ }),
2338+ fragments: [createOpenApiCatalogFragment(input)],
2339+ });
2340+
2341+export const createGoogleDiscoveryCatalogFragment = (input: {
2342+ source: Source;
2343+ documents: readonly CatalogSourceDocumentInput[];
2344+ operations: readonly GoogleDiscoveryCatalogOperationInput[];
2345+}): CatalogFragmentV1 =>
2346+ buildCatalogFragment({
23012347 source: input.source,
2302− adapterKey: "google_discovery",
23032348 documents: input.documents,
23042349 serviceScopeDefaults: (() => {
23052350 const servers = googleDiscoveryServerSpecs(input.operations[0]);
@@ -2319,14 +2364,26 @@ export const createGoogleDiscoveryCatalogSnapshot = (input: { · 13 unmodified lines
23192364 },
23202365 });
23212366
2322−export const createGraphqlCatalogSnapshot = (input: {
2367+export const createGoogleDiscoveryCatalogSnapshot = (input: {
23232368 source: Source;
23242369 documents: readonly CatalogSourceDocumentInput[];
2325− operations: readonly GraphqlCatalogOperationInput[];
2370+ operations: readonly GoogleDiscoveryCatalogOperationInput[];
23262371 }): CatalogSnapshotV1 =>
2327− buildCatalogSnapshot({
2372+ createCatalogSnapshotV1FromFragments({
2373+ import: createCatalogImportMetadata({
2374+ source: input.source,
2375+ adapterKey: "google_discovery",
2376+ }),
2377+ fragments: [createGoogleDiscoveryCatalogFragment(input)],
2378+ });
2379+
2380+export const createGraphqlCatalogFragment = (input: {
2381+ source: Source;
2382+ documents: readonly CatalogSourceDocumentInput[];
2383+ operations: readonly GraphqlCatalogOperationInput[];
2384+}): CatalogFragmentV1 =>
2385+ buildCatalogFragment({
23282386 source: input.source,
2329− adapterKey: "graphql",
23302387 documents: input.documents,
23312388 registerOperations: ({ catalog, documentId, serviceScopeId, importer }) => {
23322389 for (const operation of input.operations) {
@@ -2342,14 +2399,26 @@ export const createGraphqlCatalogSnapshot = (input: { · 9 unmodified lines
23422399 },
23432400 });
23442401
2345−export const createMcpCatalogSnapshot = (input: {
2402+export const createGraphqlCatalogSnapshot = (input: {
23462403 source: Source;
23472404 documents: readonly CatalogSourceDocumentInput[];
2348− operations: readonly McpCatalogOperationInput[];
2405+ operations: readonly GraphqlCatalogOperationInput[];
23492406 }): CatalogSnapshotV1 =>
2350− buildCatalogSnapshot({
2407+ createCatalogSnapshotV1FromFragments({
2408+ import: createCatalogImportMetadata({
2409+ source: input.source,
2410+ adapterKey: "graphql",
2411+ }),
2412+ fragments: [createGraphqlCatalogFragment(input)],
2413+ });
2414+
2415+export const createMcpCatalogFragment = (input: {
2416+ source: Source;
2417+ documents: readonly CatalogSourceDocumentInput[];
2418+ operations: readonly McpCatalogOperationInput[];
2419+}): CatalogFragmentV1 =>
2420+ buildCatalogFragment({
23512421 source: input.source,
2352− adapterKey: "mcp",
23532422 documents: input.documents,
23542423 registerOperations: ({ catalog, documentId, serviceScopeId, importer }) => {
23552424 for (const operation of input.operations) {
@@ -2364,3 +2433,16 @@ export const createMcpCatalogSnapshot = (input: { · 8 unmodified lines
23642433 }
23652434 },
23662435 });
2436+
2437+export const createMcpCatalogSnapshot = (input: {
2438+ source: Source;
2439+ documents: readonly CatalogSourceDocumentInput[];
2440+ operations: readonly McpCatalogOperationInput[];
2441+}): CatalogSnapshotV1 =>
2442+ createCatalogSnapshotV1FromFragments({
2443+ import: createCatalogImportMetadata({
2444+ source: input.source,
2445+ adapterKey: "mcp",
2446+ }),
2447+ fragments: [createMcpCatalogFragment(input)],
2448+ });
packages/control-plane/src/runtime/source-catalog-support.ts+20−2
@@ -1,5 +1,10 @@
11 import { sha256Hex } from "@executor/codemode-core";
2−import type { CatalogSnapshotV1 } from "../ir/model";
2+import { createCatalogSnapshotV1FromFragments } from "../ir/catalog";
3+import type {
4+ CatalogFragmentV1,
5+ CatalogSnapshotV1,
6+ ImportMetadata,
7+} from "../ir/model";
38
49 export const normalizeSearchText = (
510 ...parts: ReadonlyArray<string | null | undefined>
@@ -13,6 +18,19 @@ export const normalizeSearchText = ( · 7 unmodified lines
1318 export const contentHash = (value: string): string => sha256Hex(value);
1419
1520 export type SourceCatalogSyncResult = {
16− snapshot: CatalogSnapshotV1;
21+ fragment: CatalogFragmentV1;
22+ importMetadata: ImportMetadata;
1723 sourceHash: string | null;
1824 };
25+
26+export const createSourceCatalogSyncResult = (
27+ input: SourceCatalogSyncResult,
28+): SourceCatalogSyncResult => input;
29+
30+export const snapshotFromSourceCatalogSyncResult = (
31+ syncResult: SourceCatalogSyncResult,
32+): CatalogSnapshotV1 =>
33+ createCatalogSnapshotV1FromFragments({
34+ import: syncResult.importMetadata,
35+ fragments: [syncResult.fragment],
36+ });
packages/control-plane/src/runtime/source-catalog-sync.ts+5−2
@@ -31,6 +31,7 @@ import { · 30 unmodified lines
3131 catalogSyncResultFromMcpManifest,
3232 } from "./source-adapters/mcp";
3333 import { SecretMaterialResolverService } from "./secret-material-providers";
34+import { snapshotFromSourceCatalogSyncResult } from "./source-catalog-support";
3435 import {
3536 refreshSourceTypeDeclarationInBackground,
3637 } from "./source-type-declarations";
@@ -139,6 +140,7 @@ const syncSourceCatalogWithDeps = ( · 102 unmodified lines
139140 actorAccountId: input.actorAccountId,
140141 }),
141142 });
143+ const snapshot = snapshotFromSourceCatalogSyncResult(syncResult);
142144 yield* deps.sourceArtifactStore.write({
143145 context: workspaceContext,
144146 sourceId: input.source.id,
@@ -172,7 +174,7 @@ const syncSourceCatalogWithDeps = ( · 27 unmodified lines
172174 refreshSourceTypeDeclarationInBackground({
173175 context: workspaceContext,
174176 source: input.source,
175− snapshot: syncResult.snapshot,
177+ snapshot,
176178 });
177179 });
178180 }).pipe(
@@ -203,6 +205,7 @@ const persistMcpCatalogSnapshotFromManifestWithDeps = ( · 24 unmodified lines
203205 endpoint: input.source.endpoint,
204206 manifest: input.manifest,
205207 });
208+ const snapshot = snapshotFromSourceCatalogSyncResult(syncResult);
206209
207210 yield* deps.sourceArtifactStore.write({
208211 context: workspaceContext,
@@ -217,7 +220,7 @@ const persistMcpCatalogSnapshotFromManifestWithDeps = ( · 8 unmodified lines
217220 refreshSourceTypeDeclarationInBackground({
218221 context: workspaceContext,
219222 source: input.source,
220− snapshot: syncResult.snapshot,
223+ snapshot,
221224 });
222225 });
223226 });