History

Remove schema shared catalog path

Rhys Sullivan committed 7 months ago

Commit 44ce0321 parent 2c814623 files · +150 −48Browse files

Unified · Split

3 files
  1. .changeset/nine-islands-bake.md +5 −0
  2. packages/platform/sdk/src/runtime/execution/scope/source-catalog.test.ts +115 −1
  3. packages/platform/sdk/src/runtime/execution/scope/source-catalog.ts +30 −47
.changeset/nine-islands-bake.md+5−0
@@ -0,0 +1,5 @@
1---
2"executor": patch
3---
4
5Avoid ever building a full schema workspace catalog during tool description, and resolve schemaful tool details through direct one-tool lookup instead.
packages/platform/sdk/src/runtime/execution/scope/source-catalog.test.ts+115−1
@@ -125,7 +125,7 @@ describe("scope source catalog", () => { · 124 unmodified lines
125 ]);125 ]);
126 }));126 }));
127 127
128 it.effect("reuses a lean shared catalog for repeated no-schema search and lookup calls", () =>128 it.effect("reuses the lean shared catalog for repeated search, list, and no-schema lookup calls", () =>
129 Effect.gen(function* () {129 Effect.gen(function* () {
130 const loadWorkspaceSourceCatalogToolIndexCalls: Array<undefined> = [];130 const loadWorkspaceSourceCatalogToolIndexCalls: Array<undefined> = [];
131 131
@@ -195,6 +195,11 @@ describe("scope source catalog", () => { · 63 unmodified lines
195 limit: 20,195 limit: 20,
196 includeSchemas: false,196 includeSchemas: false,
197 });197 });
198 const listedWithSchemas = yield* catalog.listTools({
199 query: "GitHub list repository issues",
200 limit: 20,
201 includeSchemas: true,
202 });
198 203
199 expect(hits).toHaveLength(1);204 expect(hits).toHaveLength(1);
200 expect(hits[0]?.path).toBe("github.issues.list");205 expect(hits[0]?.path).toBe("github.issues.list");
@@ -202,6 +207,115 @@ describe("scope source catalog", () => { · 1 unmodified line
202 expect(first?.contract?.inputTypePreview).toBeUndefined();207 expect(first?.contract?.inputTypePreview).toBeUndefined();
203 expect(second?.contract?.outputTypePreview).toBeUndefined();208 expect(second?.contract?.outputTypePreview).toBeUndefined();
204 expect(listed).toHaveLength(1);209 expect(listed).toHaveLength(1);
210 expect(listedWithSchemas).toHaveLength(1);
211 expect(listedWithSchemas[0]?.contract?.inputSchema).toBeUndefined();
205 expect(loadWorkspaceSourceCatalogToolIndexCalls).toHaveLength(1);212 expect(loadWorkspaceSourceCatalogToolIndexCalls).toHaveLength(1);
206 }));213 }));
214
215 it.effect("loads schemaful tool descriptions through direct tool lookup instead of a shared schema catalog", () =>
216 Effect.gen(function* () {
217 const loadWorkspaceSourceCatalogToolIndexCalls: Array<undefined> = [];
218 const loadWorkspaceSourceCatalogToolByPathCalls: Array<{
219 path: string;
220 includeSchemas: boolean;
221 }> = [];
222
223 const catalog = createScopeSourceCatalog({
224 scopeId: "ws_test" as any,
225 actorScopeId: "acc_test" as any,
226 sourceCatalogStore: {
227 loadWorkspaceSourceCatalogs: () => Effect.die("unexpected catalog load"),
228 loadWorkspaceSourceCatalogToolIndex: () => {
229 loadWorkspaceSourceCatalogToolIndexCalls.push(undefined);
230
231 return Effect.succeed([{
232 path: "github.issues.list",
233 searchNamespace: "github.issues",
234 searchText: "github issues list repository issues",
235 source: {
236 enabled: true,
237 status: "connected",
238 },
239 sourceRecord: {},
240 capabilityId: "cap_1",
241 executableId: "exec_1",
242 capability: {
243 surface: {
244 title: "List Repository Issues",
245 },
246 },
247 executable: {
248 display: {
249 leaf: "list",
250 },
251 },
252 descriptor: {
253 path: "github.issues.list",
254 sourceKey: "github",
255 description: "List issues for a repository",
256 interaction: "auto",
257 },
258 projectedCatalog: {},
259 }] as any);
260 },
261 loadWorkspaceSourceCatalogToolByPath: (input: {
262 path: string;
263 includeSchemas: boolean;
264 }) => {
265 loadWorkspaceSourceCatalogToolByPathCalls.push(input);
266
267 return Effect.succeed({
268 path: "github.issues.list",
269 searchNamespace: "github.issues",
270 searchText: "github issues list repository issues",
271 source: {
272 enabled: true,
273 status: "connected",
274 },
275 sourceRecord: {},
276 capabilityId: "cap_1",
277 executableId: "exec_1",
278 capability: {
279 surface: {
280 title: "List Repository Issues",
281 },
282 },
283 executable: {
284 display: {
285 leaf: "list",
286 },
287 },
288 descriptor: {
289 path: "github.issues.list",
290 sourceKey: "github",
291 description: "List issues for a repository",
292 interaction: "auto",
293 contract: {
294 inputSchema: { type: "object" },
295 },
296 },
297 projectedCatalog: {},
298 } as any);
299 },
300 } as any,
301 scopeConfigStore: noopScopeConfigStore,
302 scopeStateStore: noopScopeStateStore,
303 sourceArtifactStore: noopSourceArtifactStore,
304 runtimeLocalScope: null,
305 });
306
307 const described = yield* catalog.getToolByPath({
308 path: "github.issues.list" as any,
309 includeSchemas: true,
310 });
311
312 expect(described?.contract?.inputSchema).toEqual({ type: "object" });
313 expect(loadWorkspaceSourceCatalogToolIndexCalls).toHaveLength(0);
314 expect(loadWorkspaceSourceCatalogToolByPathCalls).toEqual([{
315 scopeId: "ws_test",
316 actorScopeId: "acc_test",
317 path: "github.issues.list",
318 includeSchemas: true,
319 }]);
320 }));
207});321});
packages/platform/sdk/src/runtime/execution/scope/source-catalog.ts+30−47
@@ -13,8 +13,6 @@ import { · 12 unmodified lines
13 RuntimeSourceCatalogStoreService,13 RuntimeSourceCatalogStoreService,
14 type LoadedSourceCatalogToolIndexEntry,14 type LoadedSourceCatalogToolIndexEntry,
15 catalogToolCatalogEntry,15 catalogToolCatalogEntry,
16 expandCatalogTools,
17 loadedSourceCatalogToolIndexEntryFromLoadedTool,
18} from "../../catalog/source/runtime";16} from "../../catalog/source/runtime";
19import type {17import type {
20 RuntimeLocalScopeState,18 RuntimeLocalScopeState,
@@ -130,40 +128,21 @@ export const loadWorkspaceCatalogTools = (input: { · 109 unmodified lines
130 scopeId: Source["scopeId"];128 scopeId: Source["scopeId"];
131 actorScopeId: ScopeId;129 actorScopeId: ScopeId;
132 sourceCatalogStore: Effect.Effect.Success<typeof RuntimeSourceCatalogStoreService>;130 sourceCatalogStore: Effect.Effect.Success<typeof RuntimeSourceCatalogStoreService>;
133 includeSchemas: boolean;
134}): Effect.Effect<131}): Effect.Effect<
135 readonly LoadedSourceCatalogToolIndexEntry[],132 readonly LoadedSourceCatalogToolIndexEntry[],
136 Error,133 Error,
137 ScopeStorageServices134 ScopeStorageServices
138> =>135> =>
139 input.includeSchemas136 Effect.map(
140 ? Effect.gen(function* () {137 input.sourceCatalogStore.loadWorkspaceSourceCatalogToolIndex({
141 const catalogs = yield* input.sourceCatalogStore.loadWorkspaceSourceCatalogs({138 scopeId: input.scopeId,
142 scopeId: input.scopeId,139 actorScopeId: input.actorScopeId,
143 actorScopeId: input.actorScopeId,140 }),
144 });141 (tools) =>
145 const tools = yield* expandCatalogTools({142 tools.filter(
146 catalogs,143 (tool) => tool.source.enabled && tool.source.status === "connected",
147 includeSchemas: true,144 ),
148 includeTypePreviews: true,145 );
149 });
150
151 return tools
152 .map(loadedSourceCatalogToolIndexEntryFromLoadedTool)
153 .filter(
154 (tool) => tool.source.enabled && tool.source.status === "connected",
155 );
156 })
157 : Effect.map(
158 input.sourceCatalogStore.loadWorkspaceSourceCatalogToolIndex({
159 scopeId: input.scopeId,
160 actorScopeId: input.actorScopeId,
161 }),
162 (tools) =>
163 tools.filter(
164 (tool) => tool.source.enabled && tool.source.status === "connected",
165 ),
166 );
167 146
168export const loadWorkspaceCatalogToolByPath = (input: {147export const loadWorkspaceCatalogToolByPath = (input: {
169 scopeId: Source["scopeId"];148 scopeId: Source["scopeId"];
@@ -332,13 +311,12 @@ export const createScopeSourceCatalog = (input: { · 162 unmodified lines
332 const provideWorkspaceStorage = <A, E, R>(effect: Effect.Effect<A, E, R>) =>311 const provideWorkspaceStorage = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
333 effect.pipe(Effect.provide(scopeStorageLayer));312 effect.pipe(Effect.provide(scopeStorageLayer));
334 313
335 const buildSharedCatalog = (includeSchemas: boolean) =>314 const buildSharedCatalog = () =>
336 provideWorkspaceStorage(Effect.gen(function* () {315 provideWorkspaceStorage(Effect.gen(function* () {
337 const catalogTools = yield* loadWorkspaceCatalogTools({316 const catalogTools = yield* loadWorkspaceCatalogTools({
338 scopeId: input.scopeId,317 scopeId: input.scopeId,
339 actorScopeId: input.actorScopeId,318 actorScopeId: input.actorScopeId,
340 sourceCatalogStore: input.sourceCatalogStore,319 sourceCatalogStore: input.sourceCatalogStore,
341 includeSchemas,
342 });320 });
343 321
344 return createToolCatalogFromEntries({322 return createToolCatalogFromEntries({
@@ -352,15 +330,9 @@ export const createScopeSourceCatalog = (input: { · 7 unmodified lines
352 }));330 }));
353 331
354 const leanSharedCatalog = Effect.runSync(332 const leanSharedCatalog = Effect.runSync(
355 Effect.cached(buildSharedCatalog(false)),333 Effect.cached(buildSharedCatalog()),
356 );
357 const schemaSharedCatalog = Effect.runSync(
358 Effect.cached(buildSharedCatalog(true)),
359 );334 );
360 335
361 const createSharedCatalog = (includeSchemas: boolean): Effect.Effect<ToolCatalog, Error, never> =>
362 includeSchemas ? schemaSharedCatalog : leanSharedCatalog;
363
364 return {336 return {
365 listNamespaces: ({ limit }) =>337 listNamespaces: ({ limit }) =>
366 provideRuntimeLocalScope(338 provideRuntimeLocalScope(
@@ -375,14 +347,14 @@ export const createScopeSourceCatalog = (input: { · 8 unmodified lines
375 input.runtimeLocalScope,347 input.runtimeLocalScope,
376 ),348 ),
377 349
378 listTools: ({ namespace, query, limit, includeSchemas = false }) =>350 listTools: ({ namespace, query, limit }) =>
379 provideRuntimeLocalScope(351 provideRuntimeLocalScope(
380 Effect.flatMap(createSharedCatalog(includeSchemas), (catalog) =>352 Effect.flatMap(leanSharedCatalog, (catalog) =>
381 catalog.listTools({353 catalog.listTools({
382 ...(namespace !== undefined ? { namespace } : {}),354 ...(namespace !== undefined ? { namespace } : {}),
383 ...(query !== undefined ? { query } : {}),355 ...(query !== undefined ? { query } : {}),
384 limit,356 limit,
385 includeSchemas,357 includeSchemas: false,
386 }),358 }),
387 ),359 ),
388 input.runtimeLocalScope,360 input.runtimeLocalScope,
@@ -390,15 +362,26 @@ export const createScopeSourceCatalog = (input: { · 1 unmodified line
390 362
391 getToolByPath: ({ path, includeSchemas }) =>363 getToolByPath: ({ path, includeSchemas }) =>
392 provideRuntimeLocalScope(364 provideRuntimeLocalScope(
393 Effect.flatMap(createSharedCatalog(includeSchemas), (catalog) =>365 includeSchemas
394 catalog.getToolByPath({ path, includeSchemas }),366 ? Effect.map(
395 ),367 provideWorkspaceStorage(loadWorkspaceCatalogToolByPath({
368 scopeId: input.scopeId,
369 actorScopeId: input.actorScopeId,
370 sourceCatalogStore: input.sourceCatalogStore,
371 path,
372 includeSchemas: true,
373 })),
374 (tool) => tool?.descriptor ?? null,
375 )
376 : Effect.flatMap(leanSharedCatalog, (catalog) =>
377 catalog.getToolByPath({ path, includeSchemas: false })
378 ),
396 input.runtimeLocalScope,379 input.runtimeLocalScope,
397 ),380 ),
398 381
399 searchTools: ({ query, namespace, limit }) =>382 searchTools: ({ query, namespace, limit }) =>
400 provideRuntimeLocalScope(383 provideRuntimeLocalScope(
401 Effect.flatMap(createSharedCatalog(false), (catalog) =>384 Effect.flatMap(leanSharedCatalog, (catalog) =>
402 catalog.searchTools({385 catalog.searchTools({
403 query,386 query,
404 ...(namespace !== undefined ? { namespace } : {}),387 ...(namespace !== undefined ? { namespace } : {}),