History

feat: add profile option to OpenAPI spec generation and update related components

- Introduced a new optional `profile` parameter in the OpenAPI spec generation route, allowing for "full" and "inventory" profiles. - Updated the `ToolExplorer` and `ToolsView` components to remove inventory state handling, simplifying the UI. - Adjusted caching mechanisms to accommodate the new profile parameter. - Enhanced tests to verify behavior with the new profile options.

Rhys Sullivan committed 8 months ago

Commit aafd6db1 parent dfd0a1f10 files · +308 −185Browse files

Unified · Split

10 files
  1. executor/apps/web/src/app/api/generate/route.ts +7 −2
  2. executor/apps/web/src/components/tools/explorer.tsx +0 −66
  3. executor/apps/web/src/components/tools/view.tsx +0 −2
  4. executor/apps/web/src/lib/openapi/spec-inspector.ts +1 −0
  5. executor/apps/web/src/lib/tool/schema-traverse.test.ts +49 −0
  6. executor/apps/web/src/lib/tool/schema-traverse.ts +24 −10
  7. executor/packages/core/src/openapi/real-specs.test.ts +43 −0
  8. executor/packages/database/convex/_generated/api.d.ts +6 −1
  9. executor/packages/database/convex/runtimeNode.ts +3 −2
  10. executor/packages/database/src/runtime/tool_source_loading.ts +175 −102
executor/apps/web/src/app/api/generate/route.ts+7−2
@@ -8,6 +8,7 @@ const querySchema = z.object({ · 7 unmodified lines
8 specUrl: z.string().trim().min(1),8 specUrl: z.string().trim().min(1),
9 sourceName: z.string().trim().min(1).optional(),9 sourceName: z.string().trim().min(1).optional(),
10 includeDts: z.enum(["1", "true", "0", "false"]).optional(),10 includeDts: z.enum(["1", "true", "0", "false"]).optional(),
11 profile: z.enum(["full", "inventory"]).optional(),
11 headers: z.string().optional(),12 headers: z.string().optional(),
12});13});
13 14
@@ -109,6 +110,7 @@ async function prepareWithForwardedHeaders( · 95 unmodified lines
109 specUrl: string,110 specUrl: string,
110 sourceName: string,111 sourceName: string,
111 includeDts: boolean,112 includeDts: boolean,
113 profile: "full" | "inventory",
112 headers: Record<string, string>,114 headers: Record<string, string>,
113): Promise<Awaited<ReturnType<typeof prepareOpenApiSpec>>> {115): Promise<Awaited<ReturnType<typeof prepareOpenApiSpec>>> {
114 const controller = new AbortController();116 const controller = new AbortController();
@@ -133,7 +135,7 @@ async function prepareWithForwardedHeaders( · 18 unmodified lines
133 const parsedSpec = parseSpecPayload(raw);135 const parsedSpec = parseSpecPayload(raw);
134 return await prepareOpenApiSpec(parsedSpec, sourceName, {136 return await prepareOpenApiSpec(parsedSpec, sourceName, {
135 includeDts,137 includeDts,
136 profile: "inventory",138 profile,
137 });139 });
138 } finally {140 } finally {
139 clearTimeout(timeoutId);141 clearTimeout(timeoutId);
@@ -146,6 +148,7 @@ export async function GET(request: Request): Promise<Response> { · 6 unmodified lines
146 specUrl: url.searchParams.get("specUrl") ?? "",148 specUrl: url.searchParams.get("specUrl") ?? "",
147 sourceName: url.searchParams.get("sourceName") ?? undefined,149 sourceName: url.searchParams.get("sourceName") ?? undefined,
148 includeDts: url.searchParams.get("includeDts") ?? undefined,150 includeDts: url.searchParams.get("includeDts") ?? undefined,
151 profile: url.searchParams.get("profile") ?? undefined,
149 headers: url.searchParams.get("headers") ?? undefined,152 headers: url.searchParams.get("headers") ?? undefined,
150 });153 });
151 154
@@ -156,17 +159,19 @@ export async function GET(request: Request): Promise<Response> { · 4 unmodified lines
156 try {159 try {
157 const includeDts = parsed.data.includeDts === "1" || parsed.data.includeDts === "true";160 const includeDts = parsed.data.includeDts === "1" || parsed.data.includeDts === "true";
158 const sourceName = parsed.data.sourceName ?? "openapi";161 const sourceName = parsed.data.sourceName ?? "openapi";
162 const profile = parsed.data.profile ?? "full";
159 const forwardedHeaders = sanitizeForwardHeaders(parseHeadersQueryValue(parsed.data.headers));163 const forwardedHeaders = sanitizeForwardHeaders(parseHeadersQueryValue(parsed.data.headers));
160 164
161 const prepared = Object.keys(forwardedHeaders).length === 0165 const prepared = Object.keys(forwardedHeaders).length === 0
162 ? await prepareOpenApiSpec(parsed.data.specUrl, sourceName, {166 ? await prepareOpenApiSpec(parsed.data.specUrl, sourceName, {
163 includeDts,167 includeDts,
164 profile: "inventory",168 profile,
165 })169 })
166 : await prepareWithForwardedHeaders(170 : await prepareWithForwardedHeaders(
167 parsed.data.specUrl,171 parsed.data.specUrl,
168 sourceName,172 sourceName,
169 includeDts,173 includeDts,
174 profile,
170 forwardedHeaders,175 forwardedHeaders,
171 );176 );
172 177
executor/apps/web/src/components/tools/explorer.tsx+0−66
@@ -74,8 +74,6 @@ interface ToolExplorerProps { · 73 unmodified lines
74 onSourceDeleted?: (sourceName: string) => void;74 onSourceDeleted?: (sourceName: string) => void;
75 onRegenerate?: () => void;75 onRegenerate?: () => void;
76 isRebuilding?: boolean;76 isRebuilding?: boolean;
77 inventoryState?: "initializing" | "ready" | "rebuilding" | "failed";
78 inventoryError?: string;
79}77}
80 78
81export function ToolExplorer({79export function ToolExplorer({
@@ -107,8 +105,6 @@ export function ToolExplorer({ · 25 unmodified lines
107 onSourceDeleted,105 onSourceDeleted,
108 onRegenerate,106 onRegenerate,
109 isRebuilding = false,107 isRebuilding = false,
110 inventoryState,
111 inventoryError,
112}: ToolExplorerProps) {108}: ToolExplorerProps) {
113 const [toolDetailsByPath, setToolDetailsByPath] = useState<Record<string, Pick<ToolDescriptor, "path" | "description" | "display" | "typing">>>({});109 const [toolDetailsByPath, setToolDetailsByPath] = useState<Record<string, Pick<ToolDescriptor, "path" | "description" | "display" | "typing">>>({});
114 const [loadingDetailPaths, setLoadingDetailPaths] = useState<Set<string>>(new Set());110 const [loadingDetailPaths, setLoadingDetailPaths] = useState<Set<string>>(new Set());
@@ -426,50 +422,6 @@ export function ToolExplorer({ · 311 unmodified lines
426 });422 });
427 }, []);423 }, []);
428 424
429 const inventoryStatus = useMemo(() => {
430 const loadingSourceCount = loadingSourceSet.size;
431 const sourceStateEntries = Object.entries(sourceStates);
432 const totalSourceCount = sourceStateEntries.length;
433 const readySourceCount = sourceStateEntries.filter(([, state]) => state.state === "ready").length;
434 const failedSourceCount = sourceStateEntries.filter(([, state]) => state.state === "failed").length;
435
436 if (!inventoryState) {
437 return { label: "Checking...", tone: "muted" as const };
438 }
439 if (inventoryState === "initializing") {
440 const progress = totalSourceCount > 0
441 ? `${readySourceCount}/${totalSourceCount}`
442 : `${loadingSourceCount}`;
443 return {
444 label: `Building ${progress}`,
445 tone: "loading" as const,
446 };
447 }
448 if (inventoryState === "rebuilding") {
449 const progress = totalSourceCount > 0
450 ? `${readySourceCount}/${totalSourceCount}`
451 : `${loadingSourceCount}`;
452 return {
453 label: `Refreshing ${progress}`,
454 tone: "loading" as const,
455 };
456 }
457 if (inventoryState === "failed") {
458 return {
459 label: inventoryError
460 ? `Failed: ${inventoryError}`
461 : failedSourceCount > 0
462 ? `Failed (${failedSourceCount} sources)`
463 : "Failed",
464 tone: "error" as const,
465 };
466 }
467 return {
468 label: totalSourceCount > 0 ? `Up to date (${readySourceCount}/${totalSourceCount})` : "Up to date",
469 tone: "muted" as const,
470 };
471 }, [inventoryError, inventoryState, loadingSourceSet.size, sourceStates]);
472
473 const regenerationInProgress = isRebuilding;425 const regenerationInProgress = isRebuilding;
474 // ── Render ──────────────────────────────────────────────────────────────426 // ── Render ──────────────────────────────────────────────────────────────
475 427
@@ -484,24 +436,6 @@ export function ToolExplorer({ · 8 unmodified lines
484 <p className="text-[10px] font-mono uppercase tracking-widest text-muted-foreground/50">436 <p className="text-[10px] font-mono uppercase tracking-widest text-muted-foreground/50">
485 Tools437 Tools
486 </p>438 </p>
487 <span
488 className={cn(
489 "text-[9px] font-mono flex items-center gap-0.5",
490 inventoryStatus.tone === "loading"
491 ? "text-terminal-amber/80"
492 : inventoryStatus.tone === "error"
493 ? "text-terminal-red/80"
494 : "text-muted-foreground/40",
495 )}
496 title={inventoryStatus.label}
497 >
498 {inventoryStatus.tone === "loading" ? (
499 <Loader2 className="h-2.5 w-2.5 animate-spin" />
500 ) : inventoryStatus.tone === "error" ? (
501 <AlertTriangle className="h-2.5 w-2.5" />
502 ) : null}
503 {inventoryStatus.label}
504 </span>
505 </div>439 </div>
506 <div className="flex items-center gap-0.5">440 <div className="flex items-center gap-0.5">
507 {onRegenerate ? (441 {onRegenerate ? (
executor/apps/web/src/components/tools/view.tsx+0−2
@@ -489,8 +489,6 @@ export function ToolsView() { · 488 unmodified lines
489 onFocusedToolPathChange={setCatalogActiveToolPath}489 onFocusedToolPathChange={setCatalogActiveToolPath}
490 onFocusedSourceNameChange={setCatalogFocusedSourceName}490 onFocusedSourceNameChange={setCatalogFocusedSourceName}
491 onRegenerate={handleRegenerateInventory}491 onRegenerate={handleRegenerateInventory}
492 inventoryState={inventoryStatus?.state}
493 inventoryError={inventoryStatus?.error}
494 isRebuilding={492 isRebuilding={
495 regenerationInFlight493 regenerationInFlight
496 || inventoryStatus?.state === "rebuilding"494 || inventoryStatus?.state === "rebuilding"
executor/apps/web/src/lib/openapi/spec-inspector.ts+1−0
@@ -218,6 +218,7 @@ export async function fetchAndInspectOpenApiSpec(input: { · 217 unmodified lines
218 specUrl: input.specUrl,218 specUrl: input.specUrl,
219 sourceName: "openapi-inspect",219 sourceName: "openapi-inspect",
220 includeDts: "0",220 includeDts: "0",
221 profile: "inventory",
221 });222 });
222 223
223 if (hasHeaders) {224 if (hasHeaders) {
executor/apps/web/src/lib/tool/schema-traverse.test.ts+49−0
@@ -0,0 +1,49 @@
1import { describe, expect, test } from "bun:test";
2import { traverseSchema } from "./schema-traverse";
3
4describe("traverseSchema type labels", () => {
5 test("infers object when schema has properties but no explicit type", () => {
6 const schema = {
7 type: "object",
8 properties: {
9 body: {
10 properties: {
11 name: { type: "string" },
12 },
13 required: ["name"],
14 },
15 },
16 } satisfies Record<string, unknown>;
17
18 const { entries } = traverseSchema(schema);
19 const body = entries.find((entry) => entry.path === "body");
20
21 expect(body).toBeDefined();
22 expect(body?.type).toBe("object");
23 expect(body?.typeLabel).toBe("object");
24 });
25
26 test("dedupes oneOf object variants in array item labels", () => {
27 const schema = {
28 type: "object",
29 properties: {
30 include: {
31 type: "array",
32 items: {
33 oneOf: [
34 { type: "object", properties: { email: { type: "string" } } },
35 { type: "object", properties: { ip: { type: "string" } } },
36 ],
37 },
38 },
39 },
40 } satisfies Record<string, unknown>;
41
42 const { entries } = traverseSchema(schema);
43 const include = entries.find((entry) => entry.path === "include");
44
45 expect(include).toBeDefined();
46 expect(include?.type).toBe("array");
47 expect(include?.typeLabel).toBe("object[]");
48 });
49});
executor/apps/web/src/lib/tool/schema-traverse.ts+24−10
@@ -113,21 +113,30 @@ function resolveTypeLabel(schema: Record<string, unknown>): { · 112 unmodified lines
113 // oneOf / anyOf union113 // oneOf / anyOf union
114 const union = (schema.oneOf ?? schema.anyOf) as unknown[] | undefined;114 const union = (schema.oneOf ?? schema.anyOf) as unknown[] | undefined;
115 if (Array.isArray(union) && union.length > 0) {115 if (Array.isArray(union) && union.length > 0) {
116 const parts = union.map((s) => {116 const parts = union
117 const sub = asRecord(s);117 .map((s) => resolveTypeLabel(asRecord(s)))
118 return resolveTypeLabel(sub).label;118 .filter((part) => part.label.trim().length > 0);
119 });119 const uniqueLabels = [...new Set(parts.map((part) => part.label))];
120 return { type: "union", label: parts.join(" | ") };120 if (uniqueLabels.length === 1) {
121 const single = parts[0];
122 if (single) {
123 return { type: single.type, label: single.label, itemType: single.itemType };
124 }
125 }
126 return { type: "union", label: uniqueLabels.join(" | ") };
121 }127 }
122 128
123 // allOf intersection129 // allOf intersection
124 const allOf = schema.allOf as unknown[] | undefined;130 const allOf = schema.allOf as unknown[] | undefined;
125 if (Array.isArray(allOf) && allOf.length > 0) {131 if (Array.isArray(allOf) && allOf.length > 0) {
126 const parts = allOf.map((s) => {132 const parts = allOf
127 const sub = asRecord(s);133 .map((s) => resolveTypeLabel(asRecord(s)).label)
128 return resolveTypeLabel(sub).label;134 .filter((label) => label.trim().length > 0);
129 });135 const uniqueLabels = [...new Set(parts)];
130 return { type: "object", label: parts.join(" & ") };136 if (uniqueLabels.length === 1) {
137 return { type: "object", label: uniqueLabels[0]! };
138 }
139 return { type: "object", label: uniqueLabels.join(" & ") };
131 }140 }
132 141
133 // Type array (e.g. ["string", "null"])142 // Type array (e.g. ["string", "null"])
@@ -157,6 +166,11 @@ function resolveTypeLabel(schema: Record<string, unknown>): { · 23 unmodified lines
157 return { type: "object", label: "object" };166 return { type: "object", label: "object" };
158 }167 }
159 168
169 const hasProperties = Object.keys(asRecord(schema.properties)).length > 0;
170 if (hasProperties || schema.additionalProperties !== undefined || Array.isArray(schema.required)) {
171 return { type: "object", label: "object" };
172 }
173
160 // String with format174 // String with format
161 if (rawType === "string" && typeof schema.format === "string") {175 if (rawType === "string" && typeof schema.format === "string") {
162 return { type: "string", label: `string<${schema.format}>` };176 return { type: "string", label: `string<${schema.format}>` };
executor/packages/core/src/openapi/real-specs.test.ts+43−0
@@ -761,6 +761,49 @@ describe("real-world OpenAPI specs", () => { · 760 unmodified lines
761 300_000,761 300_000,
762 );762 );
763 763
764 test(
765 "cloudflare: access group create keeps concrete body schema in full profile",
766 async () => {
767 const cloudflareUrl = "https://raw.githubusercontent.com/cloudflare/api-schemas/main/openapi.yaml";
768
769 const prepared = await prepareOpenApiSpec(cloudflareUrl, "cloudflare", {
770 includeDts: false,
771 profile: "full",
772 });
773 const tools = buildOpenApiToolsFromPrepared(
774 {
775 type: "openapi",
776 name: "cloudflare",
777 spec: cloudflareUrl,
778 baseUrl: prepared.servers[0] || "https://api.cloudflare.com/client/v4",
779 },
780 prepared,
781 );
782
783 const tool = tools.find((t) => t.path === "cloudflare.access_groups.create_an_access_group");
784
785 expect(tool).toBeDefined();
786 const inputSchema = toRecord(tool!.typing?.inputSchema);
787 const inputProperties = toRecord(inputSchema.properties);
788 const bodySchema = toRecord(inputProperties.body);
789 expect(Object.keys(bodySchema).length).toBeGreaterThan(0);
790
791 const bodyProperties = toRecord(bodySchema.properties);
792 const includeSchema = toRecord(bodyProperties.include);
793 const excludeSchema = toRecord(bodyProperties.exclude);
794 const requireSchema = toRecord(bodyProperties.require);
795
796 expect(includeSchema.type).toBe("array");
797 expect(excludeSchema.type).toBe("array");
798 expect(requireSchema.type).toBe("array");
799
800 expect(Object.keys(toRecord(includeSchema.items)).length).toBeGreaterThan(0);
801 expect(Object.keys(toRecord(excludeSchema.items)).length).toBeGreaterThan(0);
802 expect(Object.keys(toRecord(requireSchema.items)).length).toBeGreaterThan(0);
803 },
804 300_000,
805 );
806
764 test(807 test(
765 "github + stripe: root generated array schemas include concrete item definitions",808 "github + stripe: root generated array schemas include concrete item definitions",
766 async () => {809 async () => {
executor/packages/database/convex/_generated/api.d.ts+6−1
@@ -1742,7 +1742,12 @@ export declare const internal: { · 1741 unmodified lines
1742 prepareOpenApiSpec: FunctionReference<1742 prepareOpenApiSpec: FunctionReference<
1743 "action",1743 "action",
1744 "internal",1744 "internal",
1745 { includeDts?: boolean; sourceName: string; specUrl: string },1745 {
1746 includeDts?: boolean;
1747 profile?: "full" | "inventory";
1748 sourceName: string;
1749 specUrl: string;
1750 },
1746 any1751 any
1747 >;1752 >;
1748 };1753 };
executor/packages/database/convex/runtimeNode.ts+3−2
@@ -39,12 +39,13 @@ export const prepareOpenApiSpec = internalAction({ · 38 unmodified lines
39 specUrl: v.string(),39 specUrl: v.string(),
40 sourceName: v.string(),40 sourceName: v.string(),
41 includeDts: v.optional(v.boolean()),41 includeDts: v.optional(v.boolean()),
42 profile: v.optional(v.union(v.literal("full"), v.literal("inventory"))),
42 },43 },
43 handler: async (ctx, args): Promise<string> => {44 handler: async (ctx, args): Promise<string> => {
44 const prepared = await prepareOpenApiSpecInNode(args.specUrl, args.sourceName, {45 const prepared = await prepareOpenApiSpecInNode(args.specUrl, args.sourceName, {
45 includeDts: args.includeDts,46 includeDts: args.includeDts,
46 profile: "full",47 profile: args.profile ?? "full",
47 resolveSchemaRefs: true,48 resolveSchemaRefs: args.profile === "inventory" ? false : true,
48 });49 });
49 50
50 const json = JSON.stringify(prepared);51 const json = JSON.stringify(prepared);
executor/packages/database/src/runtime/tool_source_loading.ts+175−102
@@ -32,9 +32,12 @@ const OPENAPI_PREPARE_MAX_ATTEMPTS = 3; · 31 unmodified lines
32const OPENAPI_PREPARE_RETRY_BASE_DELAY_MS = 1_500;32const OPENAPI_PREPARE_RETRY_BASE_DELAY_MS = 1_500;
33const OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS = 180_000;33const OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS = 180_000;
34 34
35const OPENAPI_PREPARED_CACHE_VERSION = "openapi_v11";35const OPENAPI_PREPARED_CACHE_VERSION = "openapi_v12";
36const OPENAPI_ARTIFACT_CACHE_VERSION = "openapi_artifact_v1";36const OPENAPI_ARTIFACT_CACHE_VERSION = "openapi_artifact_v2";
37const OPENAPI_ARTIFACT_CACHE_TTL_MS = 24 * 60 * 60_000;37const OPENAPI_ARTIFACT_CACHE_TTL_MS = 24 * 60 * 60_000;
38const OPENAPI_PROFILE_FALLBACK_WARNING_PREFIX = "OpenAPI profile fallback:";
39
40type OpenApiPrepareProfile = "full" | "inventory";
38 41
39const openApiAuthModeSchema = z.enum(["account", "workspace", "organization"]);42const openApiAuthModeSchema = z.enum(["account", "workspace", "organization"]);
40 43
@@ -105,6 +108,7 @@ async function loadPreparedOpenApiSpecFromExternalGenerate( · 64 unmodified lines
105 specUrl: string,108 specUrl: string,
106 sourceName: string,109 sourceName: string,
107 includeDts: boolean,110 includeDts: boolean,
111 profile: "full" | "inventory",
108 headers?: Record<string, string>,112 headers?: Record<string, string>,
109): Promise<PreparedOpenApiSpec | null> {113): Promise<PreparedOpenApiSpec | null> {
110 const endpoint = resolveExternalGenerateEndpoint();114 const endpoint = resolveExternalGenerateEndpoint();
@@ -120,6 +124,7 @@ async function loadPreparedOpenApiSpecFromExternalGenerate( · 9 unmodified lines
120 url.searchParams.set("specUrl", specUrl);124 url.searchParams.set("specUrl", specUrl);
121 url.searchParams.set("sourceName", sourceName);125 url.searchParams.set("sourceName", sourceName);
122 url.searchParams.set("includeDts", includeDts ? "1" : "0");126 url.searchParams.set("includeDts", includeDts ? "1" : "0");
127 url.searchParams.set("profile", profile);
123 if (headers && Object.keys(headers).length > 0) {128 if (headers && Object.keys(headers).length > 0) {
124 url.searchParams.set("headers", JSON.stringify(headers));129 url.searchParams.set("headers", JSON.stringify(headers));
125 }130 }
@@ -149,6 +154,7 @@ async function loadPreparedOpenApiSpecFromExternalGenerate( · 23 unmodified lines
149async function cachePreparedOpenApiSpec(154async function cachePreparedOpenApiSpec(
150 ctx: ActionCtx,155 ctx: ActionCtx,
151 specUrl: string,156 specUrl: string,
157 version: string,
152 sourceName: string,158 sourceName: string,
153 prepared: PreparedOpenApiSpec,159 prepared: PreparedOpenApiSpec,
154 preparedStorageId?: Id<"_storage">,160 preparedStorageId?: Id<"_storage">,
@@ -159,7 +165,7 @@ async function cachePreparedOpenApiSpec( · 4 unmodified lines
159 const storageId = preparedStorageId ?? await ctx.storage.store(new Blob([json], { type: "application/json" }));165 const storageId = preparedStorageId ?? await ctx.storage.store(new Blob([json], { type: "application/json" }));
160 await ctx.runMutation(internal.openApiSpecCache.putEntry, {166 await ctx.runMutation(internal.openApiSpecCache.putEntry, {
161 specUrl,167 specUrl,
162 version: OPENAPI_PREPARED_CACHE_VERSION,168 version,
163 storageId,169 storageId,
164 sizeBytes: preparedSizeBytes ?? json.length,170 sizeBytes: preparedSizeBytes ?? json.length,
165 });171 });
@@ -584,6 +590,10 @@ async function resolveOpenApiFetchHeaders( · 418 unmodified lines
584 return { headers, warnings: [] };590 return { headers, warnings: [] };
585}591}
586 592
593function preparedCacheVersion(profile: OpenApiPrepareProfile, includeDts: boolean): string {
594 return `${OPENAPI_PREPARED_CACHE_VERSION}:${profile}:${includeDts ? "dts" : "nodts"}`;
595}
596
587async function loadCachedOpenApiSpec(597async function loadCachedOpenApiSpec(
588 ctx: ActionCtx,598 ctx: ActionCtx,
589 specUrl: string,599 specUrl: string,
@@ -599,125 +609,185 @@ async function loadCachedOpenApiSpec( · 9 unmodified lines
599 return prepared.dts ? "ready" : "failed";609 return prepared.dts ? "ready" : "failed";
600 };610 };
601 611
602 if (!hasHeaders) {612 const parsePreparedResponse = async (preparedResponse: unknown): Promise<{
603 try {613 prepared: PreparedOpenApiSpec;
604 const entry = await ctx.runQuery(internal.openApiSpecCache.getEntry, {614 preparedStorageId?: Id<"_storage">;
605 specUrl,615 preparedSizeBytes?: number;
606 version: OPENAPI_PREPARED_CACHE_VERSION,616 }> => {
607 maxAgeMs: OPENAPI_SPEC_CACHE_TTL_MS,617 let preparedPayload: unknown;
608 });618 let preparedStorageId: Id<"_storage"> | undefined;
609 619 let preparedSizeBytes: number | undefined;
610 if (entry) {620
611 const blob = await ctx.storage.get(entry.storageId);621 if (typeof preparedResponse === "string") {
612 if (blob) {622 try {
613 const json = await blob.text();623 const parsed = JSON.parse(preparedResponse) as unknown;
614 let parsedJson: unknown;624 const parsedEnvelope = preparedOpenApiEnvelopeSchema.safeParse(parsed);
615 try {625 if (parsedEnvelope.success) {
616 parsedJson = JSON.parse(json);626 preparedStorageId = parsedEnvelope.data.storageId as Id<"_storage">;
617 } catch {627 preparedSizeBytes = parsedEnvelope.data.sizeBytes;
618 parsedJson = undefined;628
629 const blob = await ctx.storage.get(preparedStorageId);
630 if (!blob) {
631 throw new Error(`Prepared OpenAPI blob missing for '${sourceName}'`);
619 }632 }
633 const text = await blob.text();
634 preparedPayload = JSON.parse(text);
635 } else {
636 preparedPayload = parsed;
637 }
638 } catch {
639 preparedPayload = undefined;
640 }
641 } else {
642 preparedPayload = preparedResponse;
643 }
644
645 const prepared = toPreparedOpenApiSpec(preparedPayload);
646 if (!prepared) {
647 throw new Error(`Prepared OpenAPI payload for '${sourceName}' was invalid`);
648 }
649
650 return {
651 prepared,
652 preparedStorageId,
653 preparedSizeBytes,
654 };
655 };
656
657 const loadForProfile = async (profile: OpenApiPrepareProfile): Promise<PreparedOpenApiSpec> => {
658 const cacheVersion = preparedCacheVersion(profile, includeDts);
659
660 if (!hasHeaders) {
661 try {
662 const entry = await ctx.runQuery(internal.openApiSpecCache.getEntry, {
663 specUrl,
664 version: cacheVersion,
665 maxAgeMs: OPENAPI_SPEC_CACHE_TTL_MS,
666 });
667
668 if (entry) {
669 const blob = await ctx.storage.get(entry.storageId);
670 if (blob) {
671 const json = await blob.text();
672 let parsedJson: unknown;
673 try {
674 parsedJson = JSON.parse(json);
675 } catch {
676 parsedJson = undefined;
677 }
620 678
621 if (parsedJson !== undefined) {679 if (parsedJson !== undefined) {
622 const prepared = toPreparedOpenApiSpec(parsedJson);680 const prepared = toPreparedOpenApiSpec(parsedJson);
623 if (prepared && (!includeDts || getDtsStatus(prepared) !== "skipped")) {681 if (prepared && (!includeDts || getDtsStatus(prepared) !== "skipped")) {
624 return prepared;682 return prepared;
683 }
625 }684 }
626 }685 }
627 }686 }
687 } catch (error) {
688 const message = error instanceof Error ? error.message : String(error);
689 console.warn(`[executor] OpenAPI cache read failed for '${sourceName}' (${profile}): ${message}`);
628 }690 }
629 } catch (error) {
630 const message = error instanceof Error ? error.message : String(error);
631 console.warn(`[executor] OpenAPI cache read failed for '${sourceName}': ${message}`);
632 }691 }
633 }
634 692
635 const externalPrepared = await loadPreparedOpenApiSpecFromExternalGenerate(693 const externalPrepared = await loadPreparedOpenApiSpecFromExternalGenerate(
636 specUrl,694 specUrl,
637 sourceName,695 sourceName,
638 includeDts,696 includeDts,
639 headers,697 profile,
640 );698 headers,
641 if (externalPrepared) {699 );
642 if (!hasHeaders) {700 if (externalPrepared) {
643 await cachePreparedOpenApiSpec(ctx, specUrl, sourceName, externalPrepared);701 if (!hasHeaders) {
702 await cachePreparedOpenApiSpec(
703 ctx,
704 specUrl,
705 cacheVersion,
706 sourceName,
707 externalPrepared,
708 );
709 }
710 return externalPrepared;
644 }711 }
645 return externalPrepared;
646 }
647 712
648 let preparedResponse: unknown = undefined;713 let preparedResponse: unknown = undefined;
649 let lastPrepareError: string | undefined;714 let lastPrepareError: string | undefined;
650 for (let attempt = 1; attempt <= OPENAPI_PREPARE_MAX_ATTEMPTS; attempt += 1) {715 const prepareOpenApiSpecAction = internal.runtimeNode.prepareOpenApiSpec as unknown as {
651 try {716 _doNotReferenceDirectly?: true;
652 preparedResponse = await ctx.runAction(internal.runtimeNode.prepareOpenApiSpec, {717 };
653 specUrl,718 const maxAttempts = profile === "full" && !includeDts ? 1 : OPENAPI_PREPARE_MAX_ATTEMPTS;
654 sourceName,719 for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
655 includeDts,720 try {
656 });721 preparedResponse = await ctx.runAction(prepareOpenApiSpecAction as never, {
657 lastPrepareError = undefined;722 specUrl,
658 break;723 sourceName,
659 } catch (error) {724 includeDts,
660 const message = error instanceof Error ? error.message : String(error);725 profile,
661 lastPrepareError = message;726 } as never);
662 if (attempt >= OPENAPI_PREPARE_MAX_ATTEMPTS) {727 lastPrepareError = undefined;
663 break;728 break;
729 } catch (error) {
730 const message = error instanceof Error ? error.message : String(error);
731 lastPrepareError = message;
732 if (attempt >= maxAttempts) {
733 break;
734 }
735
736 const delayMs = OPENAPI_PREPARE_RETRY_BASE_DELAY_MS * attempt;
737 await new Promise((resolve) => setTimeout(resolve, delayMs));
664 }738 }
739 }
665 740
666 const delayMs = OPENAPI_PREPARE_RETRY_BASE_DELAY_MS * attempt;741 if (lastPrepareError) {
667 await new Promise((resolve) => setTimeout(resolve, delayMs));742 throw new Error(
743 `Failed to prepare OpenAPI source '${sourceName}' with profile '${profile}' after ${maxAttempts} attempt(s): ${lastPrepareError}`,
744 );
668 }745 }
669 }
670 746
671 if (lastPrepareError) {747 const parsedPrepared = await parsePreparedResponse(preparedResponse);
672 throw new Error(748 if (!hasHeaders) {
673 `Failed to prepare OpenAPI source '${sourceName}' after ${OPENAPI_PREPARE_MAX_ATTEMPTS} attempts: ${lastPrepareError}`,749 await cachePreparedOpenApiSpec(
674 );750 ctx,
675 }751 specUrl,
752 cacheVersion,
753 sourceName,
754 parsedPrepared.prepared,
755 parsedPrepared.preparedStorageId,
756 parsedPrepared.preparedSizeBytes,
757 );
758 }
759
760 return parsedPrepared.prepared;
761 };
676 762
677 let preparedPayload: unknown;763 const profilesToTry: OpenApiPrepareProfile[] = includeDts ? ["full"] : ["full", "inventory"];
678 let preparedStorageId: Id<"_storage"> | undefined;764 const profileErrors: string[] = [];
679 let preparedSizeBytes: number | undefined;765
680 if (typeof preparedResponse === "string") {766 for (let index = 0; index < profilesToTry.length; index += 1) {
767 const profile = profilesToTry[index]!;
681 try {768 try {
682 const parsed = JSON.parse(preparedResponse) as unknown;769 const prepared = await loadForProfile(profile);
683 const parsedEnvelope = preparedOpenApiEnvelopeSchema.safeParse(parsed);770 if (index === 0) {
684 if (parsedEnvelope.success) {771 return prepared;
685 preparedStorageId = parsedEnvelope.data.storageId as Id<"_storage">;
686 preparedSizeBytes = parsedEnvelope.data.sizeBytes;
687
688 const blob = await ctx.storage.get(preparedStorageId);
689 if (!blob) {
690 throw new Error(`Prepared OpenAPI blob missing for '${sourceName}'`);
691 }
692 const text = await blob.text();
693 preparedPayload = JSON.parse(text);
694 } else {
695 preparedPayload = parsed;
696 }772 }
697 } catch {
698 preparedPayload = undefined;
699 }
700 } else {
701 preparedPayload = preparedResponse;
702 }
703 773
704 const prepared = toPreparedOpenApiSpec(preparedPayload);774 const warning = `${OPENAPI_PROFILE_FALLBACK_WARNING_PREFIX} Source '${sourceName}' fell back to '${profile}' profile after full profile failed: ${profileErrors[0] ?? "unknown error"}`;
705 if (!prepared) {775 return {
706 throw new Error(`Prepared OpenAPI payload for '${sourceName}' was invalid`);776 ...prepared,
707 }777 warnings: [...(prepared.warnings ?? []), warning],
708 778 };
709 if (!hasHeaders) {779 } catch (error) {
710 await cachePreparedOpenApiSpec(780 const message = error instanceof Error ? error.message : String(error);
711 ctx,781 profileErrors.push(message);
712 specUrl,782 if (index === profilesToTry.length - 1) {
713 sourceName,783 break;
714 prepared,784 }
715 preparedStorageId,785 }
716 preparedSizeBytes,
717 );
718 }786 }
719 787
720 return prepared;788 throw new Error(
789 `Failed to prepare OpenAPI source '${sourceName}' after ${profilesToTry.length} profile attempt(s): ${profileErrors.join("; ")}`,
790 );
721}791}
722 792
723export async function loadSourceArtifact(793export async function loadSourceArtifact(
@@ -761,8 +831,11 @@ export async function loadSourceArtifact( · 37 unmodified lines
761 resolvedOpenApiHeaders.headers,831 resolvedOpenApiHeaders.headers,
762 );832 );
763 const artifact = compileOpenApiArtifactFromPrepared(source, prepared);833 const artifact = compileOpenApiArtifactFromPrepared(source, prepared);
834 const usedFallbackProfile = (prepared.warnings ?? []).some((warning) =>
835 warning.startsWith(OPENAPI_PROFILE_FALLBACK_WARNING_PREFIX)
836 );
764 837
765 if (sharedArtifactCacheEligible && sharedArtifactCacheKey) {838 if (sharedArtifactCacheEligible && sharedArtifactCacheKey && !usedFallbackProfile) {
766 await putSharedOpenApiArtifactCache(ctx, sharedArtifactCacheKey, artifact);839 await putSharedOpenApiArtifactCache(ctx, sharedArtifactCacheKey, artifact);
767 }840 }
768 841

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t