History

external generate route

Rhys Sullivan committed 8 months ago

Commit 01a76291 parent 3f694694 files · +391 −13Browse files

Unified · Split

4 files
  1. .env.example +3 −0
  2. executor/apps/web/src/app/api/generate/route.ts +45 −0
  3. executor/packages/core/src/openapi/real-specs.test.ts +240 −0
  4. executor/packages/database/src/runtime/tool_source_loading.ts +103 −13
.env.example+3−0
@@ -17,6 +17,9 @@ CONVEX_SITE_URL=https://your-deployment.convex.site · 16 unmodified lines
17# When enabled, local Bun execution is allowed.17# When enabled, local Bun execution is allowed.
18# Leave unset in production to default to Cloudflare runtime.18# Leave unset in production to default to Cloudflare runtime.
19# DANGEROUSLY_ALLOW_LOCAL_VM=119# DANGEROUSLY_ALLOW_LOCAL_VM=1
20# Optional external OpenAPI generator endpoint used by Convex for heavy parse/prepare.
21# Expects POST /api/generate with { specUrl, sourceName, includeDts }.
22# EXECUTOR_GENERATE_URL=http://localhost:4312
20 23
21# ── Agent storage backend (optional) ──24# ── Agent storage backend (optional) ──
22# Default: agentfs-local25# Default: agentfs-local
executor/apps/web/src/app/api/generate/route.ts+45−0
@@ -0,0 +1,45 @@
1import { z } from "zod";
2import { prepareOpenApiSpec } from "../../../../../../packages/core/src/openapi-prepare";
3
4const requestSchema = z.object({
5 specUrl: z.string().trim().min(1),
6 sourceName: z.string().trim().min(1),
7 includeDts: z.boolean().optional(),
8});
9
10function noStoreJson(payload: unknown, status: number): Response {
11 return Response.json(payload, {
12 status,
13 headers: {
14 "cache-control": "no-store",
15 },
16 });
17}
18
19export async function POST(request: Request): Promise<Response> {
20 const parsed = requestSchema.safeParse(await request.json().catch(() => null));
21 if (!parsed.success) {
22 return noStoreJson({ error: "Invalid generate request" }, 400);
23 }
24
25 try {
26 const prepared = await prepareOpenApiSpec(
27 parsed.data.specUrl,
28 parsed.data.sourceName,
29 {
30 includeDts: parsed.data.includeDts ?? false,
31 profile: "inventory",
32 },
33 );
34
35 return noStoreJson({
36 status: "ready",
37 prepared,
38 }, 200);
39 } catch (error) {
40 return noStoreJson({
41 status: "failed",
42 error: error instanceof Error ? error.message : "Failed to generate",
43 }, 502);
44 }
45}
executor/packages/core/src/openapi/real-specs.test.ts+240−0
@@ -436,6 +436,34 @@ function containsUnknownTypeToken(segment: string): boolean { · 435 unmodified lines
436 return /\bunknown\b/.test(withoutQuotedLiterals);436 return /\bunknown\b/.test(withoutQuotedLiterals);
437}437}
438 438
439function isSchemaRefOnly(schema: Record<string, unknown>): boolean {
440 const keys = Object.keys(schema);
441 return keys.length > 0
442 && typeof schema.$ref === "string"
443 && keys.every((key) => key === "$ref");
444}
445
446function resolveExpectedInputPrimitiveFromGeneratedSchema(
447 inputSchema: Record<string, unknown>,
448 fieldName: string,
449 componentSchemas: Record<string, unknown>,
450): PrimitiveParamType | null {
451 const rootProps = toRecord(inputSchema.properties);
452 const containerKeys = ["path", "query", "headers", "cookie", "body"] as const;
453
454 const direct = resolveSchemaPrimitiveType(rootProps[fieldName], componentSchemas);
455 if (direct) return direct;
456
457 for (const container of containerKeys) {
458 const containerSchema = toRecord(rootProps[container]);
459 const containerProps = toRecord(containerSchema.properties);
460 const resolved = resolveSchemaPrimitiveType(containerProps[fieldName], componentSchemas);
461 if (resolved) return resolved;
462 }
463
464 return null;
465}
466
439const SPECS: SpecFixture[] = [467const SPECS: SpecFixture[] = [
440 {468 {
441 name: "jira",469 name: "jira",
@@ -696,6 +724,113 @@ describe("real-world OpenAPI specs", () => { · 254 unmodified lines
696 300_000,724 300_000,
697 );725 );
698 726
727 test(
728 "github: enterprise code security configurations resolves array item schema in includeDts=false mode",
729 async () => {
730 const githubUrl =
731 "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.yaml";
732
733 const prepared = await prepareOpenApiSpec(githubUrl, "github", {
734 includeDts: false,
735 profile: "full",
736 });
737 const tools = buildOpenApiToolsFromPrepared(
738 {
739 type: "openapi",
740 name: "github",
741 spec: githubUrl,
742 baseUrl: prepared.servers[0] || "https://api.github.com",
743 },
744 prepared,
745 );
746
747 const tool = tools.find((t) => t.path === "github.code_security.get_configurations_for_enterprise");
748
749 expect(tool).toBeDefined();
750 const outputSchema = toRecord(tool!.typing?.outputSchema);
751 expect(outputSchema.type).toBe("array");
752
753 const outputItems = toRecord(outputSchema.items);
754 expect(typeof outputItems.$ref).not.toBe("string");
755 expect(outputItems.type).toBe("object");
756
757 const outputProps = toRecord(outputItems.properties);
758 expect(toRecord(outputProps.id).type).toBe("integer");
759 expect(toRecord(outputProps.name).type).toBe("string");
760 },
761 300_000,
762 );
763
764 test(
765 "github + stripe: root generated array schemas include concrete item definitions",
766 async () => {
767 const fixtures = [
768 {
769 name: "github",
770 url: "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.yaml",
771 fallbackBaseUrl: "https://api.github.com",
772 },
773 {
774 name: "stripe",
775 url: "https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json",
776 fallbackBaseUrl: "https://api.stripe.com",
777 },
778 ] as const;
779
780 for (const fixture of fixtures) {
781 const prepared = await prepareOpenApiSpec(fixture.url, fixture.name, {
782 includeDts: false,
783 profile: "full",
784 });
785 const tools = buildOpenApiToolsFromPrepared(
786 {
787 type: "openapi",
788 name: fixture.name,
789 spec: fixture.url,
790 baseUrl: prepared.servers[0] || fixture.fallbackBaseUrl,
791 },
792 prepared,
793 );
794
795 const issues: string[] = [];
796
797 const validateArrayItems = (schema: Record<string, unknown>, location: string) => {
798 const items = toRecord(schema.items);
799 const itemKeys = Object.keys(items);
800 if (itemKeys.length === 0) {
801 issues.push(`${location}.items has empty items schema`);
802 return;
803 }
804
805 const isRefOnly = typeof items.$ref === "string" && itemKeys.every((key) => key === "$ref");
806 if (isRefOnly) {
807 issues.push(`${location}.items is ref-only (${String(items.$ref)})`);
808 }
809 };
810
811 const visitTopLevelSchema = (schemaValue: unknown, location: string) => {
812 const schema = toRecord(schemaValue);
813 if (Object.keys(schema).length === 0) return;
814
815 if (schema.type === "array") {
816 validateArrayItems(schema, `${location}:schema`);
817 }
818 };
819
820 for (const tool of tools) {
821 visitTopLevelSchema(tool.typing?.inputSchema, `${tool.path}:input`);
822 visitTopLevelSchema(tool.typing?.outputSchema, `${tool.path}:output`);
823 }
824
825 expect(
826 issues,
827 `${fixture.name} has array item schema regressions:\n${issues.slice(0, 20).join("\n")}`,
828 ).toHaveLength(0);
829 }
830 },
831 300_000,
832 );
833
699 test(834 test(
700 "stripe: extraction keeps concrete schemas for forwarding + verification report list outputs",835 "stripe: extraction keeps concrete schemas for forwarding + verification report list outputs",
701 async () => {836 async () => {
@@ -1039,4 +1174,109 @@ describe("real-world OpenAPI specs", () => { · 337 unmodified lines
1039 },1174 },
1040 1_800_000,1175 1_800_000,
1041 );1176 );
1177
1178 test(
1179 "full profile schemas validate every github/stripe operation against parser-derived expectations",
1180 async () => {
1181 const fixtures = SPECS.filter((fixture) => fixture.name === "github" || fixture.name === "stripe");
1182 expect(fixtures.length).toBe(2);
1183
1184 for (const fixture of fixtures) {
1185 const parsed = await SwaggerParser.parse(fixture.url);
1186 const parsedSpec = parsed as Record<string, unknown>;
1187 const componentSchemas = toRecord(toRecord(parsedSpec.components).schemas);
1188 const operationExpectations = collectOperationTypeExpectations(parsedSpec);
1189 expect(operationExpectations.size).toBeGreaterThan(0);
1190
1191 const prepared = await prepareOpenApiSpec(parsedSpec, fixture.name, {
1192 includeDts: false,
1193 profile: "full",
1194 });
1195
1196 const tools = buildOpenApiToolsFromPrepared(
1197 {
1198 type: "openapi",
1199 name: fixture.name,
1200 spec: fixture.url,
1201 baseUrl: prepared.servers[0] || `https://${fixture.name}.example.com`,
1202 },
1203 prepared,
1204 );
1205
1206 const toolsByOperationId = new Map<string, (typeof tools)[number]>();
1207 for (const tool of tools) {
1208 const operationId = tool.typing?.typedRef?.kind === "openapi_operation"
1209 ? tool.typing.typedRef.operationId
1210 : "";
1211 if (!operationId || toolsByOperationId.has(operationId)) continue;
1212 toolsByOperationId.set(operationId, tool);
1213 }
1214
1215 expect(toolsByOperationId.size).toBe(operationExpectations.size);
1216
1217 const issues: string[] = [];
1218
1219 for (const [operationId, expectations] of operationExpectations.entries()) {
1220 const tool = toolsByOperationId.get(operationId);
1221 if (!tool) {
1222 issues.push(`missing tool for operationId ${operationId}`);
1223 continue;
1224 }
1225
1226 const inputSchema = toRecord(tool.typing?.inputSchema);
1227 const outputSchema = toRecord(tool.typing?.outputSchema);
1228
1229 if (expectations.expectsInput && Object.keys(inputSchema).length === 0) {
1230 issues.push(`${tool.path}: expected input schema but got empty`);
1231 }
1232
1233 if (expectations.expectsKnownOutput && Object.keys(outputSchema).length === 0) {
1234 issues.push(`${tool.path}: expected output schema but got empty`);
1235 }
1236
1237 if (expectations.expectsVoidOutput && Object.keys(outputSchema).length > 0) {
1238 issues.push(`${tool.path}: expected void output but schema was present`);
1239 }
1240
1241 if (outputSchema.type === "array") {
1242 const items = toRecord(outputSchema.items);
1243 if (Object.keys(items).length === 0) {
1244 issues.push(`${tool.path}: output array has empty items schema`);
1245 } else if (isSchemaRefOnly(items)) {
1246 issues.push(`${tool.path}: output array items are ref-only (${String(items.$ref)})`);
1247 }
1248 }
1249
1250 for (const expected of expectations.expectedInputFields) {
1251 const resolved = resolveExpectedInputPrimitiveFromGeneratedSchema(inputSchema, expected.name, componentSchemas);
1252 if (!resolved) {
1253 issues.push(`${tool.path}: missing input field ${expected.name}`);
1254 continue;
1255 }
1256 if (resolved !== expected.type) {
1257 issues.push(`${tool.path}: input field ${expected.name} expected ${expected.type}, got ${resolved}`);
1258 }
1259 }
1260
1261 const outputProperties = toRecord(outputSchema.properties);
1262 for (const expected of expectations.expectedOutputFields) {
1263 const resolved = resolveSchemaPrimitiveType(outputProperties[expected.name], componentSchemas);
1264 if (!resolved) {
1265 issues.push(`${tool.path}: missing output field ${expected.name}`);
1266 continue;
1267 }
1268 if (resolved !== expected.type) {
1269 issues.push(`${tool.path}: output field ${expected.name} expected ${expected.type}, got ${resolved}`);
1270 }
1271 }
1272 }
1273
1274 expect(
1275 issues,
1276 `${fixture.name} full-profile schema regressions:\n${issues.slice(0, 30).join("\n")}`,
1277 ).toHaveLength(0);
1278 }
1279 },
1280 1_800_000,
1281 );
1042});1282});
executor/packages/database/src/runtime/tool_source_loading.ts+103−13
@@ -30,6 +30,7 @@ import { · 29 unmodified lines
30const OPENAPI_SPEC_CACHE_TTL_MS = 5 * 60 * 60_000;30const OPENAPI_SPEC_CACHE_TTL_MS = 5 * 60 * 60_000;
31const OPENAPI_PREPARE_MAX_ATTEMPTS = 3;31const OPENAPI_PREPARE_MAX_ATTEMPTS = 3;
32const OPENAPI_PREPARE_RETRY_BASE_DELAY_MS = 1_500;32const OPENAPI_PREPARE_RETRY_BASE_DELAY_MS = 1_500;
33const OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS = 180_000;
33 34
34const OPENAPI_PREPARED_CACHE_VERSION = "openapi_v11";35const OPENAPI_PREPARED_CACHE_VERSION = "openapi_v11";
35const OPENAPI_ARTIFACT_CACHE_VERSION = "openapi_artifact_v1";36const OPENAPI_ARTIFACT_CACHE_VERSION = "openapi_artifact_v1";
@@ -80,6 +81,90 @@ const preparedOpenApiEnvelopeSchema = z.object({ · 44 unmodified lines
80 sizeBytes: z.number().optional(),81 sizeBytes: z.number().optional(),
81});82});
82 83
84const externalGenerateResponseSchema = z.object({
85 status: z.enum(["ready", "failed"]).optional(),
86 prepared: z.unknown().optional(),
87 error: z.string().optional(),
88});
89
90function trimEnv(value: string | undefined): string | undefined {
91 const trimmed = value?.trim();
92 return trimmed && trimmed.length > 0 ? trimmed : undefined;
93}
94
95function resolveExternalGenerateEndpoint(): string | undefined {
96 const base = trimEnv(process.env.EXECUTOR_GENERATE_URL);
97 if (!base) {
98 return undefined;
99 }
100
101 return base.endsWith("/api/generate") ? base : `${base.replace(/\/$/, "")}/api/generate`;
102}
103
104async function loadPreparedOpenApiSpecFromExternalGenerate(
105 specUrl: string,
106 sourceName: string,
107 includeDts: boolean,
108): Promise<PreparedOpenApiSpec | null> {
109 const endpoint = resolveExternalGenerateEndpoint();
110 if (!endpoint) {
111 return null;
112 }
113
114 const controller = new AbortController();
115 const timeout = setTimeout(() => controller.abort(), OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS);
116
117 try {
118 const response = await fetch(endpoint, {
119 method: "POST",
120 headers: {
121 "content-type": "application/json",
122 },
123 body: JSON.stringify({ specUrl, sourceName, includeDts }),
124 cache: "no-store",
125 signal: controller.signal,
126 });
127
128 const payload = externalGenerateResponseSchema.safeParse(await response.json().catch(() => ({})));
129 if (!payload.success) {
130 return null;
131 }
132
133 if (!response.ok || payload.data.status === "failed") {
134 return null;
135 }
136
137 return toPreparedOpenApiSpec(payload.data.prepared);
138 } catch {
139 return null;
140 } finally {
141 clearTimeout(timeout);
142 }
143}
144
145async function cachePreparedOpenApiSpec(
146 ctx: ActionCtx,
147 specUrl: string,
148 sourceName: string,
149 prepared: PreparedOpenApiSpec,
150 preparedStorageId?: Id<"_storage">,
151 preparedSizeBytes?: number,
152): Promise<void> {
153 try {
154 const json = JSON.stringify(prepared);
155 const storageId = preparedStorageId ?? await ctx.storage.store(new Blob([json], { type: "application/json" }));
156 await ctx.runMutation(internal.openApiSpecCache.putEntry, {
157 specUrl,
158 version: OPENAPI_PREPARED_CACHE_VERSION,
159 storageId,
160 sizeBytes: preparedSizeBytes ?? json.length,
161 });
162 } catch (error) {
163 const message = error instanceof Error ? error.message : String(error);
164 console.warn(`[executor] OpenAPI cache write failed for '${sourceName}': ${message}`);
165 }
166}
167
83function stableStringify(value: unknown): string {168function stableStringify(value: unknown): string {
84 if (Array.isArray(value)) {169 if (Array.isArray(value)) {
85 return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;170 return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
@@ -459,6 +544,16 @@ async function loadCachedOpenApiSpec( · 373 unmodified lines
459 console.warn(`[executor] OpenAPI cache read failed for '${sourceName}': ${message}`);544 console.warn(`[executor] OpenAPI cache read failed for '${sourceName}': ${message}`);
460 }545 }
461 546
547 const externalPrepared = await loadPreparedOpenApiSpecFromExternalGenerate(
548 specUrl,
549 sourceName,
550 includeDts,
551 );
552 if (externalPrepared) {
553 await cachePreparedOpenApiSpec(ctx, specUrl, sourceName, externalPrepared);
554 return externalPrepared;
555 }
556
462 let preparedResponse: unknown = undefined;557 let preparedResponse: unknown = undefined;
463 let lastPrepareError: string | undefined;558 let lastPrepareError: string | undefined;
464 for (let attempt = 1; attempt <= OPENAPI_PREPARE_MAX_ATTEMPTS; attempt += 1) {559 for (let attempt = 1; attempt <= OPENAPI_PREPARE_MAX_ATTEMPTS; attempt += 1) {
@@ -520,19 +615,14 @@ async function loadCachedOpenApiSpec( · 55 unmodified lines
520 throw new Error(`Prepared OpenAPI payload for '${sourceName}' was invalid`);615 throw new Error(`Prepared OpenAPI payload for '${sourceName}' was invalid`);
521 }616 }
522 617
523 try {618 await cachePreparedOpenApiSpec(
524 const json = JSON.stringify(prepared);619 ctx,
525 const storageId = preparedStorageId ?? await ctx.storage.store(new Blob([json], { type: "application/json" }));620 specUrl,
526 await ctx.runMutation(internal.openApiSpecCache.putEntry, {621 sourceName,
527 specUrl,622 prepared,
528 version: OPENAPI_PREPARED_CACHE_VERSION,623 preparedStorageId,
529 storageId,624 preparedSizeBytes,
530 sizeBytes: preparedSizeBytes ?? json.length,625 );
531 });
532 } catch (error) {
533 const message = error instanceof Error ? error.message : String(error);
534 console.warn(`[executor] OpenAPI cache write failed for '${sourceName}': ${message}`);
535 }
536 626
537 return prepared;627 return prepared;
538}628}