History

external generate route

Rhys Sullivan committed 8 months ago

Commit 01a76291 parent 3f694694 files · +391 −13Browse files

Unified · Split

4 files
  1. .env.example +3 −0
  2. executor/apps/web/src/app/api/generate/route.ts +45 −0
  3. executor/packages/core/src/openapi/real-specs.test.ts +240 −0
  4. executor/packages/database/src/runtime/tool_source_loading.ts +103 −13
.env.example+3−0
@@ -17,6 +17,9 @@ CONVEX_SITE_URL=https://your-deployment.convex.site · 16 unmodified lines
1717 # When enabled, local Bun execution is allowed.
1818 # Leave unset in production to default to Cloudflare runtime.
1919 # DANGEROUSLY_ALLOW_LOCAL_VM=1
20+# Optional external OpenAPI generator endpoint used by Convex for heavy parse/prepare.
21+# Expects POST /api/generate with { specUrl, sourceName, includeDts }.
22+# EXECUTOR_GENERATE_URL=http://localhost:4312
2023
2124 # ── Agent storage backend (optional) ──
2225 # Default: agentfs-local
executor/apps/web/src/app/api/generate/route.ts+45−0
@@ -0,0 +1,45 @@
1+import { z } from "zod";
2+import { prepareOpenApiSpec } from "../../../../../../packages/core/src/openapi-prepare";
3+
4+const requestSchema = z.object({
5+ specUrl: z.string().trim().min(1),
6+ sourceName: z.string().trim().min(1),
7+ includeDts: z.boolean().optional(),
8+});
9+
10+function noStoreJson(payload: unknown, status: number): Response {
11+ return Response.json(payload, {
12+ status,
13+ headers: {
14+ "cache-control": "no-store",
15+ },
16+ });
17+}
18+
19+export async function POST(request: Request): Promise<Response> {
20+ const parsed = requestSchema.safeParse(await request.json().catch(() => null));
21+ if (!parsed.success) {
22+ return noStoreJson({ error: "Invalid generate request" }, 400);
23+ }
24+
25+ try {
26+ const prepared = await prepareOpenApiSpec(
27+ parsed.data.specUrl,
28+ parsed.data.sourceName,
29+ {
30+ includeDts: parsed.data.includeDts ?? false,
31+ profile: "inventory",
32+ },
33+ );
34+
35+ return noStoreJson({
36+ status: "ready",
37+ prepared,
38+ }, 200);
39+ } catch (error) {
40+ return noStoreJson({
41+ status: "failed",
42+ error: error instanceof Error ? error.message : "Failed to generate",
43+ }, 502);
44+ }
45+}
executor/packages/core/src/openapi/real-specs.test.ts+240−0
@@ -436,6 +436,34 @@ function containsUnknownTypeToken(segment: string): boolean { · 435 unmodified lines
436436 return /\bunknown\b/.test(withoutQuotedLiterals);
437437 }
438438
439+function isSchemaRefOnly(schema: Record<string, unknown>): boolean {
440+ const keys = Object.keys(schema);
441+ return keys.length > 0
442+ && typeof schema.$ref === "string"
443+ && keys.every((key) => key === "$ref");
444+}
445+
446+function resolveExpectedInputPrimitiveFromGeneratedSchema(
447+ inputSchema: Record<string, unknown>,
448+ fieldName: string,
449+ componentSchemas: Record<string, unknown>,
450+): PrimitiveParamType | null {
451+ const rootProps = toRecord(inputSchema.properties);
452+ const containerKeys = ["path", "query", "headers", "cookie", "body"] as const;
453+
454+ const direct = resolveSchemaPrimitiveType(rootProps[fieldName], componentSchemas);
455+ if (direct) return direct;
456+
457+ for (const container of containerKeys) {
458+ const containerSchema = toRecord(rootProps[container]);
459+ const containerProps = toRecord(containerSchema.properties);
460+ const resolved = resolveSchemaPrimitiveType(containerProps[fieldName], componentSchemas);
461+ if (resolved) return resolved;
462+ }
463+
464+ return null;
465+}
466+
439467 const SPECS: SpecFixture[] = [
440468 {
441469 name: "jira",
@@ -696,6 +724,113 @@ describe("real-world OpenAPI specs", () => { · 254 unmodified lines
696724 300_000,
697725 );
698726
727+ test(
728+ "github: enterprise code security configurations resolves array item schema in includeDts=false mode",
729+ async () => {
730+ const githubUrl =
731+ "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.yaml";
732+
733+ const prepared = await prepareOpenApiSpec(githubUrl, "github", {
734+ includeDts: false,
735+ profile: "full",
736+ });
737+ const tools = buildOpenApiToolsFromPrepared(
738+ {
739+ type: "openapi",
740+ name: "github",
741+ spec: githubUrl,
742+ baseUrl: prepared.servers[0] || "https://api.github.com",
743+ },
744+ prepared,
745+ );
746+
747+ const tool = tools.find((t) => t.path === "github.code_security.get_configurations_for_enterprise");
748+
749+ expect(tool).toBeDefined();
750+ const outputSchema = toRecord(tool!.typing?.outputSchema);
751+ expect(outputSchema.type).toBe("array");
752+
753+ const outputItems = toRecord(outputSchema.items);
754+ expect(typeof outputItems.$ref).not.toBe("string");
755+ expect(outputItems.type).toBe("object");
756+
757+ const outputProps = toRecord(outputItems.properties);
758+ expect(toRecord(outputProps.id).type).toBe("integer");
759+ expect(toRecord(outputProps.name).type).toBe("string");
760+ },
761+ 300_000,
762+ );
763+
764+ test(
765+ "github + stripe: root generated array schemas include concrete item definitions",
766+ async () => {
767+ const fixtures = [
768+ {
769+ name: "github",
770+ url: "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.yaml",
771+ fallbackBaseUrl: "https://api.github.com",
772+ },
773+ {
774+ name: "stripe",
775+ url: "https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json",
776+ fallbackBaseUrl: "https://api.stripe.com",
777+ },
778+ ] as const;
779+
780+ for (const fixture of fixtures) {
781+ const prepared = await prepareOpenApiSpec(fixture.url, fixture.name, {
782+ includeDts: false,
783+ profile: "full",
784+ });
785+ const tools = buildOpenApiToolsFromPrepared(
786+ {
787+ type: "openapi",
788+ name: fixture.name,
789+ spec: fixture.url,
790+ baseUrl: prepared.servers[0] || fixture.fallbackBaseUrl,
791+ },
792+ prepared,
793+ );
794+
795+ const issues: string[] = [];
796+
797+ const validateArrayItems = (schema: Record<string, unknown>, location: string) => {
798+ const items = toRecord(schema.items);
799+ const itemKeys = Object.keys(items);
800+ if (itemKeys.length === 0) {
801+ issues.push(`${location}.items has empty items schema`);
802+ return;
803+ }
804+
805+ const isRefOnly = typeof items.$ref === "string" && itemKeys.every((key) => key === "$ref");
806+ if (isRefOnly) {
807+ issues.push(`${location}.items is ref-only (${String(items.$ref)})`);
808+ }
809+ };
810+
811+ const visitTopLevelSchema = (schemaValue: unknown, location: string) => {
812+ const schema = toRecord(schemaValue);
813+ if (Object.keys(schema).length === 0) return;
814+
815+ if (schema.type === "array") {
816+ validateArrayItems(schema, `${location}:schema`);
817+ }
818+ };
819+
820+ for (const tool of tools) {
821+ visitTopLevelSchema(tool.typing?.inputSchema, `${tool.path}:input`);
822+ visitTopLevelSchema(tool.typing?.outputSchema, `${tool.path}:output`);
823+ }
824+
825+ expect(
826+ issues,
827+ `${fixture.name} has array item schema regressions:\n${issues.slice(0, 20).join("\n")}`,
828+ ).toHaveLength(0);
829+ }
830+ },
831+ 300_000,
832+ );
833+
699834 test(
700835 "stripe: extraction keeps concrete schemas for forwarding + verification report list outputs",
701836 async () => {
@@ -1039,4 +1174,109 @@ describe("real-world OpenAPI specs", () => { · 337 unmodified lines
10391174 },
10401175 1_800_000,
10411176 );
1177+
1178+ test(
1179+ "full profile schemas validate every github/stripe operation against parser-derived expectations",
1180+ async () => {
1181+ const fixtures = SPECS.filter((fixture) => fixture.name === "github" || fixture.name === "stripe");
1182+ expect(fixtures.length).toBe(2);
1183+
1184+ for (const fixture of fixtures) {
1185+ const parsed = await SwaggerParser.parse(fixture.url);
1186+ const parsedSpec = parsed as Record<string, unknown>;
1187+ const componentSchemas = toRecord(toRecord(parsedSpec.components).schemas);
1188+ const operationExpectations = collectOperationTypeExpectations(parsedSpec);
1189+ expect(operationExpectations.size).toBeGreaterThan(0);
1190+
1191+ const prepared = await prepareOpenApiSpec(parsedSpec, fixture.name, {
1192+ includeDts: false,
1193+ profile: "full",
1194+ });
1195+
1196+ const tools = buildOpenApiToolsFromPrepared(
1197+ {
1198+ type: "openapi",
1199+ name: fixture.name,
1200+ spec: fixture.url,
1201+ baseUrl: prepared.servers[0] || `https://${fixture.name}.example.com`,
1202+ },
1203+ prepared,
1204+ );
1205+
1206+ const toolsByOperationId = new Map<string, (typeof tools)[number]>();
1207+ for (const tool of tools) {
1208+ const operationId = tool.typing?.typedRef?.kind === "openapi_operation"
1209+ ? tool.typing.typedRef.operationId
1210+ : "";
1211+ if (!operationId || toolsByOperationId.has(operationId)) continue;
1212+ toolsByOperationId.set(operationId, tool);
1213+ }
1214+
1215+ expect(toolsByOperationId.size).toBe(operationExpectations.size);
1216+
1217+ const issues: string[] = [];
1218+
1219+ for (const [operationId, expectations] of operationExpectations.entries()) {
1220+ const tool = toolsByOperationId.get(operationId);
1221+ if (!tool) {
1222+ issues.push(`missing tool for operationId ${operationId}`);
1223+ continue;
1224+ }
1225+
1226+ const inputSchema = toRecord(tool.typing?.inputSchema);
1227+ const outputSchema = toRecord(tool.typing?.outputSchema);
1228+
1229+ if (expectations.expectsInput && Object.keys(inputSchema).length === 0) {
1230+ issues.push(`${tool.path}: expected input schema but got empty`);
1231+ }
1232+
1233+ if (expectations.expectsKnownOutput && Object.keys(outputSchema).length === 0) {
1234+ issues.push(`${tool.path}: expected output schema but got empty`);
1235+ }
1236+
1237+ if (expectations.expectsVoidOutput && Object.keys(outputSchema).length > 0) {
1238+ issues.push(`${tool.path}: expected void output but schema was present`);
1239+ }
1240+
1241+ if (outputSchema.type === "array") {
1242+ const items = toRecord(outputSchema.items);
1243+ if (Object.keys(items).length === 0) {
1244+ issues.push(`${tool.path}: output array has empty items schema`);
1245+ } else if (isSchemaRefOnly(items)) {
1246+ issues.push(`${tool.path}: output array items are ref-only (${String(items.$ref)})`);
1247+ }
1248+ }
1249+
1250+ for (const expected of expectations.expectedInputFields) {
1251+ const resolved = resolveExpectedInputPrimitiveFromGeneratedSchema(inputSchema, expected.name, componentSchemas);
1252+ if (!resolved) {
1253+ issues.push(`${tool.path}: missing input field ${expected.name}`);
1254+ continue;
1255+ }
1256+ if (resolved !== expected.type) {
1257+ issues.push(`${tool.path}: input field ${expected.name} expected ${expected.type}, got ${resolved}`);
1258+ }
1259+ }
1260+
1261+ const outputProperties = toRecord(outputSchema.properties);
1262+ for (const expected of expectations.expectedOutputFields) {
1263+ const resolved = resolveSchemaPrimitiveType(outputProperties[expected.name], componentSchemas);
1264+ if (!resolved) {
1265+ issues.push(`${tool.path}: missing output field ${expected.name}`);
1266+ continue;
1267+ }
1268+ if (resolved !== expected.type) {
1269+ issues.push(`${tool.path}: output field ${expected.name} expected ${expected.type}, got ${resolved}`);
1270+ }
1271+ }
1272+ }
1273+
1274+ expect(
1275+ issues,
1276+ `${fixture.name} full-profile schema regressions:\n${issues.slice(0, 30).join("\n")}`,
1277+ ).toHaveLength(0);
1278+ }
1279+ },
1280+ 1_800_000,
1281+ );
10421282 });
executor/packages/database/src/runtime/tool_source_loading.ts+103−13
@@ -30,6 +30,7 @@ import { · 29 unmodified lines
3030 const OPENAPI_SPEC_CACHE_TTL_MS = 5 * 60 * 60_000;
3131 const OPENAPI_PREPARE_MAX_ATTEMPTS = 3;
3232 const OPENAPI_PREPARE_RETRY_BASE_DELAY_MS = 1_500;
33+const OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS = 180_000;
3334
3435 const OPENAPI_PREPARED_CACHE_VERSION = "openapi_v11";
3536 const OPENAPI_ARTIFACT_CACHE_VERSION = "openapi_artifact_v1";
@@ -80,6 +81,90 @@ const preparedOpenApiEnvelopeSchema = z.object({ · 44 unmodified lines
8081 sizeBytes: z.number().optional(),
8182 });
8283
84+const externalGenerateResponseSchema = z.object({
85+ status: z.enum(["ready", "failed"]).optional(),
86+ prepared: z.unknown().optional(),
87+ error: z.string().optional(),
88+});
89+
90+function trimEnv(value: string | undefined): string | undefined {
91+ const trimmed = value?.trim();
92+ return trimmed && trimmed.length > 0 ? trimmed : undefined;
93+}
94+
95+function resolveExternalGenerateEndpoint(): string | undefined {
96+ const base = trimEnv(process.env.EXECUTOR_GENERATE_URL);
97+ if (!base) {
98+ return undefined;
99+ }
100+
101+ return base.endsWith("/api/generate") ? base : `${base.replace(/\/$/, "")}/api/generate`;
102+}
103+
104+async function loadPreparedOpenApiSpecFromExternalGenerate(
105+ specUrl: string,
106+ sourceName: string,
107+ includeDts: boolean,
108+): Promise<PreparedOpenApiSpec | null> {
109+ const endpoint = resolveExternalGenerateEndpoint();
110+ if (!endpoint) {
111+ return null;
112+ }
113+
114+ const controller = new AbortController();
115+ const timeout = setTimeout(() => controller.abort(), OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS);
116+
117+ try {
118+ const response = await fetch(endpoint, {
119+ method: "POST",
120+ headers: {
121+ "content-type": "application/json",
122+ },
123+ body: JSON.stringify({ specUrl, sourceName, includeDts }),
124+ cache: "no-store",
125+ signal: controller.signal,
126+ });
127+
128+ const payload = externalGenerateResponseSchema.safeParse(await response.json().catch(() => ({})));
129+ if (!payload.success) {
130+ return null;
131+ }
132+
133+ if (!response.ok || payload.data.status === "failed") {
134+ return null;
135+ }
136+
137+ return toPreparedOpenApiSpec(payload.data.prepared);
138+ } catch {
139+ return null;
140+ } finally {
141+ clearTimeout(timeout);
142+ }
143+}
144+
145+async function cachePreparedOpenApiSpec(
146+ ctx: ActionCtx,
147+ specUrl: string,
148+ sourceName: string,
149+ prepared: PreparedOpenApiSpec,
150+ preparedStorageId?: Id<"_storage">,
151+ preparedSizeBytes?: number,
152+): Promise<void> {
153+ try {
154+ const json = JSON.stringify(prepared);
155+ const storageId = preparedStorageId ?? await ctx.storage.store(new Blob([json], { type: "application/json" }));
156+ await ctx.runMutation(internal.openApiSpecCache.putEntry, {
157+ specUrl,
158+ version: OPENAPI_PREPARED_CACHE_VERSION,
159+ storageId,
160+ sizeBytes: preparedSizeBytes ?? json.length,
161+ });
162+ } catch (error) {
163+ const message = error instanceof Error ? error.message : String(error);
164+ console.warn(`[executor] OpenAPI cache write failed for '${sourceName}': ${message}`);
165+ }
166+}
167+
83168 function stableStringify(value: unknown): string {
84169 if (Array.isArray(value)) {
85170 return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
@@ -459,6 +544,16 @@ async function loadCachedOpenApiSpec( · 373 unmodified lines
459544 console.warn(`[executor] OpenAPI cache read failed for '${sourceName}': ${message}`);
460545 }
461546
547+ const externalPrepared = await loadPreparedOpenApiSpecFromExternalGenerate(
548+ specUrl,
549+ sourceName,
550+ includeDts,
551+ );
552+ if (externalPrepared) {
553+ await cachePreparedOpenApiSpec(ctx, specUrl, sourceName, externalPrepared);
554+ return externalPrepared;
555+ }
556+
462557 let preparedResponse: unknown = undefined;
463558 let lastPrepareError: string | undefined;
464559 for (let attempt = 1; attempt <= OPENAPI_PREPARE_MAX_ATTEMPTS; attempt += 1) {
@@ -520,19 +615,14 @@ async function loadCachedOpenApiSpec( · 55 unmodified lines
520615 throw new Error(`Prepared OpenAPI payload for '${sourceName}' was invalid`);
521616 }
522617
523− try {
524− const json = JSON.stringify(prepared);
525− const storageId = preparedStorageId ?? await ctx.storage.store(new Blob([json], { type: "application/json" }));
526− await ctx.runMutation(internal.openApiSpecCache.putEntry, {
527− specUrl,
528− version: OPENAPI_PREPARED_CACHE_VERSION,
529− storageId,
530− sizeBytes: preparedSizeBytes ?? json.length,
531− });
532− } catch (error) {
533− const message = error instanceof Error ? error.message : String(error);
534− console.warn(`[executor] OpenAPI cache write failed for '${sourceName}': ${message}`);
535− }
618+ await cachePreparedOpenApiSpec(
619+ ctx,
620+ specUrl,
621+ sourceName,
622+ prepared,
623+ preparedStorageId,
624+ preparedSizeBytes,
625+ );
536626
537627 return prepared;
538628 }