History

use same genreate route on frontend and cache

Rhys Sullivan committed 8 months ago

Commit 25cd57a1 parent d1160e65 files · +233 −291Browse files

Unified · Split

5 files
  1. .env.example +1 −1
  2. executor/apps/web/src/app/api/generate/route.ts +165 −17
  3. executor/apps/web/src/app/api/openapi/inspect/route.ts +0 −237
  4. executor/apps/web/src/lib/openapi/spec-inspector.ts +60 −29
  5. executor/packages/database/src/runtime/tool_source_loading.ts +7 −7
.env.example+1−1
@@ -18,7 +18,7 @@ CONVEX_SITE_URL=https://your-deployment.convex.site · 17 unmodified lines
1818 # Leave unset in production to default to Cloudflare runtime.
1919 # DANGEROUSLY_ALLOW_LOCAL_VM=1
2020 # Optional external OpenAPI generator endpoint used by Convex for heavy parse/prepare.
21−# Expects POST /api/generate with { specUrl, sourceName, includeDts }.
21+# Expects GET /api/generate?specUrl=...&sourceName=...&includeDts=0|1.
2222 # EXECUTOR_GENERATE_URL=http://localhost:4312
2323
2424 # ── Agent storage backend (optional) ──
executor/apps/web/src/app/api/generate/route.ts+165−17
@@ -1,13 +1,46 @@
1+import { parse as parseYaml } from "yaml";
12 import { z } from "zod";
3+import { unstable_cache } from "next/cache";
24 import { prepareOpenApiSpec } from "@executor/core/openapi-prepare";
35
4−const requestSchema = z.object({
6+const GENERATED_SPEC_REVALIDATE_SECONDS = 60 * 60;
7+const HEADER_FETCH_TIMEOUT_MS = 12_000;
8+
9+const querySchema = z.object({
510 specUrl: z.string().trim().min(1),
6− sourceName: z.string().trim().min(1),
7− includeDts: z.boolean().optional(),
11+ sourceName: z.string().trim().min(1).optional(),
12+ includeDts: z.enum(["1", "true", "0", "false"]).optional(),
13+ headers: z.string().optional(),
814 });
915
10−function noStoreJson(payload: unknown, status: number): Response {
16+const recordSchema = z.record(z.string(), z.unknown());
17+
18+const blockedForwardedHeaderNames = new Set([
19+ "accept",
20+ "accept-encoding",
21+ "connection",
22+ "content-length",
23+ "content-type",
24+ "host",
25+ "origin",
26+ "referer",
27+]);
28+
29+const generatePreparedSpecCached = unstable_cache(
30+ async (specUrl: string, includeDts: boolean) => {
31+ return await prepareOpenApiSpec(specUrl, "openapi", {
32+ includeDts,
33+ profile: "inventory",
34+ });
35+ },
36+ ["openapi-generate-v1"],
37+ { revalidate: GENERATED_SPEC_REVALIDATE_SECONDS },
38+);
39+
40+function jsonResponse(
41+ payload: unknown,
42+ status: number,
43+): Response {
1144 return Response.json(payload, {
1245 status,
1346 headers: {
@@ -16,28 +49,143 @@ function noStoreJson(payload: unknown, status: number): Response { · 2 unmodified lines
1649 });
1750 }
1851
19−export async function POST(request: Request): Promise<Response> {
20− const parsed = requestSchema.safeParse(await request.json().catch(() => null));
21− if (!parsed.success) {
22− return noStoreJson({ error: "Invalid generate request" }, 400);
52+function sanitizeForwardHeaders(headers: Record<string, string> | undefined): Record<string, string> {
53+ if (!headers) {
54+ return {};
2355 }
2456
57+ const nextHeaders: Record<string, string> = {};
58+ for (const [rawKey, rawValue] of Object.entries(headers)) {
59+ const key = rawKey.trim();
60+ const value = rawValue.trim();
61+ if (!key || !value) {
62+ continue;
63+ }
64+ if (blockedForwardedHeaderNames.has(key.toLowerCase())) {
65+ continue;
66+ }
67+ nextHeaders[key] = value;
68+ }
69+
70+ return nextHeaders;
71+}
72+
73+function parseHeadersQueryValue(value: string | undefined): Record<string, string> {
74+ if (!value) {
75+ return {};
76+ }
77+
78+ let parsed: unknown;
2579 try {
26− const prepared = await prepareOpenApiSpec(
27− parsed.data.specUrl,
28− parsed.data.sourceName,
29− {
30− includeDts: parsed.data.includeDts ?? false,
31− profile: "inventory",
80+ parsed = JSON.parse(value);
81+ } catch {
82+ throw new Error("Invalid headers query param");
83+ }
84+
85+ const asRecord = z.record(z.string(), z.string()).safeParse(parsed);
86+ if (!asRecord.success) {
87+ throw new Error("Invalid headers query param");
88+ }
89+
90+ return asRecord.data;
91+}
92+
93+function parseSpecPayload(raw: string): Record<string, unknown> {
94+ const jsonCandidate = (() => {
95+ try {
96+ return JSON.parse(raw);
97+ } catch {
98+ return null;
99+ }
100+ })();
101+
102+ const yamlCandidate = jsonCandidate === null
103+ ? (() => {
104+ try {
105+ return parseYaml(raw);
106+ } catch {
107+ return null;
108+ }
109+ })()
110+ : null;
111+
112+ const parsed = jsonCandidate ?? yamlCandidate;
113+ const parsedRecord = recordSchema.safeParse(parsed);
114+ if (!parsedRecord.success) {
115+ throw new Error("Spec payload is empty or not an object");
116+ }
117+
118+ return parsedRecord.data;
119+}
120+
121+async function prepareWithForwardedHeaders(
122+ specUrl: string,
123+ sourceName: string,
124+ includeDts: boolean,
125+ headers: Record<string, string>,
126+): Promise<Awaited<ReturnType<typeof prepareOpenApiSpec>>> {
127+ const controller = new AbortController();
128+ const timeoutId = setTimeout(() => controller.abort(), HEADER_FETCH_TIMEOUT_MS);
129+
130+ try {
131+ const response = await fetch(specUrl, {
132+ method: "GET",
133+ headers: {
134+ Accept: "application/json, application/yaml, text/yaml, text/plain;q=0.9, */*;q=0.8",
135+ ...headers,
32136 },
33− );
137+ signal: controller.signal,
138+ cache: "no-store",
139+ });
140+
141+ if (!response.ok) {
142+ throw new Error(`Failed to fetch spec (${response.status} ${response.statusText})`);
143+ }
144+
145+ const raw = await response.text();
146+ const parsedSpec = parseSpecPayload(raw);
147+ return await prepareOpenApiSpec(parsedSpec, sourceName, {
148+ includeDts,
149+ profile: "inventory",
150+ });
151+ } finally {
152+ clearTimeout(timeoutId);
153+ }
154+}
155+
156+export async function GET(request: Request): Promise<Response> {
157+ const url = new URL(request.url);
158+ const parsed = querySchema.safeParse({
159+ specUrl: url.searchParams.get("specUrl") ?? "",
160+ sourceName: url.searchParams.get("sourceName") ?? undefined,
161+ includeDts: url.searchParams.get("includeDts") ?? undefined,
162+ headers: url.searchParams.get("headers") ?? undefined,
163+ });
164+
165+ if (!parsed.success) {
166+ return jsonResponse({ error: "Invalid generate request" }, 400);
167+ }
168+
169+ try {
170+ const includeDts = parsed.data.includeDts === "1" || parsed.data.includeDts === "true";
171+ const sourceName = parsed.data.sourceName ?? "openapi";
172+ const forwardedHeaders = sanitizeForwardHeaders(parseHeadersQueryValue(parsed.data.headers));
173+
174+ const prepared = Object.keys(forwardedHeaders).length === 0
175+ ? await generatePreparedSpecCached(parsed.data.specUrl, includeDts)
176+ : await prepareWithForwardedHeaders(
177+ parsed.data.specUrl,
178+ sourceName,
179+ includeDts,
180+ forwardedHeaders,
181+ );
34182
35− return noStoreJson({
183+ return jsonResponse({
36184 status: "ready",
37185 prepared,
38186 }, 200);
39187 } catch (error) {
40− return noStoreJson({
188+ return jsonResponse({
41189 status: "failed",
42190 error: error instanceof Error ? error.message : "Failed to generate",
43191 }, 502);
executor/apps/web/src/app/api/openapi/inspect/route.ts+0−237
@@ -1,237 +0,0 @@
1−import { isIP } from "node:net";
2−import { parse as parseDomain } from "tldts";
3−import { z } from "zod";
4−import { isAbortError } from "@/lib/error-utils";
5−import { inspectOpenApiPayload } from "@/lib/openapi/spec-inspector";
6−
7−const INSPECTION_TIMEOUT_MS = 12_000;
8−
9−const requestSchema = z.object({
10− specUrl: z.string().trim().min(1),
11− headers: z.record(z.string(), z.string()).optional(),
12−});
13−
14−const errorDetailObjectSchema = z.record(z.string(), z.unknown());
15−
16−const blockedForwardedHeaderNames = new Set([
17− "accept",
18− "accept-encoding",
19− "connection",
20− "content-length",
21− "content-type",
22− "host",
23− "origin",
24− "referer",
25−]);
26−
27−function jsonResponse(
28− payload: unknown,
29− status: number,
30− options?: { cacheable?: boolean },
31−): Response {
32− const cacheable = options?.cacheable === true;
33− return Response.json(payload, {
34− status,
35− headers: {
36− "cache-control": cacheable
37− ? "public, s-maxage=3600, stale-while-revalidate=86400"
38− : "no-store",
39− },
40− });
41−}
42−
43−function isLocalHost(hostname: string): boolean {
44− return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1";
45−}
46−
47−function normalizeIpHost(hostname: string): string {
48− if (hostname.startsWith("[") && hostname.endsWith("]")) {
49− return hostname.slice(1, -1);
50− }
51− return hostname;
52−}
53−
54−function isPrivateIpv4(hostname: string): boolean {
55− const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
56− if (parts.length !== 4 || parts.some((part) => Number.isNaN(part))) {
57− return true;
58− }
59− const [a, b] = parts;
60− if (a === 10) return true;
61− if (a === 127) return true;
62− if (a === 169 && b === 254) return true;
63− if (a === 172 && b >= 16 && b <= 31) return true;
64− if (a === 192 && b === 168) return true;
65− if (a === 100 && b >= 64 && b <= 127) return true;
66− if (a === 0) return true;
67− if (a >= 224) return true;
68− return false;
69−}
70−
71−function isPrivateIpv6(hostname: string): boolean {
72− const value = hostname.toLowerCase();
73− if (value === "::1") return true;
74− if (value.startsWith("fc") || value.startsWith("fd")) return true;
75− if (value.startsWith("fe8") || value.startsWith("fe9") || value.startsWith("fea") || value.startsWith("feb")) {
76− return true;
77− }
78−
79− if (value.startsWith("::ffff:")) {
80− const mapped = value.slice("::ffff:".length);
81− return isPrivateIpv4(mapped);
82− }
83−
84− return false;
85−}
86−
87−function isPrivateIp(hostname: string): boolean {
88− const normalized = normalizeIpHost(hostname);
89− const family = isIP(normalized);
90− if (family === 4) {
91− return isPrivateIpv4(normalized);
92− }
93− if (family === 6) {
94− return isPrivateIpv6(normalized);
95− }
96− return false;
97−}
98−
99−function isPublicDnsHostname(hostname: string): boolean {
100− const parsed = parseDomain(hostname);
101− return Boolean(parsed.domain && parsed.publicSuffix);
102−}
103−
104−function sanitizeForwardHeaders(headers: Record<string, string> | undefined): Record<string, string> {
105− if (!headers) {
106− return {};
107− }
108−
109− const nextHeaders: Record<string, string> = {};
110− for (const [rawKey, rawValue] of Object.entries(headers)) {
111− const key = rawKey.trim();
112− const value = rawValue.trim();
113− if (!key || !value) {
114− continue;
115− }
116− if (blockedForwardedHeaderNames.has(key.toLowerCase())) {
117− continue;
118− }
119− nextHeaders[key] = value;
120− }
121−
122− return nextHeaders;
123−}
124−
125−function extractErrorDetail(text: string): string | undefined {
126− const trimmed = text.trim();
127− if (!trimmed) {
128− return undefined;
129− }
130−
131− try {
132− const parsed = errorDetailObjectSchema.safeParse(JSON.parse(trimmed));
133− if (parsed.success) {
134− const candidates = ["message", "error", "detail", "title", "description"];
135− for (const key of candidates) {
136− const value = parsed.data[key];
137− if (typeof value === "string" && value.trim().length > 0) {
138− return value.trim().slice(0, 300);
139− }
140− }
141− }
142− } catch {
143− // Fall through to plain-text detail.
144− }
145−
146− return trimmed.replace(/\s+/g, " ").slice(0, 300);
147−}
148−
149−export async function POST(request: Request): Promise<Response> {
150− const parsedRequest = requestSchema.safeParse(await request.json().catch(() => null));
151− if (!parsedRequest.success) {
152− return jsonResponse({ detail: "Invalid OpenAPI inspection request" }, 400);
153− }
154−
155− let parsedUrl: URL;
156− try {
157− parsedUrl = new URL(parsedRequest.data.specUrl);
158− } catch {
159− return jsonResponse({ detail: "Spec URL is invalid" }, 400);
160− }
161−
162− const protocol = parsedUrl.protocol.toLowerCase();
163− const hostname = parsedUrl.hostname.toLowerCase();
164− const localAllowed = process.env.NODE_ENV !== "production";
165−
166− if (parsedUrl.username || parsedUrl.password) {
167− return jsonResponse({ detail: "Credentials in spec URL are not allowed" }, 400);
168− }
169−
170− if (protocol !== "https:" && protocol !== "http:") {
171− return jsonResponse({ detail: "Spec URL must use https:// (or http:// for localhost in dev)" }, 400);
172− }
173−
174− if (protocol === "http:" && !(localAllowed && isLocalHost(hostname))) {
175− return jsonResponse({ detail: "Spec URL must use https://" }, 400);
176− }
177−
178− if (isPrivateIp(hostname) && !(localAllowed && isLocalHost(hostname))) {
179− return jsonResponse({ detail: "Private or local hosts are not allowed" }, 400);
180− }
181−
182− if (!isIP(normalizeIpHost(hostname)) && !isPublicDnsHostname(hostname)) {
183− return jsonResponse({ detail: "Spec host must be a public DNS host" }, 400);
184− }
185−
186− const forwardedHeaders = sanitizeForwardHeaders(parsedRequest.data.headers);
187− const cacheableRequest = Object.keys(forwardedHeaders).length === 0;
188−
189− const controller = new AbortController();
190− const timeoutId = setTimeout(() => controller.abort(), INSPECTION_TIMEOUT_MS);
191−
192− try {
193− const response = await fetch(parsedUrl.toString(), {
194− method: "GET",
195− headers: {
196− Accept: "application/json, application/yaml, text/yaml, text/plain;q=0.9, */*;q=0.8",
197− ...forwardedHeaders,
198− },
199− signal: controller.signal,
200− cache: cacheableRequest ? "force-cache" : "no-store",
201− ...(cacheableRequest ? { next: { revalidate: 3600 } } : {}),
202− });
203−
204− if (!response.ok) {
205− const detail = extractErrorDetail(await response.text().catch(() => ""));
206− return jsonResponse({
207− status: response.status,
208− statusText: response.statusText,
209− detail,
210− }, response.status);
211− }
212−
213− const raw = await response.text();
214− const contentType = response.headers.get("content-type") ?? "";
215− const inspected = inspectOpenApiPayload({
216− raw,
217− sourceUrl: parsedUrl.toString(),
218− contentType,
219− });
220−
221− return jsonResponse({
222− status: response.status,
223− statusText: response.statusText,
224− spec: inspected.spec,
225− inferredAuth: inspected.inferredAuth,
226− }, 200, { cacheable: cacheableRequest });
227− } catch (error) {
228− if (isAbortError(error)) {
229− return jsonResponse({ detail: "Request timed out while fetching spec" }, 504);
230− }
231−
232− const detail = error instanceof Error ? error.message : "Failed to fetch spec";
233− return jsonResponse({ detail }, 502);
234− } finally {
235− clearTimeout(timeoutId);
236− }
237−}
executor/apps/web/src/lib/openapi/spec-inspector.ts+60−29
@@ -10,14 +10,29 @@ type OpenApiInspectionResult = { · 9 unmodified lines
1010 inferredAuth: InferredSpecAuth;
1111 };
1212
13−const inferredSpecAuthSchema = z.object({
13+export type InferredSpecAuth = {
14+ type: "none" | "bearer" | "apiKey" | "basic" | "mixed";
15+ mode?: "workspace" | "account" | "organization";
16+ header?: string;
17+ inferred: true;
18+};
19+
20+const generatedPreparedInferredAuthSchema = z.object({
1421 type: z.enum(["none", "bearer", "apiKey", "basic", "mixed"]),
1522 mode: z.enum(["workspace", "account", "organization"]).optional(),
1623 header: z.string().optional(),
17− inferred: z.literal(true),
24+}).optional();
25+
26+const generatedPreparedSpecSchema = z.object({
27+ servers: z.array(z.string()).optional(),
28+ inferredAuth: generatedPreparedInferredAuthSchema,
1829 });
1930
20−export type InferredSpecAuth = z.infer<typeof inferredSpecAuthSchema>;
31+const generatedResponseSchema = z.object({
32+ status: z.enum(["ready", "failed"]).optional(),
33+ prepared: z.unknown().optional(),
34+ error: z.string().optional(),
35+});
2136
2237 const recordSchema = z.record(z.string(), z.unknown());
2338
@@ -198,16 +213,22 @@ export async function fetchAndInspectOpenApiSpec(input: { · 174 unmodified lines
198213 headers?: Record<string, string>;
199214 signal?: AbortSignal;
200215 }): Promise<OpenApiInspectionResult> {
201− const response = await fetch("/api/openapi/inspect", {
202− method: "POST",
216+ const hasHeaders = Object.keys(input.headers ?? {}).length > 0;
217+ const params = new URLSearchParams({
218+ specUrl: input.specUrl,
219+ sourceName: "openapi-inspect",
220+ includeDts: "0",
221+ });
222+
223+ if (hasHeaders) {
224+ params.set("headers", JSON.stringify(input.headers ?? {}));
225+ }
226+
227+ const response = await fetch(`/api/generate?${params.toString()}`, {
228+ method: "GET",
203229 headers: {
204− "content-type": "application/json",
205230 Accept: "application/json",
206231 },
207− body: JSON.stringify({
208− specUrl: input.specUrl,
209− headers: input.headers ?? {},
210− }),
211232 signal: input.signal,
212233 cache: "no-store",
213234 });
@@ -220,31 +241,41 @@ export async function fetchAndInspectOpenApiSpec(input: { · 6 unmodified lines
220241 }
221242
222243 if (!response.ok) {
223− const payloadRecord = recordSchema.safeParse(payload);
224− const detail = payloadRecord.success && typeof payloadRecord.data.detail === "string"
225− ? payloadRecord.data.detail
244+ const generatedPayload = generatedResponseSchema.safeParse(payload);
245+ const detail = generatedPayload.success
246+ ? (generatedPayload.data.error ?? "")
226247 : "";
227− const status = payloadRecord.success && typeof payloadRecord.data.status === "number"
228− ? payloadRecord.data.status
229− : response.status;
230− const statusText = payloadRecord.success && typeof payloadRecord.data.statusText === "string"
231− ? payloadRecord.data.statusText
232− : response.statusText;
233− throw new Error(createSpecFetchErrorMessage({ status, statusText, detail }));
248+ throw new Error(createSpecFetchErrorMessage({ status: response.status, statusText: response.statusText, detail }));
234249 }
235250
236− const payloadRecord = recordSchema.safeParse(payload);
237− if (!payloadRecord.success) {
238− throw new Error("Spec inspection returned an invalid response");
251+ const generatedPayload = generatedResponseSchema.safeParse(payload);
252+ if (!generatedPayload.success || generatedPayload.data.status === "failed") {
253+ throw new Error("Spec generation returned an invalid response");
239254 }
240255
241− const parsedSpec = recordSchema.safeParse(payloadRecord.data.spec);
242− if (!parsedSpec.success) {
243− throw new Error("Spec inspection did not return a valid OpenAPI document");
256+ const preparedParsed = generatedPreparedSpecSchema.safeParse(generatedPayload.data.prepared);
257+ if (!preparedParsed.success) {
258+ throw new Error("Spec generation did not return a valid prepared document");
244259 }
245260
246− const parsedAuth = inferredSpecAuthSchema.safeParse(payloadRecord.data.inferredAuth);
247− const inferredAuth = parsedAuth.success ? parsedAuth.data : inferSecuritySchemaAuth(parsedSpec.data);
261+ const servers = (preparedParsed.data.servers ?? [])
262+ .filter((server): server is string => typeof server === "string" && server.trim().length > 0)
263+ .map((url) => ({ url }));
264+
265+ const generatedInferredAuth = preparedParsed.data.inferredAuth;
266+ const inferredAuth: InferredSpecAuth = generatedInferredAuth
267+ ? {
268+ type: generatedInferredAuth.type,
269+ ...(generatedInferredAuth.mode ? { mode: generatedInferredAuth.mode } : {}),
270+ ...(generatedInferredAuth.type === "apiKey" && generatedInferredAuth.header
271+ ? { header: generatedInferredAuth.header }
272+ : {}),
273+ inferred: true,
274+ }
275+ : { type: "none", inferred: true };
248276
249− return { spec: parsedSpec.data, inferredAuth };
277+ return {
278+ spec: { servers },
279+ inferredAuth,
280+ };
250281 }
executor/packages/database/src/runtime/tool_source_loading.ts+7−7
@@ -115,13 +115,13 @@ async function loadPreparedOpenApiSpecFromExternalGenerate( · 114 unmodified lines
115115 const timeout = setTimeout(() => controller.abort(), OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS);
116116
117117 try {
118− const response = await fetch(endpoint, {
119− method: "POST",
120− headers: {
121− "content-type": "application/json",
122− },
123− body: JSON.stringify({ specUrl, sourceName, includeDts }),
124− cache: "no-store",
118+ const url = new URL(endpoint);
119+ url.searchParams.set("specUrl", specUrl);
120+ url.searchParams.set("sourceName", sourceName);
121+ url.searchParams.set("includeDts", includeDts ? "1" : "0");
122+
123+ const response = await fetch(url.toString(), {
124+ method: "GET",
125125 signal: controller.signal,
126126 });
127127

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t