History

use same genreate route on frontend and cache

Rhys Sullivan committed 8 months ago

Commit 25cd57a1 parent d1160e65 files · +233 −291Browse files

Unified · Split

5 files
  1. .env.example +1 −1
  2. executor/apps/web/src/app/api/generate/route.ts +165 −17
  3. executor/apps/web/src/app/api/openapi/inspect/route.ts +0 −237
  4. executor/apps/web/src/lib/openapi/spec-inspector.ts +60 −29
  5. executor/packages/database/src/runtime/tool_source_loading.ts +7 −7
.env.example+1−1
@@ -18,7 +18,7 @@ CONVEX_SITE_URL=https://your-deployment.convex.site · 17 unmodified lines
18# Leave unset in production to default to Cloudflare runtime.18# Leave unset in production to default to Cloudflare runtime.
19# DANGEROUSLY_ALLOW_LOCAL_VM=119# DANGEROUSLY_ALLOW_LOCAL_VM=1
20# Optional external OpenAPI generator endpoint used by Convex for heavy parse/prepare.20# Optional external OpenAPI generator endpoint used by Convex for heavy parse/prepare.
21# Expects POST /api/generate with { specUrl, sourceName, includeDts }.21# Expects GET /api/generate?specUrl=...&sourceName=...&includeDts=0|1.
22# EXECUTOR_GENERATE_URL=http://localhost:431222# EXECUTOR_GENERATE_URL=http://localhost:4312
23 23
24# ── Agent storage backend (optional) ──24# ── Agent storage backend (optional) ──
executor/apps/web/src/app/api/generate/route.ts+165−17
@@ -1,13 +1,46 @@
1import { parse as parseYaml } from "yaml";
1import { z } from "zod";2import { z } from "zod";
3import { unstable_cache } from "next/cache";
2import { prepareOpenApiSpec } from "@executor/core/openapi-prepare";4import { prepareOpenApiSpec } from "@executor/core/openapi-prepare";
3 5
4const requestSchema = z.object({6const GENERATED_SPEC_REVALIDATE_SECONDS = 60 * 60;
7const HEADER_FETCH_TIMEOUT_MS = 12_000;
8
9const querySchema = z.object({
5 specUrl: z.string().trim().min(1),10 specUrl: z.string().trim().min(1),
6 sourceName: z.string().trim().min(1),11 sourceName: z.string().trim().min(1).optional(),
7 includeDts: z.boolean().optional(),12 includeDts: z.enum(["1", "true", "0", "false"]).optional(),
13 headers: z.string().optional(),
8});14});
9 15
10function noStoreJson(payload: unknown, status: number): Response {16const recordSchema = z.record(z.string(), z.unknown());
17
18const blockedForwardedHeaderNames = new Set([
19 "accept",
20 "accept-encoding",
21 "connection",
22 "content-length",
23 "content-type",
24 "host",
25 "origin",
26 "referer",
27]);
28
29const generatePreparedSpecCached = unstable_cache(
30 async (specUrl: string, includeDts: boolean) => {
31 return await prepareOpenApiSpec(specUrl, "openapi", {
32 includeDts,
33 profile: "inventory",
34 });
35 },
36 ["openapi-generate-v1"],
37 { revalidate: GENERATED_SPEC_REVALIDATE_SECONDS },
38);
39
40function jsonResponse(
41 payload: unknown,
42 status: number,
43): Response {
11 return Response.json(payload, {44 return Response.json(payload, {
12 status,45 status,
13 headers: {46 headers: {
@@ -16,28 +49,143 @@ function noStoreJson(payload: unknown, status: number): Response { · 2 unmodified lines
16 });49 });
17}50}
18 51
19export async function POST(request: Request): Promise<Response> {52function sanitizeForwardHeaders(headers: Record<string, string> | undefined): Record<string, string> {
20 const parsed = requestSchema.safeParse(await request.json().catch(() => null));53 if (!headers) {
21 if (!parsed.success) {54 return {};
22 return noStoreJson({ error: "Invalid generate request" }, 400);
23 }55 }
24 56
57 const nextHeaders: Record<string, string> = {};
58 for (const [rawKey, rawValue] of Object.entries(headers)) {
59 const key = rawKey.trim();
60 const value = rawValue.trim();
61 if (!key || !value) {
62 continue;
63 }
64 if (blockedForwardedHeaderNames.has(key.toLowerCase())) {
65 continue;
66 }
67 nextHeaders[key] = value;
68 }
69
70 return nextHeaders;
71}
72
73function parseHeadersQueryValue(value: string | undefined): Record<string, string> {
74 if (!value) {
75 return {};
76 }
77
78 let parsed: unknown;
25 try {79 try {
26 const prepared = await prepareOpenApiSpec(80 parsed = JSON.parse(value);
27 parsed.data.specUrl,81 } catch {
28 parsed.data.sourceName,82 throw new Error("Invalid headers query param");
29 {83 }
30 includeDts: parsed.data.includeDts ?? false,84
31 profile: "inventory",85 const asRecord = z.record(z.string(), z.string()).safeParse(parsed);
86 if (!asRecord.success) {
87 throw new Error("Invalid headers query param");
88 }
89
90 return asRecord.data;
91}
92
93function parseSpecPayload(raw: string): Record<string, unknown> {
94 const jsonCandidate = (() => {
95 try {
96 return JSON.parse(raw);
97 } catch {
98 return null;
99 }
100 })();
101
102 const yamlCandidate = jsonCandidate === null
103 ? (() => {
104 try {
105 return parseYaml(raw);
106 } catch {
107 return null;
108 }
109 })()
110 : null;
111
112 const parsed = jsonCandidate ?? yamlCandidate;
113 const parsedRecord = recordSchema.safeParse(parsed);
114 if (!parsedRecord.success) {
115 throw new Error("Spec payload is empty or not an object");
116 }
117
118 return parsedRecord.data;
119}
120
121async function prepareWithForwardedHeaders(
122 specUrl: string,
123 sourceName: string,
124 includeDts: boolean,
125 headers: Record<string, string>,
126): Promise<Awaited<ReturnType<typeof prepareOpenApiSpec>>> {
127 const controller = new AbortController();
128 const timeoutId = setTimeout(() => controller.abort(), HEADER_FETCH_TIMEOUT_MS);
129
130 try {
131 const response = await fetch(specUrl, {
132 method: "GET",
133 headers: {
134 Accept: "application/json, application/yaml, text/yaml, text/plain;q=0.9, */*;q=0.8",
135 ...headers,
32 },136 },
33 );137 signal: controller.signal,
138 cache: "no-store",
139 });
140
141 if (!response.ok) {
142 throw new Error(`Failed to fetch spec (${response.status} ${response.statusText})`);
143 }
144
145 const raw = await response.text();
146 const parsedSpec = parseSpecPayload(raw);
147 return await prepareOpenApiSpec(parsedSpec, sourceName, {
148 includeDts,
149 profile: "inventory",
150 });
151 } finally {
152 clearTimeout(timeoutId);
153 }
154}
155
156export async function GET(request: Request): Promise<Response> {
157 const url = new URL(request.url);
158 const parsed = querySchema.safeParse({
159 specUrl: url.searchParams.get("specUrl") ?? "",
160 sourceName: url.searchParams.get("sourceName") ?? undefined,
161 includeDts: url.searchParams.get("includeDts") ?? undefined,
162 headers: url.searchParams.get("headers") ?? undefined,
163 });
164
165 if (!parsed.success) {
166 return jsonResponse({ error: "Invalid generate request" }, 400);
167 }
168
169 try {
170 const includeDts = parsed.data.includeDts === "1" || parsed.data.includeDts === "true";
171 const sourceName = parsed.data.sourceName ?? "openapi";
172 const forwardedHeaders = sanitizeForwardHeaders(parseHeadersQueryValue(parsed.data.headers));
173
174 const prepared = Object.keys(forwardedHeaders).length === 0
175 ? await generatePreparedSpecCached(parsed.data.specUrl, includeDts)
176 : await prepareWithForwardedHeaders(
177 parsed.data.specUrl,
178 sourceName,
179 includeDts,
180 forwardedHeaders,
181 );
34 182
35 return noStoreJson({183 return jsonResponse({
36 status: "ready",184 status: "ready",
37 prepared,185 prepared,
38 }, 200);186 }, 200);
39 } catch (error) {187 } catch (error) {
40 return noStoreJson({188 return jsonResponse({
41 status: "failed",189 status: "failed",
42 error: error instanceof Error ? error.message : "Failed to generate",190 error: error instanceof Error ? error.message : "Failed to generate",
43 }, 502);191 }, 502);
executor/apps/web/src/app/api/openapi/inspect/route.ts+0−237
@@ -1,237 +0,0 @@
1import { isIP } from "node:net";
2import { parse as parseDomain } from "tldts";
3import { z } from "zod";
4import { isAbortError } from "@/lib/error-utils";
5import { inspectOpenApiPayload } from "@/lib/openapi/spec-inspector";
6
7const INSPECTION_TIMEOUT_MS = 12_000;
8
9const requestSchema = z.object({
10 specUrl: z.string().trim().min(1),
11 headers: z.record(z.string(), z.string()).optional(),
12});
13
14const errorDetailObjectSchema = z.record(z.string(), z.unknown());
15
16const blockedForwardedHeaderNames = new Set([
17 "accept",
18 "accept-encoding",
19 "connection",
20 "content-length",
21 "content-type",
22 "host",
23 "origin",
24 "referer",
25]);
26
27function jsonResponse(
28 payload: unknown,
29 status: number,
30 options?: { cacheable?: boolean },
31): Response {
32 const cacheable = options?.cacheable === true;
33 return Response.json(payload, {
34 status,
35 headers: {
36 "cache-control": cacheable
37 ? "public, s-maxage=3600, stale-while-revalidate=86400"
38 : "no-store",
39 },
40 });
41}
42
43function isLocalHost(hostname: string): boolean {
44 return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1";
45}
46
47function normalizeIpHost(hostname: string): string {
48 if (hostname.startsWith("[") && hostname.endsWith("]")) {
49 return hostname.slice(1, -1);
50 }
51 return hostname;
52}
53
54function isPrivateIpv4(hostname: string): boolean {
55 const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
56 if (parts.length !== 4 || parts.some((part) => Number.isNaN(part))) {
57 return true;
58 }
59 const [a, b] = parts;
60 if (a === 10) return true;
61 if (a === 127) return true;
62 if (a === 169 && b === 254) return true;
63 if (a === 172 && b >= 16 && b <= 31) return true;
64 if (a === 192 && b === 168) return true;
65 if (a === 100 && b >= 64 && b <= 127) return true;
66 if (a === 0) return true;
67 if (a >= 224) return true;
68 return false;
69}
70
71function isPrivateIpv6(hostname: string): boolean {
72 const value = hostname.toLowerCase();
73 if (value === "::1") return true;
74 if (value.startsWith("fc") || value.startsWith("fd")) return true;
75 if (value.startsWith("fe8") || value.startsWith("fe9") || value.startsWith("fea") || value.startsWith("feb")) {
76 return true;
77 }
78
79 if (value.startsWith("::ffff:")) {
80 const mapped = value.slice("::ffff:".length);
81 return isPrivateIpv4(mapped);
82 }
83
84 return false;
85}
86
87function isPrivateIp(hostname: string): boolean {
88 const normalized = normalizeIpHost(hostname);
89 const family = isIP(normalized);
90 if (family === 4) {
91 return isPrivateIpv4(normalized);
92 }
93 if (family === 6) {
94 return isPrivateIpv6(normalized);
95 }
96 return false;
97}
98
99function isPublicDnsHostname(hostname: string): boolean {
100 const parsed = parseDomain(hostname);
101 return Boolean(parsed.domain && parsed.publicSuffix);
102}
103
104function sanitizeForwardHeaders(headers: Record<string, string> | undefined): Record<string, string> {
105 if (!headers) {
106 return {};
107 }
108
109 const nextHeaders: Record<string, string> = {};
110 for (const [rawKey, rawValue] of Object.entries(headers)) {
111 const key = rawKey.trim();
112 const value = rawValue.trim();
113 if (!key || !value) {
114 continue;
115 }
116 if (blockedForwardedHeaderNames.has(key.toLowerCase())) {
117 continue;
118 }
119 nextHeaders[key] = value;
120 }
121
122 return nextHeaders;
123}
124
125function extractErrorDetail(text: string): string | undefined {
126 const trimmed = text.trim();
127 if (!trimmed) {
128 return undefined;
129 }
130
131 try {
132 const parsed = errorDetailObjectSchema.safeParse(JSON.parse(trimmed));
133 if (parsed.success) {
134 const candidates = ["message", "error", "detail", "title", "description"];
135 for (const key of candidates) {
136 const value = parsed.data[key];
137 if (typeof value === "string" && value.trim().length > 0) {
138 return value.trim().slice(0, 300);
139 }
140 }
141 }
142 } catch {
143 // Fall through to plain-text detail.
144 }
145
146 return trimmed.replace(/\s+/g, " ").slice(0, 300);
147}
148
149export async function POST(request: Request): Promise<Response> {
150 const parsedRequest = requestSchema.safeParse(await request.json().catch(() => null));
151 if (!parsedRequest.success) {
152 return jsonResponse({ detail: "Invalid OpenAPI inspection request" }, 400);
153 }
154
155 let parsedUrl: URL;
156 try {
157 parsedUrl = new URL(parsedRequest.data.specUrl);
158 } catch {
159 return jsonResponse({ detail: "Spec URL is invalid" }, 400);
160 }
161
162 const protocol = parsedUrl.protocol.toLowerCase();
163 const hostname = parsedUrl.hostname.toLowerCase();
164 const localAllowed = process.env.NODE_ENV !== "production";
165
166 if (parsedUrl.username || parsedUrl.password) {
167 return jsonResponse({ detail: "Credentials in spec URL are not allowed" }, 400);
168 }
169
170 if (protocol !== "https:" && protocol !== "http:") {
171 return jsonResponse({ detail: "Spec URL must use https:// (or http:// for localhost in dev)" }, 400);
172 }
173
174 if (protocol === "http:" && !(localAllowed && isLocalHost(hostname))) {
175 return jsonResponse({ detail: "Spec URL must use https://" }, 400);
176 }
177
178 if (isPrivateIp(hostname) && !(localAllowed && isLocalHost(hostname))) {
179 return jsonResponse({ detail: "Private or local hosts are not allowed" }, 400);
180 }
181
182 if (!isIP(normalizeIpHost(hostname)) && !isPublicDnsHostname(hostname)) {
183 return jsonResponse({ detail: "Spec host must be a public DNS host" }, 400);
184 }
185
186 const forwardedHeaders = sanitizeForwardHeaders(parsedRequest.data.headers);
187 const cacheableRequest = Object.keys(forwardedHeaders).length === 0;
188
189 const controller = new AbortController();
190 const timeoutId = setTimeout(() => controller.abort(), INSPECTION_TIMEOUT_MS);
191
192 try {
193 const response = await fetch(parsedUrl.toString(), {
194 method: "GET",
195 headers: {
196 Accept: "application/json, application/yaml, text/yaml, text/plain;q=0.9, */*;q=0.8",
197 ...forwardedHeaders,
198 },
199 signal: controller.signal,
200 cache: cacheableRequest ? "force-cache" : "no-store",
201 ...(cacheableRequest ? { next: { revalidate: 3600 } } : {}),
202 });
203
204 if (!response.ok) {
205 const detail = extractErrorDetail(await response.text().catch(() => ""));
206 return jsonResponse({
207 status: response.status,
208 statusText: response.statusText,
209 detail,
210 }, response.status);
211 }
212
213 const raw = await response.text();
214 const contentType = response.headers.get("content-type") ?? "";
215 const inspected = inspectOpenApiPayload({
216 raw,
217 sourceUrl: parsedUrl.toString(),
218 contentType,
219 });
220
221 return jsonResponse({
222 status: response.status,
223 statusText: response.statusText,
224 spec: inspected.spec,
225 inferredAuth: inspected.inferredAuth,
226 }, 200, { cacheable: cacheableRequest });
227 } catch (error) {
228 if (isAbortError(error)) {
229 return jsonResponse({ detail: "Request timed out while fetching spec" }, 504);
230 }
231
232 const detail = error instanceof Error ? error.message : "Failed to fetch spec";
233 return jsonResponse({ detail }, 502);
234 } finally {
235 clearTimeout(timeoutId);
236 }
237}
executor/apps/web/src/lib/openapi/spec-inspector.ts+60−29
@@ -10,14 +10,29 @@ type OpenApiInspectionResult = { · 9 unmodified lines
10 inferredAuth: InferredSpecAuth;10 inferredAuth: InferredSpecAuth;
11};11};
12 12
13const inferredSpecAuthSchema = z.object({13export type InferredSpecAuth = {
14 type: "none" | "bearer" | "apiKey" | "basic" | "mixed";
15 mode?: "workspace" | "account" | "organization";
16 header?: string;
17 inferred: true;
18};
19
20const generatedPreparedInferredAuthSchema = z.object({
14 type: z.enum(["none", "bearer", "apiKey", "basic", "mixed"]),21 type: z.enum(["none", "bearer", "apiKey", "basic", "mixed"]),
15 mode: z.enum(["workspace", "account", "organization"]).optional(),22 mode: z.enum(["workspace", "account", "organization"]).optional(),
16 header: z.string().optional(),23 header: z.string().optional(),
17 inferred: z.literal(true),24}).optional();
25
26const generatedPreparedSpecSchema = z.object({
27 servers: z.array(z.string()).optional(),
28 inferredAuth: generatedPreparedInferredAuthSchema,
18});29});
19 30
20export type InferredSpecAuth = z.infer<typeof inferredSpecAuthSchema>;31const generatedResponseSchema = z.object({
32 status: z.enum(["ready", "failed"]).optional(),
33 prepared: z.unknown().optional(),
34 error: z.string().optional(),
35});
21 36
22const recordSchema = z.record(z.string(), z.unknown());37const recordSchema = z.record(z.string(), z.unknown());
23 38
@@ -198,16 +213,22 @@ export async function fetchAndInspectOpenApiSpec(input: { · 174 unmodified lines
198 headers?: Record<string, string>;213 headers?: Record<string, string>;
199 signal?: AbortSignal;214 signal?: AbortSignal;
200}): Promise<OpenApiInspectionResult> {215}): Promise<OpenApiInspectionResult> {
201 const response = await fetch("/api/openapi/inspect", {216 const hasHeaders = Object.keys(input.headers ?? {}).length > 0;
202 method: "POST",217 const params = new URLSearchParams({
218 specUrl: input.specUrl,
219 sourceName: "openapi-inspect",
220 includeDts: "0",
221 });
222
223 if (hasHeaders) {
224 params.set("headers", JSON.stringify(input.headers ?? {}));
225 }
226
227 const response = await fetch(`/api/generate?${params.toString()}`, {
228 method: "GET",
203 headers: {229 headers: {
204 "content-type": "application/json",
205 Accept: "application/json",230 Accept: "application/json",
206 },231 },
207 body: JSON.stringify({
208 specUrl: input.specUrl,
209 headers: input.headers ?? {},
210 }),
211 signal: input.signal,232 signal: input.signal,
212 cache: "no-store",233 cache: "no-store",
213 });234 });
@@ -220,31 +241,41 @@ export async function fetchAndInspectOpenApiSpec(input: { · 6 unmodified lines
220 }241 }
221 242
222 if (!response.ok) {243 if (!response.ok) {
223 const payloadRecord = recordSchema.safeParse(payload);244 const generatedPayload = generatedResponseSchema.safeParse(payload);
224 const detail = payloadRecord.success && typeof payloadRecord.data.detail === "string"245 const detail = generatedPayload.success
225 ? payloadRecord.data.detail246 ? (generatedPayload.data.error ?? "")
226 : "";247 : "";
227 const status = payloadRecord.success && typeof payloadRecord.data.status === "number"248 throw new Error(createSpecFetchErrorMessage({ status: response.status, statusText: response.statusText, detail }));
228 ? payloadRecord.data.status
229 : response.status;
230 const statusText = payloadRecord.success && typeof payloadRecord.data.statusText === "string"
231 ? payloadRecord.data.statusText
232 : response.statusText;
233 throw new Error(createSpecFetchErrorMessage({ status, statusText, detail }));
234 }249 }
235 250
236 const payloadRecord = recordSchema.safeParse(payload);251 const generatedPayload = generatedResponseSchema.safeParse(payload);
237 if (!payloadRecord.success) {252 if (!generatedPayload.success || generatedPayload.data.status === "failed") {
238 throw new Error("Spec inspection returned an invalid response");253 throw new Error("Spec generation returned an invalid response");
239 }254 }
240 255
241 const parsedSpec = recordSchema.safeParse(payloadRecord.data.spec);256 const preparedParsed = generatedPreparedSpecSchema.safeParse(generatedPayload.data.prepared);
242 if (!parsedSpec.success) {257 if (!preparedParsed.success) {
243 throw new Error("Spec inspection did not return a valid OpenAPI document");258 throw new Error("Spec generation did not return a valid prepared document");
244 }259 }
245 260
246 const parsedAuth = inferredSpecAuthSchema.safeParse(payloadRecord.data.inferredAuth);261 const servers = (preparedParsed.data.servers ?? [])
247 const inferredAuth = parsedAuth.success ? parsedAuth.data : inferSecuritySchemaAuth(parsedSpec.data);262 .filter((server): server is string => typeof server === "string" && server.trim().length > 0)
263 .map((url) => ({ url }));
264
265 const generatedInferredAuth = preparedParsed.data.inferredAuth;
266 const inferredAuth: InferredSpecAuth = generatedInferredAuth
267 ? {
268 type: generatedInferredAuth.type,
269 ...(generatedInferredAuth.mode ? { mode: generatedInferredAuth.mode } : {}),
270 ...(generatedInferredAuth.type === "apiKey" && generatedInferredAuth.header
271 ? { header: generatedInferredAuth.header }
272 : {}),
273 inferred: true,
274 }
275 : { type: "none", inferred: true };
248 276
249 return { spec: parsedSpec.data, inferredAuth };277 return {
278 spec: { servers },
279 inferredAuth,
280 };
250}281}
executor/packages/database/src/runtime/tool_source_loading.ts+7−7
@@ -115,13 +115,13 @@ async function loadPreparedOpenApiSpecFromExternalGenerate( · 114 unmodified lines
115 const timeout = setTimeout(() => controller.abort(), OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS);115 const timeout = setTimeout(() => controller.abort(), OPENAPI_EXTERNAL_GENERATE_TIMEOUT_MS);
116 116
117 try {117 try {
118 const response = await fetch(endpoint, {118 const url = new URL(endpoint);
119 method: "POST",119 url.searchParams.set("specUrl", specUrl);
120 headers: {120 url.searchParams.set("sourceName", sourceName);
121 "content-type": "application/json",121 url.searchParams.set("includeDts", includeDts ? "1" : "0");
122 },122
123 body: JSON.stringify({ specUrl, sourceName, includeDts }),123 const response = await fetch(url.toString(), {
124 cache: "no-store",124 method: "GET",
125 signal: controller.signal,125 signal: controller.signal,
126 });126 });
127 127

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t