Support Graph-sized OpenAPI imports

Rhys Sullivan committed 4 months ago

Commit 8e9dfce1 parent 82561086 files · +60 −55Browse files

Unified · Split

6 files

bun.lock +4 −1
packages/plugins/openapi/package.json +3 −2
packages/plugins/openapi/src/sdk/parse.test.ts +14 −14
packages/plugins/openapi/src/sdk/parse.ts +4 −33
packages/plugins/openapi/src/sdk/plugin.test.ts +31 −0
packages/plugins/openapi/src/sdk/plugin.ts +4 −5

@@ -968,15 +968,16 @@ · 967 unmodified lines
968	968	"@effect/platform-node": "catalog:",
969	969	"@executor-js/config": "workspace:*",
970	970	"@executor-js/sdk": "workspace:*",
	971	+ "js-yaml": "4.1.1",
971	972	"lucide-react": "^1.7.0",
972	973	"openapi-types": "^12.1.3",
973		− "yaml": "^2.7.1",
974	974	},
975	975	"devDependencies": {
976	976	"@effect/atom-react": "catalog:",
977	977	"@effect/vitest": "catalog:",
978	978	"@executor-js/api": "workspace:*",
979	979	"@executor-js/react": "workspace:*",
	980	+ "@types/js-yaml": "4.0.9",
980	981	"@types/node": "catalog:",
981	982	"@types/react": "catalog:",
982	983	"bun-types": "catalog:",
@@ -2841,6 +2842,8 @@ · 1858 unmodified lines
2841	2842
2842	2843	"@types/js-cookie": ["@types/js-cookie@3.0.6", "", {}, "sha512-wkw9yd1kEXOPnvEeEV1Go1MmxtBJL0RR79aOTAApecWFVu7w0NNXNqhcWgvw2YgZDYadliXkl14pa3WXw5jlCQ=="],
2843	2844
	2845	+ "@types/js-yaml": ["@types/js-yaml@4.0.9", "", {}, "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg=="],
	2846	+
2844	2847	"@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="],
2845	2848
2846	2849	"@types/katex": ["@types/katex@0.16.8", "", {}, "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg=="],

packages/plugins/openapi/package.json+3−2

@@ -64,15 +64,16 @@ · 63 unmodified lines
64	64	"@effect/platform-node": "catalog:",
65	65	"@executor-js/config": "workspace:*",
66	66	"@executor-js/sdk": "workspace:*",
	67	+ "js-yaml": "4.1.1",
67	68	"lucide-react": "^1.7.0",
68		− "openapi-types": "^12.1.3",
69		− "yaml": "^2.7.1"
	69	+ "openapi-types": "^12.1.3"
70	70	},
71	71	"devDependencies": {
72	72	"@effect/atom-react": "catalog:",
73	73	"@effect/vitest": "catalog:",
74	74	"@executor-js/api": "workspace:*",
75	75	"@executor-js/react": "workspace:*",
	76	+ "@types/js-yaml": "4.0.9",
76	77	"@types/node": "catalog:",
77	78	"@types/react": "catalog:",
78	79	"bun-types": "catalog:",

packages/plugins/openapi/src/sdk/parse.test.ts+14−14

@@ -2,11 +2,7 @@ import { describe, expect, it } from "@effect/vitest"; · 1 unmodified line
2	2	import { Effect } from "effect";
3	3
4	4	import { OpenApiParseError } from "./errors";
5		−import {
6		− ensureGenericOpenApiSpecTextWithinLimit,
7		− MAX_GENERIC_OPENAPI_SPEC_TEXT_LENGTH,
8		− parse,
9		−} from "./parse";
	5	+import { parse } from "./parse";
10	6
11	7	describe("OpenAPI parse", () => {
12	8	it.effect("parses JSON OpenAPI documents", () =>
@@ -69,17 +65,21 @@ paths: {} · 56 unmodified lines
69	65	}),
70	66	);
71	67
72		− it.effect("rejects oversized generic OpenAPI documents before parsing", () =>
	68	+ it.effect("parses Graph-sized YAML OpenAPI documents", () =>
73	69	Effect.gen(function* () {
74		− const error = yield* ensureGenericOpenApiSpecTextWithinLimit(
75		− "x".repeat(MAX_GENERIC_OPENAPI_SPEC_TEXT_LENGTH + 1),
76		− ).pipe(Effect.flip);
77		−
78		− expect(error).toBeInstanceOf(OpenApiParseError);
79		− expect(error).toHaveProperty(
80		− "message",
81		− expect.stringContaining("too large for the generic OpenAPI importer"),
	70	+ const largeDescription = "x".repeat(36 * 1024 * 1024);
	71	+ const doc = yield* parse(
	72	+ `openapi: 3.0.0
	73	+info:
	74	+ title: Large Test
	75	+ version: 1.0.0
	76	+ description: "${largeDescription}"
	77	+paths: {}
	78	+`,
82	79	);
	80	+
	81	+ expect(doc.info.title).toBe("Large Test");
	82	+ expect(doc.info.description).toHaveLength(largeDescription.length);
83	83	}),
84	84	);
85	85	});

packages/plugins/openapi/src/sdk/parse.ts+4−33

@@ -1,7 +1,7 @@
1	1	import type { OpenAPI, OpenAPIV3, OpenAPIV3_1 } from "openapi-types";
2	2	import { Duration, Effect, Schema } from "effect";
3	3	import { HttpClient, HttpClientRequest } from "effect/unstable/http";
4		−import YAML from "yaml";
	4	+import { JSON_SCHEMA, load as parseYamlDocument } from "js-yaml";
5	5
6	6	import { OpenApiExtractionError, OpenApiParseError } from "./errors";
7	7
@@ -12,31 +12,6 @@ export interface SpecFetchCredentials { · 4 unmodified lines
12	12	readonly queryParams?: Record<string, string>;
13	13	}
14	14
15		−export const MAX_GENERIC_OPENAPI_SPEC_TEXT_LENGTH = 25 * 1024 * 1024;
16		−
17		−const formatMiB = (bytes: number): string => `${(bytes / 1024 / 1024).toFixed(1)} MiB`;
18		−
19		−const oversizedOpenApiSpecError = (size: number): OpenApiParseError =>
20		− new OpenApiParseError({
21		− message: `OpenAPI document is too large for the generic OpenAPI importer (${formatMiB(size)}; limit ${formatMiB(MAX_GENERIC_OPENAPI_SPEC_TEXT_LENGTH)}). Use a provider-specific integration when available, or import a smaller filtered OpenAPI document.`,
22		− });
23		−
24		−const parseContentLength = (
25		− headers: Readonly<Record<string, string \| undefined>>,
26		−): number \| undefined => {
27		− const raw = headers["content-length"] ?? headers["Content-Length"];
28		− if (!raw) return undefined;
29		− const parsed = Number(raw);
30		− return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
31		−};
32		−
33		−export const ensureGenericOpenApiSpecTextWithinLimit = (
34		− specText: string,
35		−): Effect.Effect<string, OpenApiParseError> =>
36		− specText.length > MAX_GENERIC_OPENAPI_SPEC_TEXT_LENGTH
37		− ? Effect.fail(oversizedOpenApiSpecError(specText.length))
38		− : Effect.succeed(specText);
39		−
40	15	// ExtractionError subclass raised from parse() for non-3.x specs
41	16	class OpenApiExtractionErrorFromParse extends OpenApiExtractionError {}
42	17
@@ -75,10 +50,6 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( · 32 unmodified lines
75	50	message: `Failed to fetch OpenAPI document: HTTP ${response.status}`,
76	51	});
77	52	}
78		− const contentLength = parseContentLength(response.headers);
79		− if (contentLength !== undefined && contentLength > MAX_GENERIC_OPENAPI_SPEC_TEXT_LENGTH) {
80		− return yield* oversizedOpenApiSpecError(contentLength);
81		− }
82	53	const specText = yield* response.text.pipe(
83	54	Effect.mapError(
84	55	(_cause) =>
@@ -87,7 +58,7 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( · 2 unmodified lines
87	58	}),
88	59	),
89	60	);
90		− return yield* ensureGenericOpenApiSpecTextWithinLimit(specText);
	61	+ return specText;
91	62	});
92	63
93	64	/**
@@ -97,7 +68,7 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( · 3 unmodified lines
97	68	export const resolveSpecText = (input: string, credentials?: SpecFetchCredentials) =>
98	69	input.startsWith("http://") \|\| input.startsWith("https://")
99	70	? fetchSpecText(input, credentials)
100		− : ensureGenericOpenApiSpecTextWithinLimit(input);
	71	+ : Effect.succeed(input);
101	72
102	73	/**
103	74	* Parse an OpenAPI document from spec text and validate it's OpenAPI 3.x.
@@ -158,7 +129,7 @@ const parseJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Un · 54 unmodified lines
158	129
159	130	const parseJsonLike = (text: string): Effect.Effect<unknown, unknown> => {
160	131	const parseYaml = Effect.try({
161		− try: () => YAML.parse(text) as unknown,
	132	+ try: () => parseYamlDocument(text, { json: true, schema: JSON_SCHEMA }) as unknown,
162	133	catch: () => "YamlParseFailed" as const,
163	134	});
164	135	if (!text.startsWith("{") && !text.startsWith("[")) return parseYaml;

packages/plugins/openapi/src/sdk/plugin.test.ts+31−0

@@ -628,6 +628,37 @@ describe("OpenAPI Plugin", () => { · 627 unmodified lines
628	628	),
629	629	);
630	630
	631	+ it.effect("addSpec accepts Graph-sized OpenAPI blobs", () =>
	632	+ Effect.gen(function* () {
	633	+ const executor = yield* createExecutor(makeTestConfig({ plugins: testPlugins() }));
	634	+ const largeDescription = "x".repeat(36 * 1024 * 1024);
	635	+
	636	+ const added = yield* executor.openapi.addSpec({
	637	+ spec: {
	638	+ kind: "blob",
	639	+ value: `openapi: 3.0.0
	640	+info:
	641	+ title: Large Test
	642	+ version: 1.0.0
	643	+ description: "${largeDescription}"
	644	+servers:
	645	+ - url: https://example.com
	646	+paths:
	647	+ /me:
	648	+ get:
	649	+ operationId: getMe
	650	+ responses:
	651	+ "200":
	652	+ description: OK
	653	+`,
	654	+ },
	655	+ slug: "large_api",
	656	+ });
	657	+
	658	+ expect(added.toolCount).toBe(1);
	659	+ }),
	660	+ );
	661	+
631	662	it.effect("removeSpec cleans up the integration and its tools", () =>
632	663	Effect.scoped(
633	664	Effect.gen(function* () {

packages/plugins/openapi/src/sdk/plugin.ts+4−5

@@ -22,7 +22,7 @@ import { · 21 unmodified lines
22	22
23	23	import { decodeOpenApiIntegrationConfig, type OpenApiIntegrationConfig } from "./config";
24	24	import { OpenApiExtractionError, OpenApiOAuthError, OpenApiParseError } from "./errors";
25		−import { ensureGenericOpenApiSpecTextWithinLimit, parse, resolveSpecText } from "./parse";
	25	+import { parse, resolveSpecText } from "./parse";
26	26	import {
27	27	convertGoogleDiscoveryBundleToOpenApi,
28	28	convertGoogleDiscoveryToOpenApi,
@@ -30,7 +30,7 @@ import { · 1 unmodified line
30	30	isGoogleDiscoveryUrl,
31	31	} from "./google-discovery";
32	32	import { extract } from "./extract";
33		−import { previewSpec, previewSpecText, type SpecPreview } from "./preview";
	33	+import { previewSpecText, type SpecPreview } from "./preview";
34	34	import { deriveAuthenticationTemplateFromPreview, firstBaseUrlForPreview } from "./derive-auth";
35	35	import { openApiPresets } from "./presets";
36	36	import { makeDefaultOpenapiStore, type OpenapiStore } from "./store";
@@ -473,8 +473,7 @@ export const openApiPlugin = definePlugin((options?: OpenApiPluginOptions) => { · 436 unmodified lines
473	473	const specText = yield* resolveSpecText(spec.url).pipe(Effect.provide(httpClientLayer));
474	474	return { specText };
475	475	}
476		− const specText = yield* ensureGenericOpenApiSpecTextWithinLimit(spec.value);
477		− return { specText };
	476	+ return { specText: spec.value };
478	477	});
479	478
480	479	return {
@@ -722,7 +721,7 @@ export const openApiPlugin = definePlugin((options?: OpenApiPluginOptions) => { · 241 unmodified lines
722	721	Effect.map((conversion) => conversion.specText),
723	722	)
724	723	: yield* resolveSpecText(previewInput.spec).pipe(Effect.provide(httpClientLayer));
725		− return yield* previewSpec(specText).pipe(Effect.provide(httpClientLayer));
	724	+ return yield* previewSpecText(specText);
726	725	}),
727	726
728	727	addSpec,