History

test: add real-spec OpenAPI extraction parity checks

Rhys Sullivan committed 8 months ago

Commit f81dd3f1 parent 215a07c2 files · +579 −0Browse files

Unified · Split

2 files
  1. v2/packages/management-api/src/openapi-extraction.real-specs.test.ts +541 −0
  2. v2/packages/management-api/src/openapi-extraction.test.ts +38 −0
v2/packages/management-api/src/openapi-extraction.real-specs.test.ts+541−0
@@ -0,0 +1,541 @@
1+import SwaggerParser from "@apidevtools/swagger-parser";
2+import { describe, expect, it } from "@effect/vitest";
3+import {
4+ OPEN_API_HTTP_METHODS,
5+ OPEN_API_PARAMETER_LOCATIONS,
6+ type OpenApiToolManifest,
7+} from "@executor-v2/schema";
8+import * as Data from "effect/Data";
9+import * as Effect from "effect/Effect";
10+
11+import { extractOpenApiManifest } from "./openapi-extraction";
12+
13+class OpenApiRealSpecTestError extends Data.TaggedError("OpenApiRealSpecTestError")<{
14+ stage: "fetch" | "parse_json" | "bundle" | "parse";
15+ specName: string;
16+ message: string;
17+}> {}
18+
19+type SpecFixture = {
20+ name: string;
21+ url: string;
22+};
23+
24+type ParsedOperation = {
25+ method: (typeof OPEN_API_HTTP_METHODS)[number];
26+ path: string;
27+ operationId?: string;
28+ pathItem: Record<string, unknown>;
29+ operation: Record<string, unknown>;
30+};
31+
32+type ParsedParameter = {
33+ name: string;
34+ location: (typeof OPEN_API_PARAMETER_LOCATIONS)[number];
35+ required: boolean;
36+ schema?: unknown;
37+};
38+
39+type ExpectedTyping = {
40+ inputSchemaJson?: string;
41+ outputSchemaJson?: string;
42+ refHintKeys?: Array<string>;
43+};
44+
45+type SwaggerParserInput = Parameters<typeof SwaggerParser.bundle>[0];
46+
47+const fixtures: ReadonlyArray<SpecFixture> = [
48+ {
49+ name: "cloudflare",
50+ url: "https://raw.githubusercontent.com/cloudflare/api-schemas/main/openapi.json",
51+ },
52+ {
53+ name: "vercel",
54+ url: "https://openapi.vercel.sh/",
55+ },
56+ {
57+ name: "stripe",
58+ url: "https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json",
59+ },
60+];
61+
62+const isRecord = (value: unknown): value is Record<string, unknown> =>
63+ typeof value === "object" && value !== null && !Array.isArray(value);
64+
65+const isParameterLocation = (
66+ value: unknown,
67+): value is (typeof OPEN_API_PARAMETER_LOCATIONS)[number] =>
68+ typeof value === "string" &&
69+ (OPEN_API_PARAMETER_LOCATIONS as ReadonlyArray<string>).includes(value);
70+
71+const operationSignature = (
72+ operation: Pick<ParsedOperation, "method" | "path">,
73+): string => `${operation.method} ${operation.path}`;
74+
75+const toStableValue = (value: unknown): unknown => {
76+ if (Array.isArray(value)) {
77+ return value.map(toStableValue);
78+ }
79+
80+ if (!isRecord(value)) {
81+ return value;
82+ }
83+
84+ const stable: Record<string, unknown> = {};
85+ for (const key of Object.keys(value).sort()) {
86+ stable[key] = toStableValue(value[key]);
87+ }
88+
89+ return stable;
90+};
91+
92+const toStableJson = (value: unknown): string => JSON.stringify(toStableValue(value));
93+
94+const collectRefKeys = (value: unknown, refs: Set<string>): void => {
95+ if (Array.isArray(value)) {
96+ for (const item of value) {
97+ collectRefKeys(item, refs);
98+ }
99+ return;
100+ }
101+
102+ if (!isRecord(value)) {
103+ return;
104+ }
105+
106+ const refValue = value.$ref;
107+ if (typeof refValue === "string" && refValue.startsWith("#/")) {
108+ refs.add(refValue);
109+ }
110+
111+ for (const nested of Object.values(value)) {
112+ collectRefKeys(nested, refs);
113+ }
114+};
115+
116+const resolveJsonPointer = (
117+ root: Record<string, unknown>,
118+ pointer: string,
119+): unknown | null => {
120+ if (!pointer.startsWith("#/")) {
121+ return null;
122+ }
123+
124+ const parts = pointer
125+ .slice(2)
126+ .split("/")
127+ .map((part) => part.replaceAll("~1", "/").replaceAll("~0", "~"));
128+
129+ let current: unknown = root;
130+
131+ for (const part of parts) {
132+ if (!isRecord(current)) {
133+ return null;
134+ }
135+
136+ current = current[part];
137+ if (current === undefined) {
138+ return null;
139+ }
140+ }
141+
142+ return current;
143+};
144+
145+const pickSchemaFromContent = (content: unknown): unknown | null => {
146+ if (!isRecord(content)) {
147+ return null;
148+ }
149+
150+ const preferred = ["application/json", ...Object.keys(content).sort()];
151+ const seen = new Set<string>();
152+
153+ for (const mediaType of preferred) {
154+ if (seen.has(mediaType)) {
155+ continue;
156+ }
157+
158+ seen.add(mediaType);
159+ const mediaTypeValue = content[mediaType];
160+ if (!isRecord(mediaTypeValue)) {
161+ continue;
162+ }
163+
164+ if (mediaTypeValue.schema !== undefined) {
165+ return mediaTypeValue.schema;
166+ }
167+ }
168+
169+ return null;
170+};
171+
172+const responseStatusRank = (statusCode: string): number => {
173+ if (/^2\d\d$/.test(statusCode)) {
174+ return 0;
175+ }
176+
177+ if (statusCode === "default") {
178+ return 1;
179+ }
180+
181+ return 2;
182+};
183+
184+const toParsedParameter = (value: unknown): ParsedParameter | null => {
185+ if (!isRecord(value)) {
186+ return null;
187+ }
188+
189+ const name = value.name;
190+ const location = value.in;
191+ if (typeof name !== "string" || name.trim().length === 0 || !isParameterLocation(location)) {
192+ return null;
193+ }
194+
195+ return {
196+ name: name.trim(),
197+ location,
198+ required: location === "path" || value.required === true,
199+ schema: value.schema,
200+ };
201+};
202+
203+const mergeParameters = (
204+ pathItem: Record<string, unknown>,
205+ operation: Record<string, unknown>,
206+): Array<ParsedParameter> => {
207+ const byKey = new Map<string, ParsedParameter>();
208+
209+ const add = (candidate: unknown) => {
210+ if (!Array.isArray(candidate)) {
211+ return;
212+ }
213+
214+ for (const parameterValue of candidate) {
215+ const parsed = toParsedParameter(parameterValue);
216+ if (!parsed) {
217+ continue;
218+ }
219+
220+ byKey.set(`${parsed.location}:${parsed.name}`, parsed);
221+ }
222+ };
223+
224+ add(pathItem.parameters);
225+ add(operation.parameters);
226+
227+ return [...byKey.values()].sort((left, right) => {
228+ if (left.location === right.location) {
229+ return left.name.localeCompare(right.name);
230+ }
231+
232+ return left.location.localeCompare(right.location);
233+ });
234+};
235+
236+const extractRequestBodySchema = (
237+ operation: Record<string, unknown>,
238+): { schema: unknown; required: boolean } | null => {
239+ const requestBody = operation.requestBody;
240+ if (!isRecord(requestBody)) {
241+ return null;
242+ }
243+
244+ const schema = pickSchemaFromContent(requestBody.content);
245+ if (schema === null) {
246+ return null;
247+ }
248+
249+ return {
250+ schema,
251+ required: requestBody.required === true,
252+ };
253+};
254+
255+const extractResponseSchema = (operation: Record<string, unknown>): unknown | null => {
256+ if (!isRecord(operation.responses)) {
257+ return null;
258+ }
259+
260+ const responseCodes = Object.keys(operation.responses).sort(
261+ (left, right) => responseStatusRank(left) - responseStatusRank(right),
262+ );
263+
264+ for (const responseCode of responseCodes) {
265+ const responseValue = operation.responses[responseCode];
266+ if (!isRecord(responseValue)) {
267+ continue;
268+ }
269+
270+ const schema = pickSchemaFromContent(responseValue.content);
271+ if (schema !== null) {
272+ return schema;
273+ }
274+ }
275+
276+ return null;
277+};
278+
279+const buildExpectedTyping = (
280+ pathItem: Record<string, unknown>,
281+ operation: Record<string, unknown>,
282+): ExpectedTyping | undefined => {
283+ const parameters = mergeParameters(pathItem, operation);
284+ const requestBody = extractRequestBodySchema(operation);
285+ const outputSchema = extractResponseSchema(operation);
286+
287+ const properties: Record<string, unknown> = {};
288+ const required = new Set<string>();
289+
290+ for (const parameter of parameters) {
291+ properties[parameter.name] = parameter.schema ?? { type: "string" };
292+ if (parameter.required) {
293+ required.add(parameter.name);
294+ }
295+ }
296+
297+ if (requestBody !== null) {
298+ properties.body = requestBody.schema;
299+ if (requestBody.required) {
300+ required.add("body");
301+ }
302+ }
303+
304+ const inputSchema =
305+ Object.keys(properties).length > 0
306+ ? {
307+ type: "object",
308+ properties,
309+ required: [...required].sort(),
310+ additionalProperties: false,
311+ }
312+ : null;
313+
314+ if (inputSchema === null && outputSchema === null) {
315+ return undefined;
316+ }
317+
318+ const refs = new Set<string>();
319+ if (inputSchema !== null) {
320+ collectRefKeys(inputSchema, refs);
321+ }
322+
323+ if (outputSchema !== null) {
324+ collectRefKeys(outputSchema, refs);
325+ }
326+
327+ const refHintKeys = [...refs].sort();
328+
329+ return {
330+ inputSchemaJson: inputSchema ? toStableJson(inputSchema) : undefined,
331+ outputSchemaJson: outputSchema ? toStableJson(outputSchema) : undefined,
332+ refHintKeys: refHintKeys.length > 0 ? refHintKeys : undefined,
333+ };
334+};
335+
336+const collectOperationsFromSpec = (spec: unknown): Array<ParsedOperation> => {
337+ if (!isRecord(spec) || !isRecord(spec.paths)) {
338+ return [];
339+ }
340+
341+ const operations: Array<ParsedOperation> = [];
342+
343+ for (const [path, pathItem] of Object.entries(spec.paths)) {
344+ if (!isRecord(pathItem)) {
345+ continue;
346+ }
347+
348+ for (const method of OPEN_API_HTTP_METHODS) {
349+ const operation = pathItem[method];
350+ if (!isRecord(operation)) {
351+ continue;
352+ }
353+
354+ const operationIdRaw = operation.operationId;
355+ const operationId =
356+ typeof operationIdRaw === "string" && operationIdRaw.trim().length > 0
357+ ? operationIdRaw.trim()
358+ : undefined;
359+
360+ operations.push({
361+ method,
362+ path,
363+ operationId,
364+ pathItem,
365+ operation,
366+ });
367+ }
368+ }
369+
370+ operations.sort((left, right) =>
371+ operationSignature(left).localeCompare(operationSignature(right)),
372+ );
373+
374+ return operations;
375+};
376+
377+const collectOperationsFromManifest = (
378+ manifest: OpenApiToolManifest,
379+): Array<ParsedOperation> =>
380+ manifest.tools
381+ .map((tool) => ({
382+ method: tool.method,
383+ path: tool.path,
384+ operationId: tool.toolId,
385+ pathItem: {},
386+ operation: {},
387+ }))
388+ .sort((left, right) =>
389+ operationSignature(left).localeCompare(operationSignature(right)),
390+ );
391+
392+const fetchSpecJson = (
393+ fixture: SpecFixture,
394+): Effect.Effect<unknown, OpenApiRealSpecTestError> =>
395+ Effect.gen(function* () {
396+ const response = yield* Effect.tryPromise({
397+ try: () => fetch(fixture.url),
398+ catch: (cause) =>
399+ new OpenApiRealSpecTestError({
400+ stage: "fetch",
401+ specName: fixture.name,
402+ message: cause instanceof Error ? cause.message : String(cause),
403+ }),
404+ });
405+
406+ if (!response.ok) {
407+ return yield* new OpenApiRealSpecTestError({
408+ stage: "fetch",
409+ specName: fixture.name,
410+ message: `HTTP ${response.status} ${response.statusText}`,
411+ });
412+ }
413+
414+ return yield* Effect.tryPromise({
415+ try: () => response.json(),
416+ catch: (cause) =>
417+ new OpenApiRealSpecTestError({
418+ stage: "parse_json",
419+ specName: fixture.name,
420+ message: cause instanceof Error ? cause.message : String(cause),
421+ }),
422+ });
423+ });
424+
425+const loadParserDocument = (
426+ fixture: SpecFixture,
427+ specJson: unknown,
428+): Effect.Effect<unknown, OpenApiRealSpecTestError> =>
429+ Effect.gen(function* () {
430+ const parserInput = specJson as SwaggerParserInput;
431+
432+ const bundledAttempt = yield* Effect.tryPromise({
433+ try: () => SwaggerParser.bundle(parserInput),
434+ catch: (cause) =>
435+ new OpenApiRealSpecTestError({
436+ stage: "bundle",
437+ specName: fixture.name,
438+ message: cause instanceof Error ? cause.message : String(cause),
439+ }),
440+ }).pipe(Effect.either);
441+
442+ if (bundledAttempt._tag === "Right") {
443+ return bundledAttempt.right;
444+ }
445+
446+ return yield* Effect.tryPromise({
447+ try: () => SwaggerParser.parse(parserInput),
448+ catch: (cause) =>
449+ new OpenApiRealSpecTestError({
450+ stage: "parse",
451+ specName: fixture.name,
452+ message: `bundle failed (${bundledAttempt.left.message}); parse failed (${cause instanceof Error ? cause.message : String(cause)})`,
453+ }),
454+ });
455+ });
456+
457+describe("extractOpenApiManifest real specs parity", () => {
458+ for (const fixture of fixtures) {
459+ it.effect(
460+ `matches parser operation and typing inventory for ${fixture.name}`,
461+ () =>
462+ Effect.gen(function* () {
463+ const specJson = yield* fetchSpecJson(fixture);
464+ const parserDocument = yield* loadParserDocument(fixture, specJson);
465+
466+ const parserOperations = collectOperationsFromSpec(parserDocument);
467+ const parserSignatures = parserOperations.map(operationSignature);
468+
469+ const manifest = yield* extractOpenApiManifest(fixture.name, specJson);
470+ const extractedOperations = collectOperationsFromManifest(manifest);
471+ const extractedSignatures = extractedOperations.map(operationSignature);
472+
473+ expect(manifest.tools.length).toBe(parserOperations.length);
474+ expect(extractedSignatures.length).toBe(parserSignatures.length);
475+ expect(extractedSignatures).toEqual(parserSignatures);
476+
477+ const extractedByToolId = new Map(
478+ manifest.tools.map((tool) => [tool.toolId, tool] as const),
479+ );
480+
481+ for (const operation of parserOperations) {
482+ if (!operation.operationId) {
483+ continue;
484+ }
485+
486+ const extractedTool = extractedByToolId.get(operation.operationId);
487+ expect(extractedTool).toBeDefined();
488+ expect(extractedTool?.method).toBe(operation.method);
489+ expect(extractedTool?.path).toBe(operation.path);
490+ expect(extractedTool?.invocation.method).toBe(operation.method);
491+ expect(extractedTool?.invocation.pathTemplate).toBe(operation.path);
492+
493+ const expectedTyping = buildExpectedTyping(
494+ operation.pathItem,
495+ operation.operation,
496+ );
497+
498+ if (!expectedTyping) {
499+ expect(extractedTool?.typing).toBeUndefined();
500+ continue;
501+ }
502+
503+ expect(extractedTool?.typing).toBeDefined();
504+ expect(extractedTool?.typing?.inputSchemaJson).toBe(
505+ expectedTyping.inputSchemaJson,
506+ );
507+ expect(extractedTool?.typing?.outputSchemaJson).toBe(
508+ expectedTyping.outputSchemaJson,
509+ );
510+ expect(extractedTool?.typing?.refHintKeys).toEqual(
511+ expectedTyping.refHintKeys,
512+ );
513+ }
514+
515+ const referencedRefKeys = new Set<string>();
516+ for (const tool of manifest.tools) {
517+ for (const refKey of tool.typing?.refHintKeys ?? []) {
518+ referencedRefKeys.add(refKey);
519+ }
520+ }
521+
522+ expect(Object.keys(manifest.refHintTable ?? {}).length).toBeGreaterThanOrEqual(
523+ referencedRefKeys.size,
524+ );
525+
526+ if (isRecord(specJson)) {
527+ for (const refKey of referencedRefKeys) {
528+ const hint = manifest.refHintTable?.[refKey];
529+ expect(hint).toBeDefined();
530+
531+ const resolved = resolveJsonPointer(specJson, refKey);
532+ if (resolved !== null) {
533+ expect(hint).toBe(toStableJson(resolved));
534+ }
535+ }
536+ }
537+ }),
538+ 120_000,
539+ );
540+ }
541+});
v2/packages/management-api/src/openapi-extraction.test.ts+38−0
@@ -1,8 +1,14 @@
11 import { describe, expect, it } from "@effect/vitest";
2+import * as Data from "effect/Data";
23 import * as Effect from "effect/Effect";
34
45 import { extractOpenApiManifest } from "./openapi-extraction";
56
7+class CloudflareSpecTestError extends Data.TaggedError("CloudflareSpecTestError")<{
8+ stage: "fetch" | "parse";
9+ message: string;
10+}> {}
11+
612 describe("extractOpenApiManifest invocation metadata", () => {
713 it.effect("extracts parameter and request body metadata", () =>
814 Effect.gen(function* () {
@@ -143,4 +149,36 @@ describe("extractOpenApiManifest invocation metadata", () => { · 134 unmodified lines
143149 });
144150 }),
145151 );
152+
153+ it.effect("extracts a complex Cloudflare OpenAPI schema", () =>
154+ Effect.gen(function* () {
155+ const response = yield* Effect.tryPromise({
156+ try: () =>
157+ fetch("https://raw.githubusercontent.com/cloudflare/api-schemas/main/openapi.json"),
158+ catch: (cause) =>
159+ new CloudflareSpecTestError({
160+ stage: "fetch",
161+ message: `failed to fetch Cloudflare OpenAPI spec: ${cause instanceof Error ? cause.message : String(cause)}`,
162+ }),
163+ });
164+
165+ expect(response.ok).toBe(true);
166+
167+ const openApiSpec = yield* Effect.tryPromise({
168+ try: () => response.json(),
169+ catch: (cause) =>
170+ new CloudflareSpecTestError({
171+ stage: "parse",
172+ message: `failed to parse Cloudflare OpenAPI spec JSON: ${cause instanceof Error ? cause.message : String(cause)}`,
173+ }),
174+ });
175+
176+ const manifest = yield* extractOpenApiManifest("cloudflare", openApiSpec);
177+
178+ expect(manifest.tools.length).toBeGreaterThan(1000);
179+ expect(manifest.tools.some((tool) => tool.typing !== undefined)).toBe(true);
180+ expect(Object.keys(manifest.refHintTable ?? {}).length).toBeGreaterThan(0);
181+ }),
182+ 30_000,
183+ );
146184 });