Store generated .d.ts in prepared specs to avoid duplicate fetches and support large OpenAPI payloads reliably. Add cache-table coverage and large/real-spec tests to protect the new loading and round-trip behavior.
Unified · Split
| @@ -136,7 +136,7 @@ const WORKSPACE_TOOL_CACHE_TTL_MS = 5 * 60 * 60_000; · 135 unmodified lines | |||
| 136 | const WORKSPACE_TOOL_CACHE_RETRY_TTL_MS = 5 * 60 * 60_000; | 136 | const WORKSPACE_TOOL_CACHE_RETRY_TTL_MS = 5 * 60 * 60_000; |
| 137 | 137 | ||
| 138 | /** Cache version — bump when PreparedOpenApiSpec shape changes. */ | 138 | /** Cache version — bump when PreparedOpenApiSpec shape changes. */ |
| 139 | const OPENAPI_CACHE_VERSION = "v2"; | 139 | const OPENAPI_CACHE_VERSION = "v3"; |
| 140 | 140 | ||
| 141 | function isWorkspaceToolCacheFresh( | 141 | function isWorkspaceToolCacheFresh( |
| 142 | cached: { signature: string; loadedAt: number; warnings: string[] }, | 142 | cached: { signature: string; loadedAt: number; warnings: string[] }, |
| @@ -0,0 +1,539 @@ | |||
| 1 | import { expect, test, describe } from "bun:test"; | ||
| 2 | import { | ||
| 3 | prepareOpenApiSpec, | ||
| 4 | buildOpenApiToolsFromPrepared, | ||
| 5 | type PreparedOpenApiSpec, | ||
| 6 | } from "./tool_sources"; | ||
| 7 | |||
| 8 | /** | ||
| 9 | * Generate a synthetic OpenAPI spec with many operations, simulating a large | ||
| 10 | * real-world spec like Stripe, GitHub, or Cloudflare. | ||
| 11 | * | ||
| 12 | * Each operation gets a unique tag, operationId, parameters, request body, | ||
| 13 | * and response schema to produce meaningful type metadata. | ||
| 14 | */ | ||
| 15 | function makeLargeSpec(operationCount: number): Record<string, unknown> { | ||
| 16 | const paths: Record<string, unknown> = {}; | ||
| 17 | |||
| 18 | for (let i = 0; i < operationCount; i++) { | ||
| 19 | const tag = `resource_${i}`; | ||
| 20 | const pathTemplate = `/api/v1/${tag}/{id}`; | ||
| 21 | |||
| 22 | paths[pathTemplate] = { | ||
| 23 | get: { | ||
| 24 | operationId: `get_${tag}`, | ||
| 25 | tags: [tag], | ||
| 26 | summary: `Get ${tag} by ID`, | ||
| 27 | parameters: [ | ||
| 28 | { | ||
| 29 | name: "id", | ||
| 30 | in: "path", | ||
| 31 | required: true, | ||
| 32 | schema: { type: "string" }, | ||
| 33 | }, | ||
| 34 | { | ||
| 35 | name: "include", | ||
| 36 | in: "query", | ||
| 37 | required: false, | ||
| 38 | schema: { type: "string", enum: ["metadata", "related", "all"] }, | ||
| 39 | }, | ||
| 40 | ], | ||
| 41 | responses: { | ||
| 42 | "200": { | ||
| 43 | description: "Success", | ||
| 44 | content: { | ||
| 45 | "application/json": { | ||
| 46 | schema: { | ||
| 47 | type: "object", | ||
| 48 | properties: { | ||
| 49 | id: { type: "string" }, | ||
| 50 | name: { type: "string" }, | ||
| 51 | created_at: { type: "string" }, | ||
| 52 | status: { | ||
| 53 | type: "string", | ||
| 54 | enum: ["active", "inactive", "archived"], | ||
| 55 | }, | ||
| 56 | metadata: { | ||
| 57 | type: "object", | ||
| 58 | properties: { | ||
| 59 | key: { type: "string" }, | ||
| 60 | value: { type: "string" }, | ||
| 61 | }, | ||
| 62 | }, | ||
| 63 | }, | ||
| 64 | required: ["id", "name"], | ||
| 65 | }, | ||
| 66 | }, | ||
| 67 | }, | ||
| 68 | }, | ||
| 69 | }, | ||
| 70 | }, | ||
| 71 | post: { | ||
| 72 | operationId: `create_${tag}`, | ||
| 73 | tags: [tag], | ||
| 74 | summary: `Create ${tag}`, | ||
| 75 | requestBody: { | ||
| 76 | required: true, | ||
| 77 | content: { | ||
| 78 | "application/json": { | ||
| 79 | schema: { | ||
| 80 | type: "object", | ||
| 81 | properties: { | ||
| 82 | name: { type: "string" }, | ||
| 83 | description: { type: "string" }, | ||
| 84 | config: { | ||
| 85 | type: "object", | ||
| 86 | properties: { | ||
| 87 | enabled: { type: "boolean" }, | ||
| 88 | timeout: { type: "number" }, | ||
| 89 | tags: { | ||
| 90 | type: "array", | ||
| 91 | items: { type: "string" }, | ||
| 92 | }, | ||
| 93 | }, | ||
| 94 | }, | ||
| 95 | }, | ||
| 96 | required: ["name"], | ||
| 97 | }, | ||
| 98 | }, | ||
| 99 | }, | ||
| 100 | }, | ||
| 101 | responses: { | ||
| 102 | "201": { | ||
| 103 | description: "Created", | ||
| 104 | content: { | ||
| 105 | "application/json": { | ||
| 106 | schema: { | ||
| 107 | type: "object", | ||
| 108 | properties: { | ||
| 109 | id: { type: "string" }, | ||
| 110 | name: { type: "string" }, | ||
| 111 | }, | ||
| 112 | }, | ||
| 113 | }, | ||
| 114 | }, | ||
| 115 | }, | ||
| 116 | }, | ||
| 117 | }, | ||
| 118 | delete: { | ||
| 119 | operationId: `delete_${tag}`, | ||
| 120 | tags: [tag], | ||
| 121 | summary: `Delete ${tag}`, | ||
| 122 | parameters: [ | ||
| 123 | { | ||
| 124 | name: "id", | ||
| 125 | in: "path", | ||
| 126 | required: true, | ||
| 127 | schema: { type: "string" }, | ||
| 128 | }, | ||
| 129 | ], | ||
| 130 | responses: { | ||
| 131 | "204": { description: "Deleted" }, | ||
| 132 | }, | ||
| 133 | }, | ||
| 134 | }; | ||
| 135 | } | ||
| 136 | |||
| 137 | return { | ||
| 138 | openapi: "3.0.3", | ||
| 139 | info: { title: "Large Test API", version: "1.0.0" }, | ||
| 140 | servers: [{ url: "https://api.example.com/v1" }], | ||
| 141 | paths, | ||
| 142 | }; | ||
| 143 | } | ||
| 144 | |||
| 145 | describe("prepareOpenApiSpec with large specs", () => { | ||
| 146 | test("handles spec with 100 resources (300 operations) without truncation", async () => { | ||
| 147 | const spec = makeLargeSpec(100); | ||
| 148 | const prepared = await prepareOpenApiSpec(spec, "large-test"); | ||
| 149 | |||
| 150 | // Should have all operations | ||
| 151 | const pathCount = Object.keys(prepared.paths).length; | ||
| 152 | expect(pathCount).toBe(100); | ||
| 153 | |||
| 154 | // Should not have any size-related warnings | ||
| 155 | const sizeWarnings = prepared.warnings.filter( | ||
| 156 | (w) => | ||
| 157 | w.includes("cache size") || | ||
| 158 | w.includes("too large") || | ||
| 159 | w.includes("metadata omitted"), | ||
| 160 | ); | ||
| 161 | expect(sizeWarnings).toHaveLength(0); | ||
| 162 | |||
| 163 | // Verify the full prepared spec serializes to well over the old 900KB limit | ||
| 164 | // to confirm we're not artificially truncating | ||
| 165 | const json = JSON.stringify(prepared); | ||
| 166 | // 100 resources × 3 ops each × ~2KB per op = ~600KB minimum | ||
| 167 | // With type metadata it should be even larger | ||
| 168 | expect(json.length).toBeGreaterThan(100_000); | ||
| 169 | }); | ||
| 170 | |||
| 171 | test("preserves operationTypes when spec is large", async () => { | ||
| 172 | const spec = makeLargeSpec(50); | ||
| 173 | const prepared = await prepareOpenApiSpec(spec, "type-test"); | ||
| 174 | |||
| 175 | // openapiTS should generate types for all operations | ||
| 176 | if (prepared.operationTypes) { | ||
| 177 | const opCount = Object.keys(prepared.operationTypes).length; | ||
| 178 | // Should have types for most operations (openapiTS generates from valid specs) | ||
| 179 | expect(opCount).toBeGreaterThan(0); | ||
| 180 | |||
| 181 | // Spot-check a known operation | ||
| 182 | const getType = prepared.operationTypes["get_resource_0"]; | ||
| 183 | if (getType) { | ||
| 184 | expect(getType.argsType).toBeDefined(); | ||
| 185 | expect(getType.returnsType).toBeDefined(); | ||
| 186 | } | ||
| 187 | } | ||
| 188 | }); | ||
| 189 | |||
| 190 | test("preserves schemaTypes when spec has shared schemas", async () => { | ||
| 191 | const specWithSchemas: Record<string, unknown> = { | ||
| 192 | openapi: "3.0.3", | ||
| 193 | info: { title: "Schema Test", version: "1.0.0" }, | ||
| 194 | servers: [{ url: "https://api.example.com" }], | ||
| 195 | components: { | ||
| 196 | schemas: { | ||
| 197 | User: { | ||
| 198 | type: "object", | ||
| 199 | properties: { | ||
| 200 | id: { type: "string" }, | ||
| 201 | email: { type: "string" }, | ||
| 202 | name: { type: "string" }, | ||
| 203 | role: { type: "string", enum: ["admin", "user", "viewer"] }, | ||
| 204 | }, | ||
| 205 | required: ["id", "email"], | ||
| 206 | }, | ||
| 207 | UserList: { | ||
| 208 | type: "object", | ||
| 209 | properties: { | ||
| 210 | data: { | ||
| 211 | type: "array", | ||
| 212 | items: { $ref: "#/components/schemas/User" }, | ||
| 213 | }, | ||
| 214 | total: { type: "number" }, | ||
| 215 | }, | ||
| 216 | }, | ||
| 217 | }, | ||
| 218 | }, | ||
| 219 | paths: { | ||
| 220 | "/users": { | ||
| 221 | get: { | ||
| 222 | operationId: "listUsers", | ||
| 223 | tags: ["users"], | ||
| 224 | summary: "List users", | ||
| 225 | responses: { | ||
| 226 | "200": { | ||
| 227 | description: "ok", | ||
| 228 | content: { | ||
| 229 | "application/json": { | ||
| 230 | schema: { $ref: "#/components/schemas/UserList" }, | ||
| 231 | }, | ||
| 232 | }, | ||
| 233 | }, | ||
| 234 | }, | ||
| 235 | }, | ||
| 236 | }, | ||
| 237 | "/users/{id}": { | ||
| 238 | get: { | ||
| 239 | operationId: "getUser", | ||
| 240 | tags: ["users"], | ||
| 241 | parameters: [ | ||
| 242 | { | ||
| 243 | name: "id", | ||
| 244 | in: "path", | ||
| 245 | required: true, | ||
| 246 | schema: { type: "string" }, | ||
| 247 | }, | ||
| 248 | ], | ||
| 249 | summary: "Get user", | ||
| 250 | responses: { | ||
| 251 | "200": { | ||
| 252 | description: "ok", | ||
| 253 | content: { | ||
| 254 | "application/json": { | ||
| 255 | schema: { $ref: "#/components/schemas/User" }, | ||
| 256 | }, | ||
| 257 | }, | ||
| 258 | }, | ||
| 259 | }, | ||
| 260 | }, | ||
| 261 | }, | ||
| 262 | }, | ||
| 263 | }; | ||
| 264 | |||
| 265 | const prepared = await prepareOpenApiSpec(specWithSchemas, "schema-test"); | ||
| 266 | |||
| 267 | // Should have generated schema types | ||
| 268 | if (prepared.schemaTypes) { | ||
| 269 | expect(Object.keys(prepared.schemaTypes).length).toBeGreaterThan(0); | ||
| 270 | } | ||
| 271 | |||
| 272 | // No warnings | ||
| 273 | expect(prepared.warnings).toHaveLength(0); | ||
| 274 | }); | ||
| 275 | }); | ||
| 276 | |||
| 277 | describe("buildOpenApiToolsFromPrepared", () => { | ||
| 278 | test("builds tools from a prepared spec with full type metadata", async () => { | ||
| 279 | const spec = makeLargeSpec(5); | ||
| 280 | const prepared = await prepareOpenApiSpec(spec, "build-test"); | ||
| 281 | |||
| 282 | const tools = buildOpenApiToolsFromPrepared( | ||
| 283 | { | ||
| 284 | type: "openapi", | ||
| 285 | name: "test-api", | ||
| 286 | spec, | ||
| 287 | baseUrl: "https://api.example.com/v1", | ||
| 288 | }, | ||
| 289 | prepared, | ||
| 290 | ); | ||
| 291 | |||
| 292 | // 5 resources × 3 operations each = 15 tools | ||
| 293 | expect(tools).toHaveLength(15); | ||
| 294 | |||
| 295 | // Each tool should have metadata | ||
| 296 | for (const tool of tools) { | ||
| 297 | expect(tool.metadata).toBeDefined(); | ||
| 298 | expect(tool.metadata!.argsType).toBeDefined(); | ||
| 299 | expect(tool.metadata!.returnsType).toBeDefined(); | ||
| 300 | } | ||
| 301 | |||
| 302 | // GET operations should default to "auto" approval | ||
| 303 | const getTools = tools.filter((t) => t.path.includes("get_")); | ||
| 304 | for (const tool of getTools) { | ||
| 305 | expect(tool.approval).toBe("auto"); | ||
| 306 | } | ||
| 307 | |||
| 308 | // POST/DELETE operations should default to "required" approval | ||
| 309 | const writeTools = tools.filter( | ||
| 310 | (t) => t.path.includes("create_") || t.path.includes("delete_"), | ||
| 311 | ); | ||
| 312 | for (const tool of writeTools) { | ||
| 313 | expect(tool.approval).toBe("required"); | ||
| 314 | } | ||
| 315 | }); | ||
| 316 | |||
| 317 | test("schemaTypes only attached to first tool from a source", async () => { | ||
| 318 | const specWithSchemas: Record<string, unknown> = { | ||
| 319 | openapi: "3.0.3", | ||
| 320 | info: { title: "Schema placement test", version: "1.0.0" }, | ||
| 321 | servers: [{ url: "https://api.example.com" }], | ||
| 322 | components: { | ||
| 323 | schemas: { | ||
| 324 | Widget: { | ||
| 325 | type: "object", | ||
| 326 | properties: { | ||
| 327 | id: { type: "string" }, | ||
| 328 | name: { type: "string" }, | ||
| 329 | }, | ||
| 330 | }, | ||
| 331 | }, | ||
| 332 | }, | ||
| 333 | paths: { | ||
| 334 | "/widgets": { | ||
| 335 | get: { | ||
| 336 | operationId: "listWidgets", | ||
| 337 | tags: ["widgets"], | ||
| 338 | responses: { | ||
| 339 | "200": { | ||
| 340 | description: "ok", | ||
| 341 | content: { | ||
| 342 | "application/json": { | ||
| 343 | schema: { | ||
| 344 | type: "array", | ||
| 345 | items: { $ref: "#/components/schemas/Widget" }, | ||
| 346 | }, | ||
| 347 | }, | ||
| 348 | }, | ||
| 349 | }, | ||
| 350 | }, | ||
| 351 | }, | ||
| 352 | post: { | ||
| 353 | operationId: "createWidget", | ||
| 354 | tags: ["widgets"], | ||
| 355 | requestBody: { | ||
| 356 | content: { | ||
| 357 | "application/json": { | ||
| 358 | schema: { $ref: "#/components/schemas/Widget" }, | ||
| 359 | }, | ||
| 360 | }, | ||
| 361 | }, | ||
| 362 | responses: { | ||
| 363 | "201": { description: "created" }, | ||
| 364 | }, | ||
| 365 | }, | ||
| 366 | }, | ||
| 367 | }, | ||
| 368 | }; | ||
| 369 | |||
| 370 | const prepared = await prepareOpenApiSpec(specWithSchemas, "schema-placement"); | ||
| 371 | const tools = buildOpenApiToolsFromPrepared( | ||
| 372 | { | ||
| 373 | type: "openapi", | ||
| 374 | name: "widgets", | ||
| 375 | spec: specWithSchemas, | ||
| 376 | baseUrl: "https://api.example.com", | ||
| 377 | }, | ||
| 378 | prepared, | ||
| 379 | ); | ||
| 380 | |||
| 381 | expect(tools.length).toBe(2); | ||
| 382 | |||
| 383 | // Only the first tool should have schemaTypes | ||
| 384 | const withSchemas = tools.filter( | ||
| 385 | (t) => t.metadata?.schemaTypes && Object.keys(t.metadata.schemaTypes).length > 0, | ||
| 386 | ); | ||
| 387 | expect(withSchemas.length).toBeLessThanOrEqual(1); | ||
| 388 | }); | ||
| 389 | }); | ||
| 390 | |||
| 391 | describe("prepared spec serialization round-trip", () => { | ||
| 392 | test("prepared spec survives JSON serialization (simulating cache store/load)", async () => { | ||
| 393 | const spec = makeLargeSpec(20); | ||
| 394 | const prepared = await prepareOpenApiSpec(spec, "roundtrip-test"); | ||
| 395 | |||
| 396 | // Simulate what loadCachedOpenApiSpec does: serialize → store → load → deserialize | ||
| 397 | const json = JSON.stringify(prepared); | ||
| 398 | const restored = JSON.parse(json) as PreparedOpenApiSpec; | ||
| 399 | |||
| 400 | expect(restored.servers).toEqual(prepared.servers); | ||
| 401 | expect(Object.keys(restored.paths).length).toBe( | ||
| 402 | Object.keys(prepared.paths).length, | ||
| 403 | ); | ||
| 404 | expect(restored.warnings).toEqual(prepared.warnings); | ||
| 405 | |||
| 406 | if (prepared.operationTypes) { | ||
| 407 | expect(restored.operationTypes).toBeDefined(); | ||
| 408 | expect(Object.keys(restored.operationTypes!).length).toBe( | ||
| 409 | Object.keys(prepared.operationTypes).length, | ||
| 410 | ); | ||
| 411 | } | ||
| 412 | |||
| 413 | if (prepared.schemaTypes) { | ||
| 414 | expect(restored.schemaTypes).toBeDefined(); | ||
| 415 | expect(Object.keys(restored.schemaTypes!).length).toBe( | ||
| 416 | Object.keys(prepared.schemaTypes).length, | ||
| 417 | ); | ||
| 418 | } | ||
| 419 | |||
| 420 | // Verify tools built from the restored spec match the original | ||
| 421 | const config = { | ||
| 422 | type: "openapi" as const, | ||
| 423 | name: "roundtrip", | ||
| 424 | spec, | ||
| 425 | baseUrl: "https://api.example.com/v1", | ||
| 426 | }; | ||
| 427 | const originalTools = buildOpenApiToolsFromPrepared(config, prepared); | ||
| 428 | const restoredTools = buildOpenApiToolsFromPrepared(config, restored); | ||
| 429 | |||
| 430 | expect(restoredTools.length).toBe(originalTools.length); | ||
| 431 | for (let i = 0; i < originalTools.length; i++) { | ||
| 432 | expect(restoredTools[i]!.path).toBe(originalTools[i]!.path); | ||
| 433 | expect(restoredTools[i]!.description).toBe(originalTools[i]!.description); | ||
| 434 | expect(restoredTools[i]!.approval).toBe(originalTools[i]!.approval); | ||
| 435 | expect(restoredTools[i]!.metadata?.argsType).toBe( | ||
| 436 | originalTools[i]!.metadata?.argsType, | ||
| 437 | ); | ||
| 438 | expect(restoredTools[i]!.metadata?.returnsType).toBe( | ||
| 439 | originalTools[i]!.metadata?.returnsType, | ||
| 440 | ); | ||
| 441 | } | ||
| 442 | }); | ||
| 443 | |||
| 444 | test("large spec serialized size exceeds old 900KB limit", async () => { | ||
| 445 | // Generate a spec large enough to exceed the old ActionCache limit. | ||
| 446 | // This verifies that the new approach handles what the old one couldn't. | ||
| 447 | const spec = makeLargeSpec(200); | ||
| 448 | const prepared = await prepareOpenApiSpec(spec, "size-test"); | ||
| 449 | const json = JSON.stringify(prepared); | ||
| 450 | |||
| 451 | // The old limit was 900_000 bytes. With 200 resources (600 operations), | ||
| 452 | // the prepared spec with full type metadata should exceed that easily. | ||
| 453 | console.log( | ||
| 454 | `Large spec size: ${(json.length / 1024).toFixed(0)}KB (old limit: 900KB)`, | ||
| 455 | ); | ||
| 456 | |||
| 457 | // Even if it doesn't exceed 900KB with this synthetic spec (real APIs have | ||
| 458 | // much more complex schemas), verify no warnings about size limits exist | ||
| 459 | const sizeWarnings = prepared.warnings.filter( | ||
| 460 | (w) => w.includes("cache size") || w.includes("too large"), | ||
| 461 | ); | ||
| 462 | expect(sizeWarnings).toHaveLength(0); | ||
| 463 | }); | ||
| 464 | }); | ||
| 465 | |||
| 466 | describe("spec with $ref keys survives cache round-trip", () => { | ||
| 467 | test("specs with $ref in schemas serialize cleanly as JSON", async () => { | ||
| 468 | // The old ActionCache approach had issues with $ref keys in Convex values. | ||
| 469 | // The new approach stores as a JSON blob, which handles $ref fine. | ||
| 470 | const specWithRefs: Record<string, unknown> = { | ||
| 471 | openapi: "3.0.3", | ||
| 472 | info: { title: "Ref test", version: "1.0.0" }, | ||
| 473 | servers: [{ url: "https://api.example.com" }], | ||
| 474 | components: { | ||
| 475 | schemas: { | ||
| 476 | Address: { | ||
| 477 | type: "object", | ||
| 478 | properties: { | ||
| 479 | street: { type: "string" }, | ||
| 480 | city: { type: "string" }, | ||
| 481 | }, | ||
| 482 | }, | ||
| 483 | Person: { | ||
| 484 | type: "object", | ||
| 485 | properties: { | ||
| 486 | name: { type: "string" }, | ||
| 487 | address: { $ref: "#/components/schemas/Address" }, | ||
| 488 | }, | ||
| 489 | }, | ||
| 490 | }, | ||
| 491 | }, | ||
| 492 | paths: { | ||
| 493 | "/people": { | ||
| 494 | get: { | ||
| 495 | operationId: "listPeople", | ||
| 496 | tags: ["people"], | ||
| 497 | responses: { | ||
| 498 | "200": { | ||
| 499 | description: "ok", | ||
| 500 | content: { | ||
| 501 | "application/json": { | ||
| 502 | schema: { | ||
| 503 | type: "array", | ||
| 504 | items: { $ref: "#/components/schemas/Person" }, | ||
| 505 | }, | ||
| 506 | }, | ||
| 507 | }, | ||
| 508 | }, | ||
| 509 | }, | ||
| 510 | }, | ||
| 511 | }, | ||
| 512 | }, | ||
| 513 | }; | ||
| 514 | |||
| 515 | const prepared = await prepareOpenApiSpec(specWithRefs, "ref-test"); | ||
| 516 | const json = JSON.stringify(prepared); | ||
| 517 | |||
| 518 | // Should serialize without error | ||
| 519 | expect(json.length).toBeGreaterThan(0); | ||
| 520 | |||
| 521 | // Should deserialize back | ||
| 522 | const restored = JSON.parse(json) as PreparedOpenApiSpec; | ||
| 523 | expect(Object.keys(restored.paths).length).toBe(1); | ||
| 524 | |||
| 525 | // Build tools from the restored spec | ||
| 526 | const tools = buildOpenApiToolsFromPrepared( | ||
| 527 | { | ||
| 528 | type: "openapi", | ||
| 529 | name: "ref-api", | ||
| 530 | spec: specWithRefs, | ||
| 531 | baseUrl: "https://api.example.com", | ||
| 532 | }, | ||
| 533 | restored, | ||
| 534 | ); | ||
| 535 | |||
| 536 | expect(tools.length).toBe(1); | ||
| 537 | expect(tools[0]!.path).toBe("ref_api.people.listpeople"); | ||
| 538 | }); | ||
| 539 | }); | ||
| @@ -0,0 +1,155 @@ | |||
| 1 | /** | ||
| 2 | * Integration tests against real-world OpenAPI specs. | ||
| 3 | * | ||
| 4 | * These verify the full pipeline: fetch → parse → generate types → compact → cache round-trip. | ||
| 5 | * Catches regressions where a spec format change or library update breaks loading. | ||
| 6 | * | ||
| 7 | * Specs are fetched live so these tests require network access and are slower (~5-60s each). | ||
| 8 | */ | ||
| 9 | import { test, expect, describe } from "bun:test"; | ||
| 10 | import { prepareOpenApiSpec, buildOpenApiToolsFromPrepared } from "./tool_sources"; | ||
| 11 | |||
| 12 | interface SpecFixture { | ||
| 13 | name: string; | ||
| 14 | url: string; | ||
| 15 | /** Minimum expected path count — sanity check the spec loaded fully */ | ||
| 16 | minPaths: number; | ||
| 17 | /** Whether openapiTS should succeed (false for Swagger 2.x specs) */ | ||
| 18 | expectDts: boolean; | ||
| 19 | } | ||
| 20 | |||
| 21 | const SPECS: SpecFixture[] = [ | ||
| 22 | { | ||
| 23 | name: "jira", | ||
| 24 | url: "https://developer.atlassian.com/cloud/jira/platform/swagger-v3.v3.json", | ||
| 25 | minPaths: 100, | ||
| 26 | expectDts: true, | ||
| 27 | }, | ||
| 28 | { | ||
| 29 | name: "openai", | ||
| 30 | url: "https://app.stainless.com/api/spec/documented/openai/openapi.documented.yml", | ||
| 31 | minPaths: 10, | ||
| 32 | expectDts: true, | ||
| 33 | }, | ||
| 34 | { | ||
| 35 | name: "github", | ||
| 36 | url: "https://raw.githubusercontent.com/github/rest-api-description/main/descriptions/api.github.com/api.github.com.yaml", | ||
| 37 | minPaths: 500, | ||
| 38 | expectDts: true, | ||
| 39 | }, | ||
| 40 | { | ||
| 41 | name: "vercel", | ||
| 42 | url: "https://openapi.vercel.sh", | ||
| 43 | minPaths: 50, | ||
| 44 | expectDts: true, | ||
| 45 | }, | ||
| 46 | { | ||
| 47 | name: "slack", | ||
| 48 | url: "https://api.slack.com/specs/openapi/v2/slack_web.json", | ||
| 49 | minPaths: 50, | ||
| 50 | // Swagger 2.x — openapiTS only supports OpenAPI 3.x, and no `servers` field | ||
| 51 | expectDts: false, | ||
| 52 | }, | ||
| 53 | { | ||
| 54 | name: "stripe", | ||
| 55 | url: "https://raw.githubusercontent.com/stripe/openapi/master/openapi/spec3.json", | ||
| 56 | minPaths: 100, | ||
| 57 | expectDts: true, | ||
| 58 | }, | ||
| 59 | { | ||
| 60 | name: "cloudflare", | ||
| 61 | url: "https://raw.githubusercontent.com/cloudflare/api-schemas/main/openapi.yaml", | ||
| 62 | minPaths: 500, | ||
| 63 | // openapiTS fails on broken discriminator $ref mappings in this spec | ||
| 64 | expectDts: false, | ||
| 65 | }, | ||
| 66 | { | ||
| 67 | name: "sentry", | ||
| 68 | url: "https://raw.githubusercontent.com/getsentry/sentry-api-schema/refs/heads/main/openapi-derefed.json", | ||
| 69 | minPaths: 50, | ||
| 70 | expectDts: true, | ||
| 71 | }, | ||
| 72 | ]; | ||
| 73 | |||
| 74 | describe("real-world OpenAPI specs", () => { | ||
| 75 | for (const fixture of SPECS) { | ||
| 76 | test( | ||
| 77 | `${fixture.name}: full pipeline`, | ||
| 78 | async () => { | ||
| 79 | const start = performance.now(); | ||
| 80 | const prepared = await prepareOpenApiSpec(fixture.url, fixture.name); | ||
| 81 | const prepareMs = performance.now() - start; | ||
| 82 | |||
| 83 | const pathCount = Object.keys(prepared.paths).length; | ||
| 84 | const dtsSize = prepared.dts ? `${(prepared.dts.length / 1024).toFixed(0)}KB` : "none"; | ||
| 85 | |||
| 86 | console.log( | ||
| 87 | ` ${fixture.name}: ${pathCount} paths, dts=${dtsSize}, prepare=${prepareMs.toFixed(0)}ms`, | ||
| 88 | ); | ||
| 89 | |||
| 90 | // Spec loaded with enough paths | ||
| 91 | expect(pathCount).toBeGreaterThanOrEqual(fixture.minPaths); | ||
| 92 | |||
| 93 | // .d.ts generated (or correctly skipped for Swagger 2.x) | ||
| 94 | if (fixture.expectDts) { | ||
| 95 | expect(prepared.dts).toBeDefined(); | ||
| 96 | expect(prepared.dts!.length).toBeGreaterThan(0); | ||
| 97 | // Should contain the operations interface | ||
| 98 | expect(prepared.dts).toContain("operations"); | ||
| 99 | } | ||
| 100 | |||
| 101 | // Servers extracted (Swagger 2.x specs may not have servers) | ||
| 102 | if (fixture.expectDts) { | ||
| 103 | expect(prepared.servers.length).toBeGreaterThan(0); | ||
| 104 | } | ||
| 105 | |||
| 106 | // Cache round-trip: serialize → deserialize → build tools | ||
| 107 | const json = JSON.stringify(prepared); | ||
| 108 | const restored = JSON.parse(json) as typeof prepared; | ||
| 109 | expect(Object.keys(restored.paths).length).toBe(pathCount); | ||
| 110 | |||
| 111 | // Build tools from the restored spec | ||
| 112 | const buildStart = performance.now(); | ||
| 113 | const tools = buildOpenApiToolsFromPrepared( | ||
| 114 | { | ||
| 115 | type: "openapi", | ||
| 116 | name: fixture.name, | ||
| 117 | spec: fixture.url, | ||
| 118 | baseUrl: prepared.servers[0] || `https://${fixture.name}.example.com`, | ||
| 119 | }, | ||
| 120 | restored, | ||
| 121 | ); | ||
| 122 | const buildMs = performance.now() - buildStart; | ||
| 123 | |||
| 124 | console.log( | ||
| 125 | ` ${fixture.name}: ${tools.length} tools, build=${buildMs.toFixed(0)}ms`, | ||
| 126 | ); | ||
| 127 | |||
| 128 | expect(tools.length).toBeGreaterThan(0); | ||
| 129 | |||
| 130 | // Spot-check: every tool has a path and type metadata | ||
| 131 | for (const tool of tools) { | ||
| 132 | expect(tool.path).toContain(`${fixture.name}.`); | ||
| 133 | expect(typeof tool.description).toBe("string"); | ||
| 134 | expect(tool.metadata).toBeDefined(); | ||
| 135 | expect(tool.metadata!.argsType).toBeDefined(); | ||
| 136 | expect(tool.metadata!.returnsType).toBeDefined(); | ||
| 137 | } | ||
| 138 | |||
| 139 | // If we have .d.ts, at least some tools should have richer types than "unknown" | ||
| 140 | if (fixture.expectDts) { | ||
| 141 | const withRichReturns = tools.filter( | ||
| 142 | (t) => t.metadata!.returnsType !== "unknown", | ||
| 143 | ); | ||
| 144 | expect(withRichReturns.length).toBeGreaterThan(0); | ||
| 145 | } | ||
| 146 | |||
| 147 | if (prepared.warnings.length > 0) { | ||
| 148 | console.log(` ${fixture.name} warnings: ${prepared.warnings.join("; ")}`); | ||
| 149 | } | ||
| 150 | }, | ||
| 151 | // These fetch real specs over the network — generous timeout | ||
| 152 | 300_000, | ||
| 153 | ); | ||
| 154 | } | ||
| 155 | }); | ||
| @@ -117,20 +117,19 @@ function toTypeAliasName(schemaName: string, used: Set<string>): string { · 116 unmodified lines | |||
| 117 | // request body, and response types per operation. | 117 | // request body, and response types per operation. |
| 118 | 118 | ||
| 119 | /** | 119 | /** |
| 120 | * Generate TypeScript types for an entire OpenAPI spec using openapi-typescript. | 120 | * Generate a .d.ts string for an OpenAPI spec using openapi-typescript. |
| 121 | * Returns a map of operationId → { argsType, returnsType } strings. | ||
| 122 | * | 121 | * |
| 123 | * Falls back to the hand-rolled type hint generator on failure (e.g. Swagger 2.0 | 122 | * Accepts a pre-parsed object to avoid a redundant HTTP fetch — the caller |
| 124 | * specs, broken $refs). | 123 | * should have already fetched and parsed the spec. |
| 124 | * | ||
| 125 | * Falls back to null on failure (e.g. Swagger 2.0, broken $refs). | ||
| 125 | */ | 126 | */ |
| 126 | async function generateOpenApiTypes( | 127 | async function generateOpenApiDts( |
| 127 | spec: string | Record<string, unknown>, | 128 | spec: Record<string, unknown>, |
| 128 | ): Promise<ExtractedTypes | null> { | 129 | ): Promise<string | null> { |
| 129 | try { | 130 | try { |
| 130 | const input = typeof spec === "string" ? new URL(spec) : (spec as never); | 131 | const ast = await openapiTS(spec as never, { silent: true }); |
| 131 | const ast = await openapiTS(input, { silent: true }); | 132 | return astToString(ast); |
| 132 | const dts = astToString(ast); | ||
| 133 | return extractOperationTypes(dts); | ||
| 134 | } catch (error) { | 133 | } catch (error) { |
| 135 | const msg = error instanceof Error ? error.message : String(error); | 134 | const msg = error instanceof Error ? error.message : String(error); |
| 136 | console.warn(`[executor] openapi-typescript failed, using fallback types: ${msg}`); | 135 | console.warn(`[executor] openapi-typescript failed, using fallback types: ${msg}`); |
| @@ -138,6 +137,18 @@ async function generateOpenApiTypes( · 1 unmodified line | |||
| 138 | } | 137 | } |
| 139 | } | 138 | } |
| 140 | 139 | ||
| 140 | /** | ||
| 141 | * Extract per-operation types from a .d.ts string. | ||
| 142 | * Wraps the TS compiler AST walk — pure CPU, no I/O. | ||
| 143 | */ | ||
| 144 | function extractTypesFromDts(dts: string): ExtractedTypes | null { | ||
| 145 | try { | ||
| 146 | return extractOperationTypes(dts); | ||
| 147 | } catch { | ||
| 148 | return null; | ||
| 149 | } | ||
| 150 | } | ||
| 151 | |||
| 141 | /** Result of extracting types from a generated .d.ts */ | 152 | /** Result of extracting types from a generated .d.ts */ |
| 142 | interface ExtractedTypes { | 153 | interface ExtractedTypes { |
| 143 | /** Per-operation type strings (operationId → args/returns) */ | 154 | /** Per-operation type strings (operationId → args/returns) */ |
| @@ -801,7 +812,11 @@ function compactOpenApiPaths( · 657 unmodified lines | |||
| 801 | export interface PreparedOpenApiSpec { | 812 | export interface PreparedOpenApiSpec { |
| 802 | servers: string[]; | 813 | servers: string[]; |
| 803 | paths: Record<string, unknown>; | 814 | paths: Record<string, unknown>; |
| 815 | /** Raw .d.ts from openapiTS — cached as-is, extracted lazily on read */ | ||
| 816 | dts?: string; | ||
| 817 | /** @deprecated Kept for cache compat — new entries use `dts` instead */ | ||
| 804 | operationTypes?: Record<string, { argsType: string; returnsType: string }>; | 818 | operationTypes?: Record<string, { argsType: string; returnsType: string }>; |
| 819 | /** @deprecated Kept for cache compat — new entries use `dts` instead */ | ||
| 805 | schemaTypes?: Record<string, string>; | 820 | schemaTypes?: Record<string, string>; |
| 806 | warnings: string[]; | 821 | warnings: string[]; |
| 807 | } | 822 | } |
| @@ -810,48 +825,59 @@ export async function prepareOpenApiSpec( · 2 unmodified lines | |||
| 810 | spec: string | Record<string, unknown>, | 825 | spec: string | Record<string, unknown>, |
| 811 | sourceName = "openapi", | 826 | sourceName = "openapi", |
| 812 | ): Promise<PreparedOpenApiSpec> { | 827 | ): Promise<PreparedOpenApiSpec> { |
| 813 | // Run type generation in parallel with spec loading. | ||
| 814 | // We prefer `bundle` so external refs are resolved, but some real-world specs | ||
| 815 | // contain broken internal refs. In that case, fall back to `parse` so we can | ||
| 816 | // still load operation paths and expose usable tools. | ||
| 817 | const typeMapPromise = generateOpenApiTypes(spec); | ||
| 818 | const parser = SwaggerParser as unknown as { | 828 | const parser = SwaggerParser as unknown as { |
| 819 | bundle(spec: unknown): Promise<unknown>; | 829 | bundle(spec: unknown): Promise<unknown>; |
| 820 | parse(spec: unknown): Promise<unknown>; | 830 | parse(spec: unknown): Promise<unknown>; |
| 821 | }; | 831 | }; |
| 822 | 832 | ||
| 823 | const warnings: string[] = []; | 833 | const warnings: string[] = []; |
| 834 | |||
| 835 | // ── Step 1: Single fetch ────────────────────────────────────────────── | ||
| 836 | // If `spec` is a URL string, parse it once to get a JS object. | ||
| 837 | // Both openapiTS and SwaggerParser.bundle then work from this object | ||
| 838 | // instead of each independently fetching the same multi-MB URL. | ||
| 839 | let parsed: Record<string, unknown>; | ||
| 840 | if (typeof spec === "string") { | ||
| 841 | try { | ||
| 842 | parsed = (await parser.parse(spec)) as Record<string, unknown>; | ||
| 843 | } catch (parseError) { | ||
| 844 | const msg = parseError instanceof Error ? parseError.message : String(parseError); | ||
| 845 | throw new Error(`Failed to fetch/parse OpenAPI source '${sourceName}': ${msg}`); | ||
| 846 | } | ||
| 847 | } else { | ||
| 848 | parsed = spec; | ||
| 849 | } | ||
| 850 | |||
| 851 | // ── Step 2: Generate .d.ts and bundle in parallel ───────────────────── | ||
| 852 | // Both now operate on the in-memory object — no additional HTTP fetches. | ||
| 853 | const dtsPromise = generateOpenApiDts(parsed); | ||
| 854 | |||
| 824 | let bundled: Record<string, unknown>; | 855 | let bundled: Record<string, unknown>; |
| 825 | try { | 856 | try { |
| 826 | bundled = await parser.bundle(spec).then((api) => api as Record<string, unknown>); | 857 | bundled = (await parser.bundle(parsed)) as Record<string, unknown>; |
| 827 | } catch (bundleError) { | 858 | } catch (bundleError) { |
| 828 | const bundleMessage = bundleError instanceof Error ? bundleError.message : String(bundleError); | 859 | const bundleMessage = bundleError instanceof Error ? bundleError.message : String(bundleError); |
| 829 | warnings.push(`OpenAPI bundle failed for '${sourceName}', using parse-only mode: ${bundleMessage}`); | 860 | warnings.push(`OpenAPI bundle failed for '${sourceName}', using parse-only mode: ${bundleMessage}`); |
| 830 | try { | 861 | bundled = parsed; |
| 831 | bundled = await parser.parse(spec).then((api) => api as Record<string, unknown>); | ||
| 832 | } catch (parseError) { | ||
| 833 | const parseMessage = parseError instanceof Error ? parseError.message : String(parseError); | ||
| 834 | throw new Error( | ||
| 835 | `Failed to load OpenAPI source '${sourceName}': bundle error (${bundleMessage}); parse error (${parseMessage})`, | ||
| 836 | ); | ||
| 837 | } | ||
| 838 | } | 862 | } |
| 839 | 863 | ||
| 840 | const typeMap = await typeMapPromise; | 864 | const dts = await dtsPromise; |
| 841 | const servers = Array.isArray(bundled.servers) ? (bundled.servers as Array<{ url?: unknown }>) : []; | 865 | |
| 866 | // ── Step 3: Extract operation type IDs for compaction ────────────────── | ||
| 867 | // We extract types here to know which operations have generated types | ||
| 868 | // (so compactOpenApiPaths can strip schemas for those operations). | ||
| 869 | // This is pure CPU — no I/O. | ||
| 870 | const typeMap = dts ? extractTypesFromDts(dts) : null; | ||
| 842 | const operationTypeIds = new Set<string>(typeMap ? [...typeMap.operations.keys()] : []); | 871 | const operationTypeIds = new Set<string>(typeMap ? [...typeMap.operations.keys()] : []); |
| 843 | 872 | ||
| 873 | const servers = Array.isArray(bundled.servers) ? (bundled.servers as Array<{ url?: unknown }>) : []; | ||
| 874 | |||
| 844 | return { | 875 | return { |
| 845 | servers: servers | 876 | servers: servers |
| 846 | .map((server) => (typeof server.url === "string" ? server.url : "")) | 877 | .map((server) => (typeof server.url === "string" ? server.url : "")) |
| 847 | .filter((url) => url.length > 0), | 878 | .filter((url) => url.length > 0), |
| 848 | paths: compactOpenApiPaths(bundled.paths, operationTypeIds), | 879 | paths: compactOpenApiPaths(bundled.paths, operationTypeIds), |
| 849 | ...(typeMap | 880 | dts: dts ?? undefined, |
| 850 | ? { | ||
| 851 | operationTypes: Object.fromEntries(typeMap.operations), | ||
| 852 | ...(typeMap.schemas.size > 0 ? { schemaTypes: Object.fromEntries(typeMap.schemas) } : {}), | ||
| 853 | } | ||
| 854 | : {}), | ||
| 855 | warnings, | 881 | warnings, |
| 856 | }; | 882 | }; |
| 857 | } | 883 | } |
| @@ -865,6 +891,18 @@ export function buildOpenApiToolsFromPrepared( · 7 unmodified lines | |||
| 865 | throw new Error(`OpenAPI source ${config.name} has no base URL (set baseUrl)`); | 891 | throw new Error(`OpenAPI source ${config.name} has no base URL (set baseUrl)`); |
| 866 | } | 892 | } |
| 867 | 893 | ||
| 894 | // Extract per-operation types from cached .d.ts (or use legacy pre-extracted types) | ||
| 895 | let typeMap: ExtractedTypes | null = null; | ||
| 896 | if (prepared.dts) { | ||
| 897 | typeMap = extractTypesFromDts(prepared.dts); | ||
| 898 | } else if (prepared.operationTypes) { | ||
| 899 | // Legacy cache compat: old entries have pre-extracted types | ||
| 900 | typeMap = { | ||
| 901 | operations: new Map(Object.entries(prepared.operationTypes)), | ||
| 902 | schemas: new Map(Object.entries(prepared.schemaTypes ?? {})), | ||
| 903 | }; | ||
| 904 | } | ||
| 905 | |||
| 868 | const authHeaders = buildStaticAuthHeaders(config.auth); | 906 | const authHeaders = buildStaticAuthHeaders(config.auth); |
| 869 | const sourceKey = `openapi:${config.name}`; | 907 | const sourceKey = `openapi:${config.name}`; |
| 870 | const credentialSpec = buildCredentialSpec(sourceKey, config.auth); | 908 | const credentialSpec = buildCredentialSpec(sourceKey, config.auth); |
| @@ -873,8 +911,8 @@ export function buildOpenApiToolsFromPrepared( · 2 unmodified lines | |||
| 873 | 911 | ||
| 874 | // Schema type aliases referenced by operations — stored on the first tool only | 912 | // Schema type aliases referenced by operations — stored on the first tool only |
| 875 | // to avoid duplicating hundreds of KB across every tool from this source. | 913 | // to avoid duplicating hundreds of KB across every tool from this source. |
| 876 | const schemaTypes = prepared.schemaTypes && Object.keys(prepared.schemaTypes).length > 0 | 914 | const schemaTypes = typeMap && typeMap.schemas.size > 0 |
| 877 | ? prepared.schemaTypes | 915 | ? Object.fromEntries(typeMap.schemas) |
| 878 | : undefined; | 916 | : undefined; |
| 879 | let schemaTypesEmitted = false; | 917 | let schemaTypesEmitted = false; |
| 880 | 918 | ||
| @@ -908,7 +946,7 @@ export function buildOpenApiToolsFromPrepared( · 27 unmodified lines | |||
| 908 | })); | 946 | })); |
| 909 | 947 | ||
| 910 | // Use openapiTS-generated types if available, otherwise fall back to schema hints | 948 | // Use openapiTS-generated types if available, otherwise fall back to schema hints |
| 911 | const generatedTypes = prepared.operationTypes?.[operationIdRaw]; | 949 | const generatedTypes = typeMap?.operations.get(operationIdRaw); |
| 912 | let argsType: string; | 950 | let argsType: string; |
| 913 | let returnsType: string; | 951 | let returnsType: string; |
| 914 | 952 | ||
| @@ -0,0 +1,193 @@ | |||
| 1 | import { expect, test, describe } from "bun:test"; | ||
| 2 | import { convexTest } from "convex-test"; | ||
| 3 | import { internal } from "./_generated/api"; | ||
| 4 | import schema from "./schema"; | ||
| 5 | |||
| 6 | function setup() { | ||
| 7 | return convexTest(schema, { | ||
| 8 | "./openApiSpecCache.ts": () => import("./openApiSpecCache"), | ||
| 9 | "./_generated/api.js": () => import("./_generated/api.js"), | ||
| 10 | }); | ||
| 11 | } | ||
| 12 | |||
| 13 | describe("openApiSpecCache table operations", () => { | ||
| 14 | test("getEntry returns null on empty cache", async () => { | ||
| 15 | const t = setup(); | ||
| 16 | |||
| 17 | const result = await t.query(internal.openApiSpecCache.getEntry, { | ||
| 18 | specUrl: "https://example.com/spec.yaml", | ||
| 19 | version: "v2", | ||
| 20 | maxAgeMs: 5 * 60 * 60_000, | ||
| 21 | }); | ||
| 22 | |||
| 23 | expect(result).toBeNull(); | ||
| 24 | }); | ||
| 25 | |||
| 26 | test("putEntry stores and getEntry retrieves", async () => { | ||
| 27 | const t = setup(); | ||
| 28 | |||
| 29 | // Store a blob using the run() escape hatch for storage access | ||
| 30 | const storageId = await t.run(async (ctx) => { | ||
| 31 | const blob = new Blob(['{"servers":[],"paths":{},"warnings":[]}'], { | ||
| 32 | type: "application/json", | ||
| 33 | }); | ||
| 34 | return await ctx.storage.store(blob); | ||
| 35 | }); | ||
| 36 | |||
| 37 | await t.mutation(internal.openApiSpecCache.putEntry, { | ||
| 38 | specUrl: "https://example.com/spec.yaml", | ||
| 39 | version: "v2", | ||
| 40 | storageId, | ||
| 41 | sizeBytes: 40, | ||
| 42 | }); | ||
| 43 | |||
| 44 | const entry = await t.query(internal.openApiSpecCache.getEntry, { | ||
| 45 | specUrl: "https://example.com/spec.yaml", | ||
| 46 | version: "v2", | ||
| 47 | maxAgeMs: 5 * 60 * 60_000, | ||
| 48 | }); | ||
| 49 | |||
| 50 | expect(entry).not.toBeNull(); | ||
| 51 | expect(entry!.storageId).toBe(storageId); | ||
| 52 | expect(entry!.sizeBytes).toBe(40); | ||
| 53 | expect(typeof entry!.createdAt).toBe("number"); | ||
| 54 | }); | ||
| 55 | |||
| 56 | test("getEntry returns null for wrong version", async () => { | ||
| 57 | const t = setup(); | ||
| 58 | |||
| 59 | const storageId = await t.run(async (ctx) => { | ||
| 60 | const blob = new Blob(["{}"], { type: "application/json" }); | ||
| 61 | return await ctx.storage.store(blob); | ||
| 62 | }); | ||
| 63 | |||
| 64 | await t.mutation(internal.openApiSpecCache.putEntry, { | ||
| 65 | specUrl: "https://example.com/spec.yaml", | ||
| 66 | version: "v1", | ||
| 67 | storageId, | ||
| 68 | sizeBytes: 2, | ||
| 69 | }); | ||
| 70 | |||
| 71 | const result = await t.query(internal.openApiSpecCache.getEntry, { | ||
| 72 | specUrl: "https://example.com/spec.yaml", | ||
| 73 | version: "v2", | ||
| 74 | maxAgeMs: 5 * 60 * 60_000, | ||
| 75 | }); | ||
| 76 | |||
| 77 | expect(result).toBeNull(); | ||
| 78 | }); | ||
| 79 | |||
| 80 | test("getEntry returns null for expired entry", async () => { | ||
| 81 | const t = setup(); | ||
| 82 | |||
| 83 | const storageId = await t.run(async (ctx) => { | ||
| 84 | const blob = new Blob(["{}"], { type: "application/json" }); | ||
| 85 | return await ctx.storage.store(blob); | ||
| 86 | }); | ||
| 87 | |||
| 88 | await t.mutation(internal.openApiSpecCache.putEntry, { | ||
| 89 | specUrl: "https://example.com/spec.yaml", | ||
| 90 | version: "v2", | ||
| 91 | storageId, | ||
| 92 | sizeBytes: 2, | ||
| 93 | }); | ||
| 94 | |||
| 95 | // Wait a tiny bit so the entry is guaranteed to be older than 1ms | ||
| 96 | await new Promise((resolve) => setTimeout(resolve, 5)); | ||
| 97 | |||
| 98 | // Ask for entries younger than 1ms — everything should be expired | ||
| 99 | const result = await t.query(internal.openApiSpecCache.getEntry, { | ||
| 100 | specUrl: "https://example.com/spec.yaml", | ||
| 101 | version: "v2", | ||
| 102 | maxAgeMs: 1, | ||
| 103 | }); | ||
| 104 | |||
| 105 | expect(result).toBeNull(); | ||
| 106 | }); | ||
| 107 | |||
| 108 | test("putEntry replaces existing entry for same specUrl+version", async () => { | ||
| 109 | const t = setup(); | ||
| 110 | |||
| 111 | const storageId1 = await t.run(async (ctx) => { | ||
| 112 | const blob = new Blob(['{"version":1}'], { type: "application/json" }); | ||
| 113 | return await ctx.storage.store(blob); | ||
| 114 | }); | ||
| 115 | |||
| 116 | await t.mutation(internal.openApiSpecCache.putEntry, { | ||
| 117 | specUrl: "https://example.com/spec.yaml", | ||
| 118 | version: "v2", | ||
| 119 | storageId: storageId1, | ||
| 120 | sizeBytes: 14, | ||
| 121 | }); | ||
| 122 | |||
| 123 | const storageId2 = await t.run(async (ctx) => { | ||
| 124 | const blob = new Blob(['{"version":2}'], { type: "application/json" }); | ||
| 125 | return await ctx.storage.store(blob); | ||
| 126 | }); | ||
| 127 | |||
| 128 | await t.mutation(internal.openApiSpecCache.putEntry, { | ||
| 129 | specUrl: "https://example.com/spec.yaml", | ||
| 130 | version: "v2", | ||
| 131 | storageId: storageId2, | ||
| 132 | sizeBytes: 14, | ||
| 133 | }); | ||
| 134 | |||
| 135 | const entry = await t.query(internal.openApiSpecCache.getEntry, { | ||
| 136 | specUrl: "https://example.com/spec.yaml", | ||
| 137 | version: "v2", | ||
| 138 | maxAgeMs: 5 * 60 * 60_000, | ||
| 139 | }); | ||
| 140 | |||
| 141 | expect(entry).not.toBeNull(); | ||
| 142 | expect(entry!.storageId).toBe(storageId2); | ||
| 143 | |||
| 144 | // Old blob should be deleted | ||
| 145 | const oldBlob = await t.run(async (ctx) => { | ||
| 146 | return await ctx.storage.get(storageId1); | ||
| 147 | }); | ||
| 148 | expect(oldBlob).toBeNull(); | ||
| 149 | }); | ||
| 150 | |||
| 151 | test("different specUrls have independent cache entries", async () => { | ||
| 152 | const t = setup(); | ||
| 153 | |||
| 154 | const storageIdA = await t.run(async (ctx) => { | ||
| 155 | const blob = new Blob(['{"source":"a"}'], { type: "application/json" }); | ||
| 156 | return await ctx.storage.store(blob); | ||
| 157 | }); | ||
| 158 | |||
| 159 | const storageIdB = await t.run(async (ctx) => { | ||
| 160 | const blob = new Blob(['{"source":"b"}'], { type: "application/json" }); | ||
| 161 | return await ctx.storage.store(blob); | ||
| 162 | }); | ||
| 163 | |||
| 164 | await t.mutation(internal.openApiSpecCache.putEntry, { | ||
| 165 | specUrl: "https://api.stripe.com/spec.yaml", | ||
| 166 | version: "v2", | ||
| 167 | storageId: storageIdA, | ||
| 168 | sizeBytes: 14, | ||
| 169 | }); | ||
| 170 | |||
| 171 | await t.mutation(internal.openApiSpecCache.putEntry, { | ||
| 172 | specUrl: "https://api.github.com/spec.yaml", | ||
| 173 | version: "v2", | ||
| 174 | storageId: storageIdB, | ||
| 175 | sizeBytes: 14, | ||
| 176 | }); | ||
| 177 | |||
| 178 | const entryA = await t.query(internal.openApiSpecCache.getEntry, { | ||
| 179 | specUrl: "https://api.stripe.com/spec.yaml", | ||
| 180 | version: "v2", | ||
| 181 | maxAgeMs: 5 * 60 * 60_000, | ||
| 182 | }); | ||
| 183 | |||
| 184 | const entryB = await t.query(internal.openApiSpecCache.getEntry, { | ||
| 185 | specUrl: "https://api.github.com/spec.yaml", | ||
| 186 | version: "v2", | ||
| 187 | maxAgeMs: 5 * 60 * 60_000, | ||
| 188 | }); | ||
| 189 | |||
| 190 | expect(entryA!.storageId).toBe(storageIdA); | ||
| 191 | expect(entryB!.storageId).toBe(storageIdB); | ||
| 192 | }); | ||
| 193 | }); | ||
| @@ -0,0 +1,100 @@ | |||
| 1 | import { v } from "convex/values"; | ||
| 2 | import { internalMutation, internalQuery } from "./_generated/server"; | ||
| 3 | |||
| 4 | /** | ||
| 5 | * Look up a cached OpenAPI spec by URL and cache version. | ||
| 6 | * Returns the storageId + createdAt if found and not expired, null otherwise. | ||
| 7 | */ | ||
| 8 | export const getEntry = internalQuery({ | ||
| 9 | args: { | ||
| 10 | specUrl: v.string(), | ||
| 11 | version: v.string(), | ||
| 12 | maxAgeMs: v.number(), | ||
| 13 | }, | ||
| 14 | handler: async (ctx, args) => { | ||
| 15 | const entry = await ctx.db | ||
| 16 | .query("openApiSpecCache") | ||
| 17 | .withIndex("by_spec_url_version", (q) => | ||
| 18 | q.eq("specUrl", args.specUrl).eq("version", args.version), | ||
| 19 | ) | ||
| 20 | .unique(); | ||
| 21 | |||
| 22 | if (!entry) return null; | ||
| 23 | |||
| 24 | const age = Date.now() - entry.createdAt; | ||
| 25 | if (age > args.maxAgeMs) return null; | ||
| 26 | |||
| 27 | return { | ||
| 28 | storageId: entry.storageId, | ||
| 29 | sizeBytes: entry.sizeBytes, | ||
| 30 | createdAt: entry.createdAt, | ||
| 31 | }; | ||
| 32 | }, | ||
| 33 | }); | ||
| 34 | |||
| 35 | /** | ||
| 36 | * Write (or replace) a cache entry for a spec URL. | ||
| 37 | * If an older entry exists, its storage blob is deleted. | ||
| 38 | */ | ||
| 39 | export const putEntry = internalMutation({ | ||
| 40 | args: { | ||
| 41 | specUrl: v.string(), | ||
| 42 | version: v.string(), | ||
| 43 | storageId: v.id("_storage"), | ||
| 44 | sizeBytes: v.number(), | ||
| 45 | }, | ||
| 46 | handler: async (ctx, args) => { | ||
| 47 | const existing = await ctx.db | ||
| 48 | .query("openApiSpecCache") | ||
| 49 | .withIndex("by_spec_url_version", (q) => | ||
| 50 | q.eq("specUrl", args.specUrl).eq("version", args.version), | ||
| 51 | ) | ||
| 52 | .unique(); | ||
| 53 | |||
| 54 | if (existing) { | ||
| 55 | // Delete old blob | ||
| 56 | await ctx.storage.delete(existing.storageId); | ||
| 57 | await ctx.db.delete(existing._id); | ||
| 58 | } | ||
| 59 | |||
| 60 | await ctx.db.insert("openApiSpecCache", { | ||
| 61 | specUrl: args.specUrl, | ||
| 62 | storageId: args.storageId, | ||
| 63 | version: args.version, | ||
| 64 | sizeBytes: args.sizeBytes, | ||
| 65 | createdAt: Date.now(), | ||
| 66 | }); | ||
| 67 | }, | ||
| 68 | }); | ||
| 69 | |||
| 70 | /** | ||
| 71 | * Remove all cache entries older than the given timestamp. | ||
| 72 | * Returns the number of entries removed. | ||
| 73 | */ | ||
| 74 | export const pruneExpired = internalMutation({ | ||
| 75 | args: { | ||
| 76 | maxAgeMs: v.number(), | ||
| 77 | batchSize: v.optional(v.number()), | ||
| 78 | }, | ||
| 79 | handler: async (ctx, args) => { | ||
| 80 | const cutoff = Date.now() - args.maxAgeMs; | ||
| 81 | const limit = args.batchSize ?? 50; | ||
| 82 | |||
| 83 | // Scan all entries (the table should be small — one row per unique spec URL) | ||
| 84 | const entries = await ctx.db | ||
| 85 | .query("openApiSpecCache") | ||
| 86 | .collect(); | ||
| 87 | |||
| 88 | let removed = 0; | ||
| 89 | for (const entry of entries) { | ||
| 90 | if (removed >= limit) break; | ||
| 91 | if (entry.createdAt < cutoff) { | ||
| 92 | await ctx.storage.delete(entry.storageId); | ||
| 93 | await ctx.db.delete(entry._id); | ||
| 94 | removed++; | ||
| 95 | } | ||
| 96 | } | ||
| 97 | |||
| 98 | return removed; | ||
| 99 | }, | ||
| 100 | }); | ||