With cloud backfilled out-of-band and the libSQL hosts rewriting at boot through the data-migration ledger, every database converges on the pointer shape (specHash / introspectionHash) before this code reads it. The transitional dual-read loaders and the optional inline fields (openapi config.spec, graphql config.introspectionJson) are gone from the stored-config schemas; addIntegration still ACCEPTS inline introspection JSON as input, but persists only the blob hash. The graphql getConfig endpoint also stops serving the introspection snapshot — same reasoning as the openapi spec removal: it's a multi-MB build artifact no client reads.
Unified · Split
| @@ -45,12 +45,13 @@ const AddIntegrationResponse = Schema.Struct({ · 44 unmodified lines | |||
| 45 | name: Schema.String, | 45 | name: Schema.String, |
| 46 | }); | 46 | }); |
| 47 | 47 | ||
| 48 | // The full opaque integration config, surfaced for the configure UX. Carries | 48 | // The integration config surfaced for the configure UX. Carries the |
| 49 | // the `authenticationTemplate` the configure / custom-method flow reads/writes. | 49 | // `authenticationTemplate` the configure / custom-method flow reads/writes. |
| 50 | // The introspection snapshot is deliberately NOT served: it's a multi-MB | ||
| 51 | // build artifact in the plugin blob store, and no client reads it. | ||
| 50 | const GraphqlConfigView = Schema.Struct({ | 52 | const GraphqlConfigView = Schema.Struct({ |
| 51 | endpoint: Schema.String, | 53 | endpoint: Schema.String, |
| 52 | name: Schema.String, | 54 | name: Schema.String, |
| 53 | introspectionJson: Schema.optional(Schema.String), | ||
| 54 | headers: Schema.optional(Schema.Record(Schema.String, Schema.String)), | 55 | headers: Schema.optional(Schema.Record(Schema.String, Schema.String)), |
| 55 | queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)), | 56 | queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)), |
| 56 | authenticationTemplate: Schema.Array(GraphqlAuthMethod), | 57 | authenticationTemplate: Schema.Array(GraphqlAuthMethod), |
| @@ -66,7 +66,6 @@ export const GraphqlHandlers = HttpApiBuilder.group(ExecutorApiWithGraphql, "gra · 65 unmodified lines | |||
| 66 | ? { | 66 | ? { |
| 67 | endpoint: config.endpoint, | 67 | endpoint: config.endpoint, |
| 68 | name: config.name, | 68 | name: config.name, |
| 69 | introspectionJson: config.introspectionJson, | ||
| 70 | headers: config.headers ? { ...config.headers } : undefined, | 69 | headers: config.headers ? { ...config.headers } : undefined, |
| 71 | queryParams: config.queryParams ? { ...config.queryParams } : undefined, | 70 | queryParams: config.queryParams ? { ...config.queryParams } : undefined, |
| 72 | authenticationTemplate: [...config.authenticationTemplate], | 71 | authenticationTemplate: [...config.authenticationTemplate], |
| @@ -414,18 +414,18 @@ const introspectHeadersForConnection = ( · 413 unmodified lines | |||
| 414 | return { headers, queryParams }; | 414 | return { headers, queryParams }; |
| 415 | }; | 415 | }; |
| 416 | 416 | ||
| 417 | /** Resolve a config's introspection snapshot text: legacy rows inline it in | 417 | /** Resolve a config's introspection snapshot text from the plugin blob store |
| 418 | * `introspectionJson`; new rows point at the plugin blob store via | 418 | * (`introspectionHash`). Null when the integration has no snapshot (live |
| 419 | * `introspectionHash`. Null when the integration has no snapshot (live | 419 | * introspection territory). Pre-blob rows that inlined the JSON are |
| 420 | * introspection territory). */ | 420 | * rewritten by the introspection-to-blob migrations before this code reads |
| 421 | * them. */ | ||
| 421 | const loadIntrospectionJson = ( | 422 | const loadIntrospectionJson = ( |
| 422 | storage: GraphqlStore, | 423 | storage: GraphqlStore, |
| 423 | config: GraphqlIntegrationConfig, | 424 | config: GraphqlIntegrationConfig, |
| 424 | ): Effect.Effect<string | null, StorageFailure> => { | 425 | ): Effect.Effect<string | null, StorageFailure> => |
| 425 | if (config.introspectionJson != null) return Effect.succeed(config.introspectionJson); | 426 | config.introspectionHash != null |
| 426 | if (config.introspectionHash != null) return storage.getIntrospection(config.introspectionHash); | 427 | ? storage.getIntrospection(config.introspectionHash) |
| 427 | return Effect.succeed(null); | 428 | : Effect.succeed(null); |
| 428 | }; | ||
| 429 | 429 | ||
| 430 | /** Introspect a config live or from its stored snapshot, applying connection | 430 | /** Introspect a config live or from its stored snapshot, applying connection |
| 431 | * auth. A non-null `introspectionJson` (loaded via `loadIntrospectionJson`) | 431 | * auth. A non-null `introspectionJson` (loaded via `loadIntrospectionJson`) |
| @@ -558,9 +558,6 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 126 unmodified lines | |||
| 558 | GraphqlIntegrationConfig.make({ | 558 | GraphqlIntegrationConfig.make({ |
| 559 | endpoint: input.endpoint, | 559 | endpoint: input.endpoint, |
| 560 | name: input.name?.trim() || slugFromEndpoint(input.endpoint), | 560 | name: input.name?.trim() || slugFromEndpoint(input.endpoint), |
| 561 | ...(input.introspectionJson !== undefined | ||
| 562 | ? { introspectionJson: input.introspectionJson } | ||
| 563 | : {}), | ||
| 564 | ...(input.headers !== undefined ? { headers: input.headers } : {}), | 561 | ...(input.headers !== undefined ? { headers: input.headers } : {}), |
| 565 | ...(input.queryParams !== undefined ? { queryParams: input.queryParams } : {}), | 562 | ...(input.queryParams !== undefined ? { queryParams: input.queryParams } : {}), |
| 566 | authenticationTemplate: input.authenticationTemplate | 563 | authenticationTemplate: input.authenticationTemplate |
| @@ -602,7 +599,7 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 35 unmodified lines | |||
| 602 | // their operation bindings) are produced lazily when a connection is | 599 | // their operation bindings) are produced lazily when a connection is |
| 603 | // created (`resolveTools`) / a tool is first invoked (`invokeTool`), | 600 | // created (`resolveTools`) / a tool is first invoked (`invokeTool`), |
| 604 | // using that connection's credential. | 601 | // using that connection's credential. |
| 605 | if (baseConfig.introspectionJson === undefined) { | 602 | if (input.introspectionJson === undefined) { |
| 606 | yield* ctx.transaction( | 603 | yield* ctx.transaction( |
| 607 | ctx.core.integrations.register({ | 604 | ctx.core.integrations.register({ |
| 608 | slug, | 605 | slug, |
| @@ -617,7 +614,7 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 8 unmodified lines | |||
| 617 | 614 | ||
| 618 | // Pre-supplied introspection JSON: parse it offline (no network) and | 615 | // Pre-supplied introspection JSON: parse it offline (no network) and |
| 619 | // persist the operation bindings + snapshot so production stays offline. | 616 | // persist the operation bindings + snapshot so production stays offline. |
| 620 | const introspection = yield* parseIntrospectionJson(baseConfig.introspectionJson); | 617 | const introspection = yield* parseIntrospectionJson(input.introspectionJson); |
| 621 | const { result } = yield* extract(introspection); | 618 | const { result } = yield* extract(introspection); |
| 622 | const prepared = prepareOperations(result.fields, introspection); | 619 | const prepared = prepareOperations(result.fields, introspection); |
| 623 | 620 | ||
| @@ -629,9 +626,8 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 5 unmodified lines | |||
| 629 | // carries only its hash. | 626 | // carries only its hash. |
| 630 | const snapshotJson = JSON.stringify({ data: introspection }); | 627 | const snapshotJson = JSON.stringify({ data: introspection }); |
| 631 | const introspectionHash = yield* sha256Hex(snapshotJson); | 628 | const introspectionHash = yield* sha256Hex(snapshotJson); |
| 632 | const { introspectionJson: _inline, ...withoutInline } = baseConfig; | ||
| 633 | const config = GraphqlIntegrationConfig.make({ | 629 | const config = GraphqlIntegrationConfig.make({ |
| 634 | ...withoutInline, | 630 | ...baseConfig, |
| 635 | introspectionHash, | 631 | introspectionHash, |
| 636 | }); | 632 | }); |
| 637 | 633 | ||
| @@ -677,9 +673,6 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 39 unmodified lines | |||
| 677 | const next = GraphqlIntegrationConfig.make({ | 673 | const next = GraphqlIntegrationConfig.make({ |
| 678 | endpoint: input.endpoint ?? current.endpoint, | 674 | endpoint: input.endpoint ?? current.endpoint, |
| 679 | name: input.name?.trim() || current.name, | 675 | name: input.name?.trim() || current.name, |
| 680 | ...(current.introspectionJson !== undefined | ||
| 681 | ? { introspectionJson: current.introspectionJson } | ||
| 682 | : {}), | ||
| 683 | ...(current.introspectionHash !== undefined | 676 | ...(current.introspectionHash !== undefined |
| 684 | ? { introspectionHash: current.introspectionHash } | 677 | ? { introspectionHash: current.introspectionHash } |
| 685 | : {}), | 678 | : {}), |
| @@ -172,12 +172,10 @@ export const GraphqlIntegrationConfig = Schema.Struct({ · 171 unmodified lines | |||
| 172 | endpoint: Schema.String, | 172 | endpoint: Schema.String, |
| 173 | /** Display name for the integration. */ | 173 | /** Display name for the integration. */ |
| 174 | name: Schema.String, | 174 | name: Schema.String, |
| 175 | /** Inlined introspection JSON text. Legacy rows only: new snapshots live in | ||
| 176 | * the plugin blob store and the config carries `introspectionHash` instead, | ||
| 177 | * so a multi-MB schema never rides along on catalog reads. */ | ||
| 178 | introspectionJson: Schema.optional(Schema.String), | ||
| 179 | /** Hex SHA-256 of the introspection JSON snapshot — the content address of | 175 | /** Hex SHA-256 of the introspection JSON snapshot — the content address of |
| 180 | * the blob (`introspection/<hash>` in the plugin blob store). */ | 176 | * the blob (`introspection/<hash>` in the plugin blob store). Rows that |
| 177 | * predate the blob store (inline `introspectionJson` text) are rewritten | ||
| 178 | * by the introspection-to-blob migrations before this schema sees them. */ | ||
| 181 | introspectionHash: Schema.optional(Schema.String), | 179 | introspectionHash: Schema.optional(Schema.String), |
| 182 | /** Static headers applied to every request (and to add-time introspection). */ | 180 | /** Static headers applied to every request (and to add-time introspection). */ |
| 183 | headers: Schema.optional(Schema.Record(Schema.String, Schema.String)), | 181 | headers: Schema.optional(Schema.Record(Schema.String, Schema.String)), |
| @@ -15,13 +15,16 @@ import type { Authentication } from "./types"; · 14 unmodified lines | |||
| 15 | // | 15 | // |
| 16 | // In v2 there are NO credential bindings, NO per-source secret slots, and NO | 16 | // In v2 there are NO credential bindings, NO per-source secret slots, and NO |
| 17 | // StoredSource credential config. The config carries only: | 17 | // StoredSource credential config. The config carries only: |
| 18 | // - the content hash of the spec blob (legacy rows inline the text instead) | 18 | // - the content hash of the spec blob and/or the source URL to (re)fetch |
| 19 | // and/or the source URL to (re)fetch from, | 19 | // from, |
| 20 | // - the optional base URL override, | 20 | // - the optional base URL override, |
| 21 | // - the auth templates a connection's value is rendered through. | 21 | // - the auth templates a connection's value is rendered through. |
| 22 | // The resolved spec text itself lives in the plugin blob store, keyed | 22 | // The resolved spec text itself lives in the plugin blob store, keyed |
| 23 | // `spec/<specHash>` — it's a build input for resolveTools/refresh, not data | 23 | // `spec/<specHash>` — it's a build input for resolveTools/refresh, not data |
| 24 | // any list/invoke path should pay to load. | 24 | // any list/invoke path should pay to load. Rows that predate the blob store |
| 25 | // (inline `spec` text) are rewritten before this schema sees them: cloud by | ||
| 26 | // the out-of-band migrate-specs-to-blobs script, the libSQL hosts by the | ||
| 27 | // boot-time ledger migration. | ||
| 25 | // --------------------------------------------------------------------------- | 28 | // --------------------------------------------------------------------------- |
| 26 | 29 | ||
| 27 | const OAuthAuthenticationSchema = Schema.Struct({ | 30 | const OAuthAuthenticationSchema = Schema.Struct({ |
| @@ -35,10 +38,6 @@ const OAuthAuthenticationSchema = Schema.Struct({ · 7 unmodified lines | |||
| 35 | export const AuthenticationSchema = Schema.Union([OAuthAuthenticationSchema, ApiKeyAuthMethod]); | 38 | export const AuthenticationSchema = Schema.Union([OAuthAuthenticationSchema, ApiKeyAuthMethod]); |
| 36 | 39 | ||
| 37 | export const OpenApiIntegrationConfigSchema = Schema.Struct({ | 40 | export const OpenApiIntegrationConfigSchema = Schema.Struct({ |
| 38 | /** Inlined OpenAPI document text. Legacy rows only: new registrations store | ||
| 39 | * the resolved text in the plugin blob store and carry `specHash` instead, | ||
| 40 | * so multi-MB documents never ride along on catalog reads. */ | ||
| 41 | spec: Schema.optional(Schema.String), | ||
| 42 | /** Hex SHA-256 of the resolved spec text — the content address of the spec | 41 | /** Hex SHA-256 of the resolved spec text — the content address of the spec |
| 43 | * blob (`spec/<hash>` in the plugin blob store). */ | 42 | * blob (`spec/<hash>` in the plugin blob store). */ |
| 44 | specHash: Schema.optional(Schema.String), | 43 | specHash: Schema.optional(Schema.String), |
| @@ -542,19 +542,15 @@ export const describeOpenApiIntegrationDisplay = ( · 541 unmodified lines | |||
| 542 | 542 | ||
| 543 | // --------------------------------------------------------------------------- | 543 | // --------------------------------------------------------------------------- |
| 544 | // Spec text resolution — the stored config carries the spec's content hash | 544 | // Spec text resolution — the stored config carries the spec's content hash |
| 545 | // (`specHash` → blob `spec/<hash>`); legacy rows still inline the text in | 545 | // (`specHash` → blob `spec/<hash>`). Pre-blob rows that inlined the text are |
| 546 | // `spec`. Every reader goes through this loader so both shapes work until the | 546 | // rewritten by the spec-to-blob migrations before this code reads them. |
| 547 | // inline rows are backfilled. | ||
| 548 | // --------------------------------------------------------------------------- | 547 | // --------------------------------------------------------------------------- |
| 549 | 548 | ||
| 550 | const loadSpecText = ( | 549 | const loadSpecText = ( |
| 551 | storage: OpenapiStore, | 550 | storage: OpenapiStore, |
| 552 | config: OpenApiIntegrationConfig, | 551 | config: OpenApiIntegrationConfig, |
| 553 | ): Effect.Effect<string | null, StorageFailure> => { | 552 | ): Effect.Effect<string | null, StorageFailure> => |
| 554 | if (config.spec != null) return Effect.succeed(config.spec); | 553 | config.specHash != null ? storage.getSpec(config.specHash) : Effect.succeed(null); |
| 555 | if (config.specHash != null) return storage.getSpec(config.specHash); | ||
| 556 | return Effect.succeed(null); | ||
| 557 | }; | ||
| 558 | 554 | ||
| 559 | // --------------------------------------------------------------------------- | 555 | // --------------------------------------------------------------------------- |
| 560 | // Spec → tool definitions (shared by addSpec, resolveTools, and detect) | 556 | // Spec → tool definitions (shared by addSpec, resolveTools, and detect) |
| @@ -85,7 +85,7 @@ describe("OpenAPI plugin — spec blob storage", () => { · 84 unmodified lines | |||
| 85 | }); | 85 | }); |
| 86 | 86 | ||
| 87 | const config = yield* executor.openapi.getConfig("blob_api"); | 87 | const config = yield* executor.openapi.getConfig("blob_api"); |
| 88 | expect(config?.spec).toBeUndefined(); | 88 | expect(Object.keys(config ?? {})).not.toContain("spec"); |
| 89 | expect(config?.specHash).toBe(yield* sha256Hex(text)); | 89 | expect(config?.specHash).toBe(yield* sha256Hex(text)); |
| 90 | }), | 90 | }), |
| 91 | ), | 91 | ), |
| @@ -126,41 +126,50 @@ describe("OpenAPI plugin — spec blob storage", () => { · 34 unmodified lines | |||
| 126 | ), | 126 | ), |
| 127 | ); | 127 | ); |
| 128 | 128 | ||
| 129 | it.effect("resolveTools still derives tools from a legacy inline-spec config", () => | 129 | it.effect("resolveTools reads the spec from the store, never an inline field", () => |
| 130 | Effect.gen(function* () { | 130 | Effect.gen(function* () { |
| 131 | const plugin = openApiPlugin({ httpClientLayer: FetchHttpClient.layer }); | 131 | const plugin = openApiPlugin({ httpClientLayer: FetchHttpClient.layer }); |
| 132 | // A legacy row inlines the spec; the loader must never touch the store. | 132 | const text = specText(); |
| 133 | const hash = yield* sha256Hex(text); | ||
| 133 | const storage: OpenapiStore = { | 134 | const storage: OpenapiStore = { |
| 134 | putOperations: () => Effect.void, | 135 | putOperations: () => Effect.void, |
| 135 | getOperation: () => Effect.succeed(null), | 136 | getOperation: () => Effect.succeed(null), |
| 136 | listOperations: () => Effect.succeed([]), | 137 | listOperations: () => Effect.succeed([]), |
| 137 | removeOperations: () => Effect.void, | 138 | removeOperations: () => Effect.void, |
| 138 | putSpec: () => Effect.void, | 139 | putSpec: () => Effect.void, |
| 139 | getSpec: () => Effect.succeed(null), | 140 | getSpec: (specHash) => Effect.succeed(specHash === hash ? text : null), |
| 140 | }; | 141 | }; |
| 141 | 142 | ||
| 142 | const result = yield* plugin.resolveTools!({ | 143 | const resolve = (config: IntegrationConfig) => |
| 143 | integration: { | 144 | plugin.resolveTools!({ |
| 144 | slug: IntegrationSlug.make("legacy_api"), | 145 | integration: { |
| 145 | description: "legacy", | 146 | slug: IntegrationSlug.make("pointer_api"), |
| 146 | kind: "openapi", | 147 | description: "pointer", |
| 147 | canRemove: true, | 148 | kind: "openapi", |
| 148 | canRefresh: false, | 149 | canRemove: true, |
| 149 | authMethods: [], | 150 | canRefresh: false, |
| 150 | }, | 151 | authMethods: [], |
| 151 | config: { spec: specText() } as IntegrationConfig, | 152 | }, |
| 152 | connection: { | 153 | config, |
| 153 | owner: "org", | 154 | connection: { |
| 154 | integration: IntegrationSlug.make("legacy_api"), | 155 | owner: "org", |
| 155 | name: ConnectionName.make("main"), | 156 | integration: IntegrationSlug.make("pointer_api"), |
| 156 | }, | 157 | name: ConnectionName.make("main"), |
| 157 | template: null, | 158 | }, |
| 158 | storage, | 159 | template: null, |
| 159 | getValue: () => Effect.succeed(null), | 160 | storage, |
| 160 | getValues: () => Effect.succeed({}), | 161 | getValue: () => Effect.succeed(null), |
| 161 | }); | 162 | getValues: () => Effect.succeed({}), |
| 162 | 163 | }); | |
| 163 | expect(result.tools.map((tool) => String(tool.name))).toContain("items.echoHeaders"); | 164 | |
| 165 | const fromPointer = yield* resolve({ specHash: hash } as IntegrationConfig); | ||
| 166 | expect(fromPointer.tools.map((tool) => String(tool.name))).toContain("items.echoHeaders"); | ||
| 167 | |||
| 168 | // A pre-migration row that still inlines `spec` yields no tools: the | ||
| 169 | // spec-to-blob migrations rewrite those rows before this code runs, so | ||
| 170 | // the runtime carries no inline-read path. | ||
| 171 | const fromInline = yield* resolve({ spec: text } as IntegrationConfig); | ||
| 172 | expect(fromInline.tools).toHaveLength(0); | ||
| 164 | }), | 173 | }), |
| 165 | ); | 174 | ); |
| 166 | 175 | ||