History

Drop the inline-spec read paths; pointer configs are the only shape (#969)

With cloud backfilled out-of-band and the libSQL hosts rewriting at boot through the data-migration ledger, every database converges on the pointer shape (specHash / introspectionHash) before this code reads it. The transitional dual-read loaders and the optional inline fields (openapi config.spec, graphql config.introspectionJson) are gone from the stored-config schemas; addIntegration still ACCEPTS inline introspection JSON as input, but persists only the blob hash. The graphql getConfig endpoint also stops serving the introspection snapshot — same reasoning as the openapi spec removal: it's a multi-MB build artifact no client reads.

Rhys Sullivan committed 5 months ago

Commit 9b8baa11 parent 8a74ca07 files · +64 −69Browse files

Unified · Split

7 files
  1. packages/plugins/graphql/src/api/group.ts +4 −3
  2. packages/plugins/graphql/src/api/handlers.ts +0 −1
  3. packages/plugins/graphql/src/sdk/plugin.ts +12 −19
  4. packages/plugins/graphql/src/sdk/types.ts +3 −5
  5. packages/plugins/openapi/src/sdk/config.ts +6 −7
  6. packages/plugins/openapi/src/sdk/plugin.ts +4 −8
  7. packages/plugins/openapi/src/sdk/spec-blob.test.ts +35 −26
packages/plugins/graphql/src/api/group.ts+4−3
@@ -45,12 +45,13 @@ const AddIntegrationResponse = Schema.Struct({ · 44 unmodified lines
45 name: Schema.String,45 name: Schema.String,
46});46});
47 47
48// The full opaque integration config, surfaced for the configure UX. Carries48// The integration config surfaced for the configure UX. Carries the
49// the `authenticationTemplate` the configure / custom-method flow reads/writes.49// `authenticationTemplate` the configure / custom-method flow reads/writes.
50// The introspection snapshot is deliberately NOT served: it's a multi-MB
51// build artifact in the plugin blob store, and no client reads it.
50const GraphqlConfigView = Schema.Struct({52const GraphqlConfigView = Schema.Struct({
51 endpoint: Schema.String,53 endpoint: Schema.String,
52 name: Schema.String,54 name: Schema.String,
53 introspectionJson: Schema.optional(Schema.String),
54 headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),55 headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
55 queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)),56 queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)),
56 authenticationTemplate: Schema.Array(GraphqlAuthMethod),57 authenticationTemplate: Schema.Array(GraphqlAuthMethod),
packages/plugins/graphql/src/api/handlers.ts+0−1
@@ -66,7 +66,6 @@ export const GraphqlHandlers = HttpApiBuilder.group(ExecutorApiWithGraphql, "gra · 65 unmodified lines
66 ? {66 ? {
67 endpoint: config.endpoint,67 endpoint: config.endpoint,
68 name: config.name,68 name: config.name,
69 introspectionJson: config.introspectionJson,
70 headers: config.headers ? { ...config.headers } : undefined,69 headers: config.headers ? { ...config.headers } : undefined,
71 queryParams: config.queryParams ? { ...config.queryParams } : undefined,70 queryParams: config.queryParams ? { ...config.queryParams } : undefined,
72 authenticationTemplate: [...config.authenticationTemplate],71 authenticationTemplate: [...config.authenticationTemplate],
packages/plugins/graphql/src/sdk/plugin.ts+12−19
@@ -414,18 +414,18 @@ const introspectHeadersForConnection = ( · 413 unmodified lines
414 return { headers, queryParams };414 return { headers, queryParams };
415};415};
416 416
417/** Resolve a config's introspection snapshot text: legacy rows inline it in417/** Resolve a config's introspection snapshot text from the plugin blob store
418 * `introspectionJson`; new rows point at the plugin blob store via418 * (`introspectionHash`). Null when the integration has no snapshot (live
419 * `introspectionHash`. Null when the integration has no snapshot (live419 * introspection territory). Pre-blob rows that inlined the JSON are
420 * introspection territory). */420 * rewritten by the introspection-to-blob migrations before this code reads
421 * them. */
421const loadIntrospectionJson = (422const loadIntrospectionJson = (
422 storage: GraphqlStore,423 storage: GraphqlStore,
423 config: GraphqlIntegrationConfig,424 config: GraphqlIntegrationConfig,
424): Effect.Effect<string | null, StorageFailure> => {425): Effect.Effect<string | null, StorageFailure> =>
425 if (config.introspectionJson != null) return Effect.succeed(config.introspectionJson);426 config.introspectionHash != null
426 if (config.introspectionHash != null) return storage.getIntrospection(config.introspectionHash);427 ? storage.getIntrospection(config.introspectionHash)
427 return Effect.succeed(null);428 : Effect.succeed(null);
428};
429 429
430/** Introspect a config live or from its stored snapshot, applying connection430/** Introspect a config live or from its stored snapshot, applying connection
431 * auth. A non-null `introspectionJson` (loaded via `loadIntrospectionJson`)431 * auth. A non-null `introspectionJson` (loaded via `loadIntrospectionJson`)
@@ -558,9 +558,6 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 126 unmodified lines
558 GraphqlIntegrationConfig.make({558 GraphqlIntegrationConfig.make({
559 endpoint: input.endpoint,559 endpoint: input.endpoint,
560 name: input.name?.trim() || slugFromEndpoint(input.endpoint),560 name: input.name?.trim() || slugFromEndpoint(input.endpoint),
561 ...(input.introspectionJson !== undefined
562 ? { introspectionJson: input.introspectionJson }
563 : {}),
564 ...(input.headers !== undefined ? { headers: input.headers } : {}),561 ...(input.headers !== undefined ? { headers: input.headers } : {}),
565 ...(input.queryParams !== undefined ? { queryParams: input.queryParams } : {}),562 ...(input.queryParams !== undefined ? { queryParams: input.queryParams } : {}),
566 authenticationTemplate: input.authenticationTemplate563 authenticationTemplate: input.authenticationTemplate
@@ -602,7 +599,7 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 35 unmodified lines
602 // their operation bindings) are produced lazily when a connection is599 // their operation bindings) are produced lazily when a connection is
603 // created (`resolveTools`) / a tool is first invoked (`invokeTool`),600 // created (`resolveTools`) / a tool is first invoked (`invokeTool`),
604 // using that connection's credential.601 // using that connection's credential.
605 if (baseConfig.introspectionJson === undefined) {602 if (input.introspectionJson === undefined) {
606 yield* ctx.transaction(603 yield* ctx.transaction(
607 ctx.core.integrations.register({604 ctx.core.integrations.register({
608 slug,605 slug,
@@ -617,7 +614,7 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 8 unmodified lines
617 614
618 // Pre-supplied introspection JSON: parse it offline (no network) and615 // Pre-supplied introspection JSON: parse it offline (no network) and
619 // persist the operation bindings + snapshot so production stays offline.616 // persist the operation bindings + snapshot so production stays offline.
620 const introspection = yield* parseIntrospectionJson(baseConfig.introspectionJson);617 const introspection = yield* parseIntrospectionJson(input.introspectionJson);
621 const { result } = yield* extract(introspection);618 const { result } = yield* extract(introspection);
622 const prepared = prepareOperations(result.fields, introspection);619 const prepared = prepareOperations(result.fields, introspection);
623 620
@@ -629,9 +626,8 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 5 unmodified lines
629 // carries only its hash.626 // carries only its hash.
630 const snapshotJson = JSON.stringify({ data: introspection });627 const snapshotJson = JSON.stringify({ data: introspection });
631 const introspectionHash = yield* sha256Hex(snapshotJson);628 const introspectionHash = yield* sha256Hex(snapshotJson);
632 const { introspectionJson: _inline, ...withoutInline } = baseConfig;
633 const config = GraphqlIntegrationConfig.make({629 const config = GraphqlIntegrationConfig.make({
634 ...withoutInline,630 ...baseConfig,
635 introspectionHash,631 introspectionHash,
636 });632 });
637 633
@@ -677,9 +673,6 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 39 unmodified lines
677 const next = GraphqlIntegrationConfig.make({673 const next = GraphqlIntegrationConfig.make({
678 endpoint: input.endpoint ?? current.endpoint,674 endpoint: input.endpoint ?? current.endpoint,
679 name: input.name?.trim() || current.name,675 name: input.name?.trim() || current.name,
680 ...(current.introspectionJson !== undefined
681 ? { introspectionJson: current.introspectionJson }
682 : {}),
683 ...(current.introspectionHash !== undefined676 ...(current.introspectionHash !== undefined
684 ? { introspectionHash: current.introspectionHash }677 ? { introspectionHash: current.introspectionHash }
685 : {}),678 : {}),
packages/plugins/graphql/src/sdk/types.ts+3−5
@@ -172,12 +172,10 @@ export const GraphqlIntegrationConfig = Schema.Struct({ · 171 unmodified lines
172 endpoint: Schema.String,172 endpoint: Schema.String,
173 /** Display name for the integration. */173 /** Display name for the integration. */
174 name: Schema.String,174 name: Schema.String,
175 /** Inlined introspection JSON text. Legacy rows only: new snapshots live in
176 * the plugin blob store and the config carries `introspectionHash` instead,
177 * so a multi-MB schema never rides along on catalog reads. */
178 introspectionJson: Schema.optional(Schema.String),
179 /** Hex SHA-256 of the introspection JSON snapshot — the content address of175 /** Hex SHA-256 of the introspection JSON snapshot — the content address of
180 * the blob (`introspection/<hash>` in the plugin blob store). */176 * the blob (`introspection/<hash>` in the plugin blob store). Rows that
177 * predate the blob store (inline `introspectionJson` text) are rewritten
178 * by the introspection-to-blob migrations before this schema sees them. */
181 introspectionHash: Schema.optional(Schema.String),179 introspectionHash: Schema.optional(Schema.String),
182 /** Static headers applied to every request (and to add-time introspection). */180 /** Static headers applied to every request (and to add-time introspection). */
183 headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),181 headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
packages/plugins/openapi/src/sdk/config.ts+6−7
@@ -15,13 +15,16 @@ import type { Authentication } from "./types"; · 14 unmodified lines
15//15//
16// In v2 there are NO credential bindings, NO per-source secret slots, and NO16// In v2 there are NO credential bindings, NO per-source secret slots, and NO
17// StoredSource credential config. The config carries only:17// StoredSource credential config. The config carries only:
18// - the content hash of the spec blob (legacy rows inline the text instead)18// - the content hash of the spec blob and/or the source URL to (re)fetch
19// and/or the source URL to (re)fetch from,19// from,
20// - the optional base URL override,20// - the optional base URL override,
21// - the auth templates a connection's value is rendered through.21// - the auth templates a connection's value is rendered through.
22// The resolved spec text itself lives in the plugin blob store, keyed22// The resolved spec text itself lives in the plugin blob store, keyed
23// `spec/<specHash>` — it's a build input for resolveTools/refresh, not data23// `spec/<specHash>` — it's a build input for resolveTools/refresh, not data
24// any list/invoke path should pay to load.24// any list/invoke path should pay to load. Rows that predate the blob store
25// (inline `spec` text) are rewritten before this schema sees them: cloud by
26// the out-of-band migrate-specs-to-blobs script, the libSQL hosts by the
27// boot-time ledger migration.
25// ---------------------------------------------------------------------------28// ---------------------------------------------------------------------------
26 29
27const OAuthAuthenticationSchema = Schema.Struct({30const OAuthAuthenticationSchema = Schema.Struct({
@@ -35,10 +38,6 @@ const OAuthAuthenticationSchema = Schema.Struct({ · 7 unmodified lines
35export const AuthenticationSchema = Schema.Union([OAuthAuthenticationSchema, ApiKeyAuthMethod]);38export const AuthenticationSchema = Schema.Union([OAuthAuthenticationSchema, ApiKeyAuthMethod]);
36 39
37export const OpenApiIntegrationConfigSchema = Schema.Struct({40export const OpenApiIntegrationConfigSchema = Schema.Struct({
38 /** Inlined OpenAPI document text. Legacy rows only: new registrations store
39 * the resolved text in the plugin blob store and carry `specHash` instead,
40 * so multi-MB documents never ride along on catalog reads. */
41 spec: Schema.optional(Schema.String),
42 /** Hex SHA-256 of the resolved spec text — the content address of the spec41 /** Hex SHA-256 of the resolved spec text — the content address of the spec
43 * blob (`spec/<hash>` in the plugin blob store). */42 * blob (`spec/<hash>` in the plugin blob store). */
44 specHash: Schema.optional(Schema.String),43 specHash: Schema.optional(Schema.String),
packages/plugins/openapi/src/sdk/plugin.ts+4−8
@@ -542,19 +542,15 @@ export const describeOpenApiIntegrationDisplay = ( · 541 unmodified lines
542 542
543// ---------------------------------------------------------------------------543// ---------------------------------------------------------------------------
544// Spec text resolution — the stored config carries the spec's content hash544// Spec text resolution — the stored config carries the spec's content hash
545// (`specHash` → blob `spec/<hash>`); legacy rows still inline the text in545// (`specHash` → blob `spec/<hash>`). Pre-blob rows that inlined the text are
546// `spec`. Every reader goes through this loader so both shapes work until the546// rewritten by the spec-to-blob migrations before this code reads them.
547// inline rows are backfilled.
548// ---------------------------------------------------------------------------547// ---------------------------------------------------------------------------
549 548
550const loadSpecText = (549const loadSpecText = (
551 storage: OpenapiStore,550 storage: OpenapiStore,
552 config: OpenApiIntegrationConfig,551 config: OpenApiIntegrationConfig,
553): Effect.Effect<string | null, StorageFailure> => {552): Effect.Effect<string | null, StorageFailure> =>
554 if (config.spec != null) return Effect.succeed(config.spec);553 config.specHash != null ? storage.getSpec(config.specHash) : Effect.succeed(null);
555 if (config.specHash != null) return storage.getSpec(config.specHash);
556 return Effect.succeed(null);
557};
558 554
559// ---------------------------------------------------------------------------555// ---------------------------------------------------------------------------
560// Spec → tool definitions (shared by addSpec, resolveTools, and detect)556// Spec → tool definitions (shared by addSpec, resolveTools, and detect)
packages/plugins/openapi/src/sdk/spec-blob.test.ts+35−26
@@ -85,7 +85,7 @@ describe("OpenAPI plugin — spec blob storage", () => { · 84 unmodified lines
85 });85 });
86 86
87 const config = yield* executor.openapi.getConfig("blob_api");87 const config = yield* executor.openapi.getConfig("blob_api");
88 expect(config?.spec).toBeUndefined();88 expect(Object.keys(config ?? {})).not.toContain("spec");
89 expect(config?.specHash).toBe(yield* sha256Hex(text));89 expect(config?.specHash).toBe(yield* sha256Hex(text));
90 }),90 }),
91 ),91 ),
@@ -126,41 +126,50 @@ describe("OpenAPI plugin — spec blob storage", () => { · 34 unmodified lines
126 ),126 ),
127 );127 );
128 128
129 it.effect("resolveTools still derives tools from a legacy inline-spec config", () =>129 it.effect("resolveTools reads the spec from the store, never an inline field", () =>
130 Effect.gen(function* () {130 Effect.gen(function* () {
131 const plugin = openApiPlugin({ httpClientLayer: FetchHttpClient.layer });131 const plugin = openApiPlugin({ httpClientLayer: FetchHttpClient.layer });
132 // A legacy row inlines the spec; the loader must never touch the store.132 const text = specText();
133 const hash = yield* sha256Hex(text);
133 const storage: OpenapiStore = {134 const storage: OpenapiStore = {
134 putOperations: () => Effect.void,135 putOperations: () => Effect.void,
135 getOperation: () => Effect.succeed(null),136 getOperation: () => Effect.succeed(null),
136 listOperations: () => Effect.succeed([]),137 listOperations: () => Effect.succeed([]),
137 removeOperations: () => Effect.void,138 removeOperations: () => Effect.void,
138 putSpec: () => Effect.void,139 putSpec: () => Effect.void,
139 getSpec: () => Effect.succeed(null),140 getSpec: (specHash) => Effect.succeed(specHash === hash ? text : null),
140 };141 };
141 142
142 const result = yield* plugin.resolveTools!({143 const resolve = (config: IntegrationConfig) =>
143 integration: {144 plugin.resolveTools!({
144 slug: IntegrationSlug.make("legacy_api"),145 integration: {
145 description: "legacy",146 slug: IntegrationSlug.make("pointer_api"),
146 kind: "openapi",147 description: "pointer",
147 canRemove: true,148 kind: "openapi",
148 canRefresh: false,149 canRemove: true,
149 authMethods: [],150 canRefresh: false,
150 },151 authMethods: [],
151 config: { spec: specText() } as IntegrationConfig,152 },
152 connection: {153 config,
153 owner: "org",154 connection: {
154 integration: IntegrationSlug.make("legacy_api"),155 owner: "org",
155 name: ConnectionName.make("main"),156 integration: IntegrationSlug.make("pointer_api"),
156 },157 name: ConnectionName.make("main"),
157 template: null,158 },
158 storage,159 template: null,
159 getValue: () => Effect.succeed(null),160 storage,
160 getValues: () => Effect.succeed({}),161 getValue: () => Effect.succeed(null),
161 });162 getValues: () => Effect.succeed({}),
162 163 });
163 expect(result.tools.map((tool) => String(tool.name))).toContain("items.echoHeaders");164
165 const fromPointer = yield* resolve({ specHash: hash } as IntegrationConfig);
166 expect(fromPointer.tools.map((tool) => String(tool.name))).toContain("items.echoHeaders");
167
168 // A pre-migration row that still inlines `spec` yields no tools: the
169 // spec-to-blob migrations rewrite those rows before this code runs, so
170 // the runtime carries no inline-read path.
171 const fromInline = yield* resolve({ spec: text } as IntegrationConfig);
172 expect(fromInline.tools).toHaveLength(0);
164 }),173 }),
165 );174 );
166 175

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t