History

Drop the inline-spec read paths; pointer configs are the only shape (#969)

With cloud backfilled out-of-band and the libSQL hosts rewriting at boot through the data-migration ledger, every database converges on the pointer shape (specHash / introspectionHash) before this code reads it. The transitional dual-read loaders and the optional inline fields (openapi config.spec, graphql config.introspectionJson) are gone from the stored-config schemas; addIntegration still ACCEPTS inline introspection JSON as input, but persists only the blob hash. The graphql getConfig endpoint also stops serving the introspection snapshot — same reasoning as the openapi spec removal: it's a multi-MB build artifact no client reads.

Rhys Sullivan committed 5 months ago

Commit 9b8baa11 parent 8a74ca07 files · +64 −69Browse files

Unified · Split

7 files
  1. packages/plugins/graphql/src/api/group.ts +4 −3
  2. packages/plugins/graphql/src/api/handlers.ts +0 −1
  3. packages/plugins/graphql/src/sdk/plugin.ts +12 −19
  4. packages/plugins/graphql/src/sdk/types.ts +3 −5
  5. packages/plugins/openapi/src/sdk/config.ts +6 −7
  6. packages/plugins/openapi/src/sdk/plugin.ts +4 −8
  7. packages/plugins/openapi/src/sdk/spec-blob.test.ts +35 −26
packages/plugins/graphql/src/api/group.ts+4−3
@@ -45,12 +45,13 @@ const AddIntegrationResponse = Schema.Struct({ · 44 unmodified lines
4545 name: Schema.String,
4646 });
4747
48−// The full opaque integration config, surfaced for the configure UX. Carries
49−// the `authenticationTemplate` the configure / custom-method flow reads/writes.
48+// The integration config surfaced for the configure UX. Carries the
49+// `authenticationTemplate` the configure / custom-method flow reads/writes.
50+// The introspection snapshot is deliberately NOT served: it's a multi-MB
51+// build artifact in the plugin blob store, and no client reads it.
5052 const GraphqlConfigView = Schema.Struct({
5153 endpoint: Schema.String,
5254 name: Schema.String,
53− introspectionJson: Schema.optional(Schema.String),
5455 headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
5556 queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)),
5657 authenticationTemplate: Schema.Array(GraphqlAuthMethod),
packages/plugins/graphql/src/api/handlers.ts+0−1
@@ -66,7 +66,6 @@ export const GraphqlHandlers = HttpApiBuilder.group(ExecutorApiWithGraphql, "gra · 65 unmodified lines
6666 ? {
6767 endpoint: config.endpoint,
6868 name: config.name,
69− introspectionJson: config.introspectionJson,
7069 headers: config.headers ? { ...config.headers } : undefined,
7170 queryParams: config.queryParams ? { ...config.queryParams } : undefined,
7271 authenticationTemplate: [...config.authenticationTemplate],
packages/plugins/graphql/src/sdk/plugin.ts+12−19
@@ -414,18 +414,18 @@ const introspectHeadersForConnection = ( · 413 unmodified lines
414414 return { headers, queryParams };
415415 };
416416
417−/** Resolve a config's introspection snapshot text: legacy rows inline it in
418− * `introspectionJson`; new rows point at the plugin blob store via
419− * `introspectionHash`. Null when the integration has no snapshot (live
420− * introspection territory). */
417+/** Resolve a config's introspection snapshot text from the plugin blob store
418+ * (`introspectionHash`). Null when the integration has no snapshot (live
419+ * introspection territory). Pre-blob rows that inlined the JSON are
420+ * rewritten by the introspection-to-blob migrations before this code reads
421+ * them. */
421422 const loadIntrospectionJson = (
422423 storage: GraphqlStore,
423424 config: GraphqlIntegrationConfig,
424−): Effect.Effect<string | null, StorageFailure> => {
425− if (config.introspectionJson != null) return Effect.succeed(config.introspectionJson);
426− if (config.introspectionHash != null) return storage.getIntrospection(config.introspectionHash);
427− return Effect.succeed(null);
428−};
425+): Effect.Effect<string | null, StorageFailure> =>
426+ config.introspectionHash != null
427+ ? storage.getIntrospection(config.introspectionHash)
428+ : Effect.succeed(null);
429429
430430 /** Introspect a config live or from its stored snapshot, applying connection
431431 * auth. A non-null `introspectionJson` (loaded via `loadIntrospectionJson`)
@@ -558,9 +558,6 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 126 unmodified lines
558558 GraphqlIntegrationConfig.make({
559559 endpoint: input.endpoint,
560560 name: input.name?.trim() || slugFromEndpoint(input.endpoint),
561− ...(input.introspectionJson !== undefined
562− ? { introspectionJson: input.introspectionJson }
563− : {}),
564561 ...(input.headers !== undefined ? { headers: input.headers } : {}),
565562 ...(input.queryParams !== undefined ? { queryParams: input.queryParams } : {}),
566563 authenticationTemplate: input.authenticationTemplate
@@ -602,7 +599,7 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 35 unmodified lines
602599 // their operation bindings) are produced lazily when a connection is
603600 // created (`resolveTools`) / a tool is first invoked (`invokeTool`),
604601 // using that connection's credential.
605− if (baseConfig.introspectionJson === undefined) {
602+ if (input.introspectionJson === undefined) {
606603 yield* ctx.transaction(
607604 ctx.core.integrations.register({
608605 slug,
@@ -617,7 +614,7 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 8 unmodified lines
617614
618615 // Pre-supplied introspection JSON: parse it offline (no network) and
619616 // persist the operation bindings + snapshot so production stays offline.
620− const introspection = yield* parseIntrospectionJson(baseConfig.introspectionJson);
617+ const introspection = yield* parseIntrospectionJson(input.introspectionJson);
621618 const { result } = yield* extract(introspection);
622619 const prepared = prepareOperations(result.fields, introspection);
623620
@@ -629,9 +626,8 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 5 unmodified lines
629626 // carries only its hash.
630627 const snapshotJson = JSON.stringify({ data: introspection });
631628 const introspectionHash = yield* sha256Hex(snapshotJson);
632− const { introspectionJson: _inline, ...withoutInline } = baseConfig;
633629 const config = GraphqlIntegrationConfig.make({
634− ...withoutInline,
630+ ...baseConfig,
635631 introspectionHash,
636632 });
637633
@@ -677,9 +673,6 @@ const makeGraphqlExtension = (ctx: PluginCtx<GraphqlStore>) => { · 39 unmodified lines
677673 const next = GraphqlIntegrationConfig.make({
678674 endpoint: input.endpoint ?? current.endpoint,
679675 name: input.name?.trim() || current.name,
680− ...(current.introspectionJson !== undefined
681− ? { introspectionJson: current.introspectionJson }
682− : {}),
683676 ...(current.introspectionHash !== undefined
684677 ? { introspectionHash: current.introspectionHash }
685678 : {}),
packages/plugins/graphql/src/sdk/types.ts+3−5
@@ -172,12 +172,10 @@ export const GraphqlIntegrationConfig = Schema.Struct({ · 171 unmodified lines
172172 endpoint: Schema.String,
173173 /** Display name for the integration. */
174174 name: Schema.String,
175− /** Inlined introspection JSON text. Legacy rows only: new snapshots live in
176− * the plugin blob store and the config carries `introspectionHash` instead,
177− * so a multi-MB schema never rides along on catalog reads. */
178− introspectionJson: Schema.optional(Schema.String),
179175 /** Hex SHA-256 of the introspection JSON snapshot — the content address of
180− * the blob (`introspection/<hash>` in the plugin blob store). */
176+ * the blob (`introspection/<hash>` in the plugin blob store). Rows that
177+ * predate the blob store (inline `introspectionJson` text) are rewritten
178+ * by the introspection-to-blob migrations before this schema sees them. */
181179 introspectionHash: Schema.optional(Schema.String),
182180 /** Static headers applied to every request (and to add-time introspection). */
183181 headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
packages/plugins/openapi/src/sdk/config.ts+6−7
@@ -15,13 +15,16 @@ import type { Authentication } from "./types"; · 14 unmodified lines
1515 //
1616 // In v2 there are NO credential bindings, NO per-source secret slots, and NO
1717 // StoredSource credential config. The config carries only:
18−// - the content hash of the spec blob (legacy rows inline the text instead)
19−// and/or the source URL to (re)fetch from,
18+// - the content hash of the spec blob and/or the source URL to (re)fetch
19+// from,
2020 // - the optional base URL override,
2121 // - the auth templates a connection's value is rendered through.
2222 // The resolved spec text itself lives in the plugin blob store, keyed
2323 // `spec/<specHash>` — it's a build input for resolveTools/refresh, not data
24−// any list/invoke path should pay to load.
24+// any list/invoke path should pay to load. Rows that predate the blob store
25+// (inline `spec` text) are rewritten before this schema sees them: cloud by
26+// the out-of-band migrate-specs-to-blobs script, the libSQL hosts by the
27+// boot-time ledger migration.
2528 // ---------------------------------------------------------------------------
2629
2730 const OAuthAuthenticationSchema = Schema.Struct({
@@ -35,10 +38,6 @@ const OAuthAuthenticationSchema = Schema.Struct({ · 7 unmodified lines
3538 export const AuthenticationSchema = Schema.Union([OAuthAuthenticationSchema, ApiKeyAuthMethod]);
3639
3740 export const OpenApiIntegrationConfigSchema = Schema.Struct({
38− /** Inlined OpenAPI document text. Legacy rows only: new registrations store
39− * the resolved text in the plugin blob store and carry `specHash` instead,
40− * so multi-MB documents never ride along on catalog reads. */
41− spec: Schema.optional(Schema.String),
4241 /** Hex SHA-256 of the resolved spec text — the content address of the spec
4342 * blob (`spec/<hash>` in the plugin blob store). */
4443 specHash: Schema.optional(Schema.String),
packages/plugins/openapi/src/sdk/plugin.ts+4−8
@@ -542,19 +542,15 @@ export const describeOpenApiIntegrationDisplay = ( · 541 unmodified lines
542542
543543 // ---------------------------------------------------------------------------
544544 // Spec text resolution — the stored config carries the spec's content hash
545−// (`specHash` → blob `spec/<hash>`); legacy rows still inline the text in
546−// `spec`. Every reader goes through this loader so both shapes work until the
547−// inline rows are backfilled.
545+// (`specHash` → blob `spec/<hash>`). Pre-blob rows that inlined the text are
546+// rewritten by the spec-to-blob migrations before this code reads them.
548547 // ---------------------------------------------------------------------------
549548
550549 const loadSpecText = (
551550 storage: OpenapiStore,
552551 config: OpenApiIntegrationConfig,
553−): Effect.Effect<string | null, StorageFailure> => {
554− if (config.spec != null) return Effect.succeed(config.spec);
555− if (config.specHash != null) return storage.getSpec(config.specHash);
556− return Effect.succeed(null);
557−};
552+): Effect.Effect<string | null, StorageFailure> =>
553+ config.specHash != null ? storage.getSpec(config.specHash) : Effect.succeed(null);
558554
559555 // ---------------------------------------------------------------------------
560556 // Spec → tool definitions (shared by addSpec, resolveTools, and detect)
packages/plugins/openapi/src/sdk/spec-blob.test.ts+35−26
@@ -85,7 +85,7 @@ describe("OpenAPI plugin — spec blob storage", () => { · 84 unmodified lines
8585 });
8686
8787 const config = yield* executor.openapi.getConfig("blob_api");
88− expect(config?.spec).toBeUndefined();
88+ expect(Object.keys(config ?? {})).not.toContain("spec");
8989 expect(config?.specHash).toBe(yield* sha256Hex(text));
9090 }),
9191 ),
@@ -126,41 +126,50 @@ describe("OpenAPI plugin — spec blob storage", () => { · 34 unmodified lines
126126 ),
127127 );
128128
129− it.effect("resolveTools still derives tools from a legacy inline-spec config", () =>
129+ it.effect("resolveTools reads the spec from the store, never an inline field", () =>
130130 Effect.gen(function* () {
131131 const plugin = openApiPlugin({ httpClientLayer: FetchHttpClient.layer });
132− // A legacy row inlines the spec; the loader must never touch the store.
132+ const text = specText();
133+ const hash = yield* sha256Hex(text);
133134 const storage: OpenapiStore = {
134135 putOperations: () => Effect.void,
135136 getOperation: () => Effect.succeed(null),
136137 listOperations: () => Effect.succeed([]),
137138 removeOperations: () => Effect.void,
138139 putSpec: () => Effect.void,
139− getSpec: () => Effect.succeed(null),
140+ getSpec: (specHash) => Effect.succeed(specHash === hash ? text : null),
140141 };
141142
142− const result = yield* plugin.resolveTools!({
143− integration: {
144− slug: IntegrationSlug.make("legacy_api"),
145− description: "legacy",
146− kind: "openapi",
147− canRemove: true,
148− canRefresh: false,
149− authMethods: [],
150− },
151− config: { spec: specText() } as IntegrationConfig,
152− connection: {
153− owner: "org",
154− integration: IntegrationSlug.make("legacy_api"),
155− name: ConnectionName.make("main"),
156− },
157− template: null,
158− storage,
159− getValue: () => Effect.succeed(null),
160− getValues: () => Effect.succeed({}),
161− });
162−
163− expect(result.tools.map((tool) => String(tool.name))).toContain("items.echoHeaders");
143+ const resolve = (config: IntegrationConfig) =>
144+ plugin.resolveTools!({
145+ integration: {
146+ slug: IntegrationSlug.make("pointer_api"),
147+ description: "pointer",
148+ kind: "openapi",
149+ canRemove: true,
150+ canRefresh: false,
151+ authMethods: [],
152+ },
153+ config,
154+ connection: {
155+ owner: "org",
156+ integration: IntegrationSlug.make("pointer_api"),
157+ name: ConnectionName.make("main"),
158+ },
159+ template: null,
160+ storage,
161+ getValue: () => Effect.succeed(null),
162+ getValues: () => Effect.succeed({}),
163+ });
164+
165+ const fromPointer = yield* resolve({ specHash: hash } as IntegrationConfig);
166+ expect(fromPointer.tools.map((tool) => String(tool.name))).toContain("items.echoHeaders");
167+
168+ // A pre-migration row that still inlines `spec` yields no tools: the
169+ // spec-to-blob migrations rewrite those rows before this code runs, so
170+ // the runtime carries no inline-read path.
171+ const fromInline = yield* resolve({ spec: text } as IntegrationConfig);
172+ expect(fromInline.tools).toHaveLength(0);
164173 }),
165174 );
166175

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t