History

feat: add source-builtins package and enhance source discovery

- Introduced the new package `@executor/source-builtins` with dependencies on core source packages. - Updated `bun.lock` and `package.json` to include `@executor/source-builtins`. - Enhanced source discovery logic to utilize built-in source adapters for improved detection. - Refactored source adapters to streamline the integration of local configuration binding schemas. - Improved error handling and type definitions across various source-related modules.

Rhys Sullivan committed 8 months ago

Commit 7df9c201 parent 1e3200e25 files · +1029 −356Browse files

Unified · Split

25 files
  1. bun.lock +18 −0
  2. packages/platform/control-plane/package.json +1 −0
  3. packages/platform/control-plane/src/api/sources/http.ts +22 −13
  4. packages/platform/control-plane/src/runtime/catalog/source/snapshot.test.ts +1 −1
  5. packages/platform/control-plane/src/runtime/catalog/source/sync.ts +2 −2
  6. packages/platform/control-plane/src/runtime/control-plane-runtime.test.ts +361 −0
  7. packages/platform/control-plane/src/runtime/local/source-artifacts.ts +27 −4
  8. packages/platform/control-plane/src/runtime/local/storage.ts +1 −1
  9. packages/platform/control-plane/src/runtime/sources/source-adapters/index.ts +28 −46
  10. packages/platform/control-plane/src/runtime/sources/source-adapters/internal.ts +4 −2
  11. packages/platform/control-plane/src/runtime/sources/source-auth-service.ts +236 −144
  12. packages/platform/control-plane/src/runtime/sources/source-discovery.ts +28 −51
  13. packages/platform/control-plane/src/runtime/sources/source-store/config.ts +13 −42
  14. packages/platform/control-plane/src/schema/models/local-config.ts +36 −42
  15. packages/sources/builtins/package.json +26 −0
  16. packages/sources/builtins/src/index.ts +12 −0
  17. packages/sources/builtins/tsconfig.json +15 −0
  18. packages/sources/core/src/composition.ts +88 −0
  19. packages/sources/core/src/index.ts +1 −0
  20. packages/sources/core/src/registry.ts +4 −0
  21. packages/sources/core/src/types.ts +14 −0
  22. packages/sources/google-discovery/src/adapter.ts +27 −2
  23. packages/sources/graphql/src/adapter.ts +19 −2
  24. packages/sources/mcp/src/adapter.ts +25 −2
  25. packages/sources/openapi/src/adapter.ts +20 −2
bun.lock+18−0
@@ -271,6 +271,7 @@ · 270 unmodified lines
271271 "@executor/runtime-deno-subprocess": "workspace:*",
272272 "@executor/runtime-quickjs": "workspace:*",
273273 "@executor/runtime-ses": "workspace:*",
274+ "@executor/source-builtins": "workspace:*",
274275 "@executor/source-core": "workspace:*",
275276 "@executor/source-google-discovery": "workspace:*",
276277 "@executor/source-graphql": "workspace:*",
@@ -316,6 +317,21 @@ · 39 unmodified lines
316317 "bun-types": "catalog:",
317318 },
318319 },
320+ "packages/sources/builtins": {
321+ "name": "@executor/source-builtins",
322+ "version": "0.0.0",
323+ "dependencies": {
324+ "@executor/source-core": "workspace:*",
325+ "@executor/source-google-discovery": "workspace:*",
326+ "@executor/source-graphql": "workspace:*",
327+ "@executor/source-mcp": "workspace:*",
328+ "@executor/source-openapi": "workspace:*",
329+ },
330+ "devDependencies": {
331+ "@types/node": "catalog:",
332+ "bun-types": "catalog:",
333+ },
334+ },
319335 "packages/sources/core": {
320336 "name": "@executor/source-core",
321337 "version": "0.0.0",
@@ -636,6 +652,8 @@ · 314 unmodified lines
636652
637653 "@executor/server": ["@executor/server@workspace:packages/platform/server"],
638654
655+ "@executor/source-builtins": ["@executor/source-builtins@workspace:packages/sources/builtins"],
656+
639657 "@executor/source-core": ["@executor/source-core@workspace:packages/sources/core"],
640658
641659 "@executor/source-google-discovery": ["@executor/source-google-discovery@workspace:packages/sources/google-discovery"],
packages/platform/control-plane/package.json+1−0
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 "dependencies": {
2222 "@effect/platform": "catalog:",
2323 "@executor/source-core": "workspace:*",
24+ "@executor/source-builtins": "workspace:*",
2425 "@executor/auth-mcp-oauth": "workspace:*",
2526 "@executor/auth-oauth2": "workspace:*",
2627 "@executor/codemode-core": "workspace:*",
packages/platform/control-plane/src/api/sources/http.ts+22−13
@@ -25,7 +25,10 @@ import { · 24 unmodified lines
2525 } from "../../runtime/local/operations";
2626 import { discoverSource } from "../../runtime/sources/source-discovery";
2727 import { sourceAdapterRequiresInteractiveConnect } from "../../runtime/sources/source-adapters";
28−import { RuntimeSourceAuthServiceTag } from "../../runtime/sources/source-auth-service";
28+import {
29+ RuntimeSourceAuthServiceTag,
30+ type ExecutorAddSourceInput,
31+} from "../../runtime/sources/source-auth-service";
2932
3033 import {
3134 ControlPlaneBadRequestError,
@@ -33,6 +36,7 @@ import { · 1 unmodified line
3336 ControlPlaneStorageError,
3437 } from "../errors";
3538 import { ControlPlaneApi } from "../api";
39+import type { ConnectSourcePayload } from "./api";
3640 import { resolveRequestedLocalWorkspace } from "../local-context";
3741
3842 const readHeader = (headers: unknown, name: string): string | null => {
@@ -79,6 +83,11 @@ const toBadRequestError = (operation: string, cause: unknown) => { · 40 unmodified lines
7983 });
8084 };
8185
86+const isInteractiveConnectPayload = (
87+ payload: ConnectSourcePayload,
88+): payload is Extract<ConnectSourcePayload, { kind?: "mcp" }> =>
89+ payload.kind === undefined || sourceAdapterRequiresInteractiveConnect(payload.kind);
90+
8291 const escapeHtml = (value: string): string =>
8392 value
8493 .replaceAll("&", "&amp;")
@@ -677,8 +686,8 @@ export const ControlPlaneSourcesLive = HttpApiBuilder.group( · 592 unmodified lines
677686 const sourceAuthService = yield* RuntimeSourceAuthServiceTag;
678687 const baseUrl = resolveRequestOrigin(request);
679688
680− if (sourceAdapterRequiresInteractiveConnect(payload.kind)) {
681− return yield* sourceAuthService.connectMcpSource({
689+ if (isInteractiveConnectPayload(payload)) {
690+ return yield* sourceAuthService.connectMcpSource({
682691 workspaceId: path.workspaceId,
683692 actorAccountId: runtimeLocalWorkspace.installation.accountId,
684693 endpoint: payload.endpoint,
@@ -695,16 +704,16 @@ export const ControlPlaneSourcesLive = HttpApiBuilder.group( · 10 unmodified lines
695704 });
696705 }
697706
698− return yield* sourceAuthService.addExecutorSource(
699− {
700− workspaceId: path.workspaceId,
701− actorAccountId: runtimeLocalWorkspace.installation.accountId,
702− executionId: null,
703− interactionId: null,
704− ...payload,
705− },
706− { baseUrl },
707− );
707+ return yield* sourceAuthService.addExecutorSource(
708+ ({
709+ workspaceId: path.workspaceId,
710+ actorAccountId: runtimeLocalWorkspace.installation.accountId,
711+ executionId: null,
712+ interactionId: null,
713+ ...(payload as Record<string, unknown>),
714+ } as ExecutorAddSourceInput),
715+ { baseUrl },
716+ );
708717 }).pipe(
709718 Effect.catchAll((cause) =>
710719 Effect.fail(toBadRequestError("sources.connect", cause)),
packages/platform/control-plane/src/runtime/catalog/source/snapshot.test.ts+1−1
@@ -6,7 +6,7 @@ import { · 5 unmodified lines
66 buildGraphqlToolPresentation,
77 compileGraphqlToolDefinitions,
88 type GraphqlToolManifest,
9−} from "../../sources/graphql-tools";
9+} from "@executor/source-graphql";
1010 import { createGoogleDiscoveryCatalogSnapshot } from "@executor/source-google-discovery";
1111 import { createGraphqlCatalogSnapshot } from "@executor/source-graphql";
1212 import { createMcpCatalogSnapshot } from "@executor/source-mcp";
packages/platform/control-plane/src/runtime/catalog/source/sync.ts+2−2
@@ -29,9 +29,9 @@ import { · 28 unmodified lines
2929 } from "../../sources/source-adapters";
3030 import {
3131 catalogSyncResultFromMcpManifest,
32−} from "../../sources/source-adapters/mcp";
32+} from "@executor/source-mcp";
3333 import { SecretMaterialResolverService } from "../../local/secret-material-providers";
34−import { snapshotFromSourceCatalogSyncResult } from "../../sources/catalog-sync-result";
34+import { snapshotFromSourceCatalogSyncResult } from "@executor/source-core";
3535 import {
3636 refreshSourceTypeDeclarationInBackground,
3737 } from "./type-declarations";
packages/platform/control-plane/src/runtime/control-plane-runtime.test.ts+361−0
@@ -21,6 +21,8 @@ import { · 20 unmodified lines
2121 WorkspaceOauthClientIdSchema,
2222 } from "#schema";
2323 import type { ToolPath } from "@executor/codemode-core";
24+import { createCatalogImportMetadata } from "@executor/source-core";
25+import { createGraphqlCatalogFragment } from "@executor/source-graphql";
2426
2527 import {
2628 type ControlPlaneRuntime,
@@ -33,6 +35,14 @@ import { decodeSourceCredentialSelectionContent } from "./sources/source-credent · 6 unmodified lines
3335 import { persistSource } from "./sources/source-store";
3436 import { withControlPlaneClient } from "./execution/test-http-client";
3537 import { runtimeEffectError } from "./effect-errors";
38+import { resolveLocalWorkspaceContext } from "./local/config";
39+import { writeLocalControlPlaneState } from "./local/control-plane-store";
40+import { deriveLocalInstallation } from "./local/installation";
41+import {
42+ buildLocalSourceArtifact,
43+ writeLocalSourceArtifact,
44+} from "./local/source-artifacts";
45+import { writeLocalWorkspaceState } from "./local/workspace-state";
3646
3747 const makeRuntime = Effect.gen(function* () {
3848 const fs = yield* FileSystem.FileSystem;
@@ -699,6 +709,188 @@ describe("control-plane-runtime", () => { · 660 unmodified lines
699709 }),
700710 );
701711
712+ it.scoped("loads a v1.2.3-style workspace artifact on startup", () =>
713+ Effect.gen(function* () {
714+ const fs = yield* FileSystem.FileSystem;
715+ const workspaceRoot = yield* fs.makeTempDirectoryScoped({
716+ prefix: "executor-control-plane-runtime-v123-",
717+ });
718+ const homeConfigPath = join(workspaceRoot, ".executor-home.jsonc");
719+ const homeStateDirectory = join(workspaceRoot, ".executor-home-state");
720+ const configDirectory = join(workspaceRoot, ".executor");
721+ const configPath = join(configDirectory, "executor.jsonc");
722+ const sourceId = SourceIdSchema.make("graphql");
723+ const now = Date.now();
724+
725+ yield* fs.makeDirectory(configDirectory, { recursive: true });
726+ yield* fs.writeFileString(
727+ configPath,
728+ `{
729+ "sources": {
730+ "graphql": {
731+ "kind": "graphql",
732+ "name": "GraphQL API",
733+ "namespace": "graphql",
734+ "connection": {
735+ "endpoint": "https://example.com/graphql"
736+ },
737+ "binding": {
738+ "defaultHeaders": null
739+ }
740+ }
741+ }
742+}
743+`,
744+ );
745+
746+ const context = yield* resolveLocalWorkspaceContext({
747+ workspaceRoot,
748+ homeConfigPath,
749+ homeStateDirectory,
750+ });
751+ const installation = deriveLocalInstallation(context);
752+ const source = yield* createSourceFromPayload({
753+ workspaceId: installation.workspaceId,
754+ sourceId,
755+ payload: {
756+ name: "GraphQL API",
757+ kind: "graphql",
758+ endpoint: "https://example.com/graphql",
759+ status: "connected",
760+ enabled: true,
761+ namespace: "graphql",
762+ importAuthPolicy: "reuse_runtime",
763+ binding: {
764+ defaultHeaders: null,
765+ },
766+ importAuth: { kind: "none" },
767+ auth: { kind: "none" },
768+ },
769+ now,
770+ }).pipe(Effect.orDie);
771+
772+ const artifact = buildLocalSourceArtifact({
773+ source,
774+ syncResult: {
775+ fragment: createGraphqlCatalogFragment({
776+ source,
777+ documents: [
778+ {
779+ documentKind: "graphql_introspection",
780+ documentKey: source.endpoint,
781+ contentText: '{"__schema":{}}',
782+ fetchedAt: now,
783+ },
784+ ],
785+ operations: [
786+ {
787+ toolId: "viewer",
788+ title: "Viewer",
789+ description: "Load the current viewer",
790+ effect: "read",
791+ inputSchema: { type: "object", properties: {} },
792+ outputSchema: {
793+ type: "object",
794+ properties: { login: { type: "string" } },
795+ },
796+ providerData: {
797+ kind: "graphql",
798+ toolKind: "field",
799+ toolId: "viewer",
800+ rawToolId: "viewer",
801+ group: "query",
802+ leaf: "viewer",
803+ fieldName: "viewer",
804+ operationType: "query",
805+ operationName: "ViewerQuery",
806+ operationDocument: "query ViewerQuery { viewer { login } }",
807+ queryTypeName: "Query",
808+ mutationTypeName: null,
809+ subscriptionTypeName: null,
810+ },
811+ },
812+ ],
813+ }),
814+ importMetadata: createCatalogImportMetadata({
815+ source,
816+ adapterKey: "graphql",
817+ }),
818+ sourceHash: source.sourceHash,
819+ },
820+ });
821+
822+ yield* writeLocalWorkspaceState({
823+ context,
824+ state: {
825+ version: 1,
826+ sources: {
827+ [sourceId]: {
828+ status: source.status,
829+ lastError: source.lastError,
830+ sourceHash: source.sourceHash,
831+ createdAt: source.createdAt,
832+ updatedAt: source.updatedAt,
833+ },
834+ },
835+ policies: {},
836+ },
837+ });
838+ yield* writeLocalControlPlaneState({
839+ context,
840+ state: {
841+ version: 1,
842+ authArtifacts: [],
843+ authLeases: [],
844+ sourceOauthClients: [],
845+ workspaceOauthClients: [],
846+ providerAuthGrants: [],
847+ sourceAuthSessions: [],
848+ secretMaterials: [],
849+ executions: [],
850+ executionInteractions: [],
851+ executionSteps: [],
852+ },
853+ });
854+ yield* writeLocalSourceArtifact({
855+ context,
856+ sourceId,
857+ artifact,
858+ });
859+
860+ const artifactPath = join(context.artifactsDirectory, "sources", `${sourceId}.json`);
861+ const legacyArtifact = JSON.parse(
862+ yield* fs.readFileString(artifactPath, "utf8"),
863+ ) as Record<string, unknown>;
864+ legacyArtifact.version = 3;
865+ yield* fs.writeFileString(artifactPath, `${JSON.stringify(legacyArtifact)}\n`);
866+
867+ const runtime = yield* Effect.acquireRelease(
868+ createControlPlaneRuntime({
869+ workspaceRoot,
870+ homeConfigPath,
871+ homeStateDirectory,
872+ }),
873+ (createdRuntime) => Effect.promise(() => createdRuntime.close()).pipe(Effect.orDie),
874+ );
875+
876+ const inspection = yield* withControlPlaneClient(
877+ { runtime, accountId: installation.accountId },
878+ (client) =>
879+ client.sources.inspection({
880+ path: {
881+ workspaceId: installation.workspaceId,
882+ sourceId,
883+ },
884+ }),
885+ );
886+
887+ expect(inspection.source.id).toBe(sourceId);
888+ expect(inspection.source.kind).toBe("graphql");
889+ expect(inspection.toolCount).toBe(1);
890+ expect(inspection.tools.map((tool) => tool.path)).toContain("graphql.viewer");
891+ }).pipe(Effect.provide(NodeFileSystem.layer)),
892+ );
893+
702894 it.scoped("returns an empty inspection bundle for auth-required sources without a catalog artifact", () =>
703895 Effect.gen(function* () {
704896 const runtime = yield* makeRuntime;
@@ -932,6 +1124,175 @@ describe("control-plane-runtime", () => { · 227 unmodified lines
9321124 }),
9331125 );
9341126
1127+ it.scoped("reuses existing Google provider grants for single connects and clears orphan state", () =>
1128+ Effect.gen(function* () {
1129+ const runtime = yield* makeRuntime;
1130+ const googleServer = yield* makeGoogleWorkspaceTestServer;
1131+ const installation = runtime.localInstallation;
1132+
1133+ const oauthClient = yield* withControlPlaneClient(
1134+ { runtime },
1135+ (client) =>
1136+ client.sources.createWorkspaceOauthClient({
1137+ path: {
1138+ workspaceId: installation.workspaceId,
1139+ },
1140+ payload: {
1141+ providerKey: "google_workspace",
1142+ label: "Local Google Workspace Client",
1143+ oauthClient: {
1144+ clientId: "google-client-id",
1145+ clientSecret: "google-client-secret",
1146+ },
1147+ },
1148+ }),
1149+ );
1150+
1151+ const refreshSecretId = "sec_google_refresh_single_reuse";
1152+ yield* upsertLocalSecret(runtime, {
1153+ id: refreshSecretId,
1154+ name: "Google Refresh Token",
1155+ purpose: "oauth_refresh_token",
1156+ value: "refresh-token-reuse",
1157+ });
1158+
1159+ const grantId = ProviderAuthGrantIdSchema.make(`provider_grant_${crypto.randomUUID()}`);
1160+ yield* runtime.persistence.rows.providerAuthGrants.upsert({
1161+ id: grantId,
1162+ workspaceId: installation.workspaceId,
1163+ actorAccountId: installation.accountId,
1164+ providerKey: "google_workspace",
1165+ oauthClientId: oauthClient.id,
1166+ tokenEndpoint: googleServer.tokenEndpoint,
1167+ clientAuthentication: "client_secret_post",
1168+ headerName: "Authorization",
1169+ prefix: "Bearer ",
1170+ refreshToken: {
1171+ providerId: "local",
1172+ handle: refreshSecretId,
1173+ },
1174+ grantedScopes: [
1175+ "scope:drive.readonly",
1176+ ],
1177+ lastRefreshedAt: null,
1178+ orphanedAt: 123,
1179+ createdAt: Date.now(),
1180+ updatedAt: Date.now(),
1181+ });
1182+
1183+ googleServer.queueTokenResponse({
1184+ access_token: "reuse-access-token",
1185+ scope: "scope:drive.readonly",
1186+ });
1187+
1188+ const result = yield* withControlPlaneClient(
1189+ { runtime },
1190+ (client) =>
1191+ client.sources.connect({
1192+ path: {
1193+ workspaceId: installation.workspaceId,
1194+ },
1195+ payload: {
1196+ kind: "google_discovery",
1197+ workspaceOauthClientId: oauthClient.id,
1198+ service: "drive",
1199+ version: "v3",
1200+ discoveryUrl: googleServer.discoveryUrl({
1201+ service: "drive",
1202+ version: "v3",
1203+ scope: "scope:drive.readonly",
1204+ }),
1205+ scopes: ["scope:drive.readonly"],
1206+ name: "Drive",
1207+ namespace: "google.drive",
1208+ },
1209+ }),
1210+ );
1211+
1212+ expect(result.kind).toBe("connected");
1213+ if (result.kind !== "connected") {
1214+ throw new Error(`Expected connected result, received ${result.kind}`);
1215+ }
1216+ expect(result.source.auth).toEqual({
1217+ kind: "provider_grant_ref",
1218+ grantId,
1219+ providerKey: "google_workspace",
1220+ requiredScopes: ["scope:drive.readonly"],
1221+ headerName: "Authorization",
1222+ prefix: "Bearer ",
1223+ });
1224+
1225+ const storedGrant = yield* runtime.persistence.rows.providerAuthGrants.getById(grantId);
1226+ assertTrue(Option.isSome(storedGrant));
1227+ expect(storedGrant.value.orphanedAt).toBeNull();
1228+ expect(googleServer.discoveryAuthorizations).toContain("Bearer reuse-access-token");
1229+ }),
1230+ 60_000,
1231+ );
1232+
1233+ it.scoped("uses the app callback redirect for browser-started Google single-source OAuth", () =>
1234+ Effect.gen(function* () {
1235+ const runtime = yield* makeRuntime;
1236+ const googleServer = yield* makeGoogleWorkspaceTestServer;
1237+ const installation = runtime.localInstallation;
1238+
1239+ const oauthClient = yield* withControlPlaneClient(
1240+ { runtime },
1241+ (client) =>
1242+ client.sources.createWorkspaceOauthClient({
1243+ path: {
1244+ workspaceId: installation.workspaceId,
1245+ },
1246+ payload: {
1247+ providerKey: "google_workspace",
1248+ label: "Local Google Workspace Client",
1249+ oauthClient: {
1250+ clientId: "google-client-id",
1251+ clientSecret: "google-client-secret",
1252+ },
1253+ },
1254+ }),
1255+ );
1256+
1257+ const result = yield* withControlPlaneClient(
1258+ { runtime },
1259+ (client) =>
1260+ client.sources.connect({
1261+ path: {
1262+ workspaceId: installation.workspaceId,
1263+ },
1264+ payload: {
1265+ kind: "google_discovery",
1266+ workspaceOauthClientId: oauthClient.id,
1267+ service: "gmail",
1268+ version: "v1",
1269+ discoveryUrl: googleServer.discoveryUrl({
1270+ service: "gmail",
1271+ version: "v1",
1272+ scope: "scope:gmail.readonly",
1273+ }),
1274+ scopes: ["scope:gmail.readonly"],
1275+ name: "Gmail",
1276+ namespace: "google.gmail",
1277+ },
1278+ }),
1279+ );
1280+
1281+ expect(result.kind).toBe("oauth_required");
1282+ if (result.kind !== "oauth_required") {
1283+ throw new Error(`Expected oauth_required result, received ${result.kind}`);
1284+ }
1285+
1286+ const redirectUri = new URL(result.authorizationUrl).searchParams.get("redirect_uri");
1287+ expect(redirectUri).not.toBeNull();
1288+ expect(new URL(redirectUri!).pathname).toBe(
1289+ `/v1/workspaces/${encodeURIComponent(installation.workspaceId)}/oauth/provider/callback`,
1290+ );
1291+ expect(result.source.status).toBe("auth_required");
1292+ }),
1293+ 60_000,
1294+ );
1295+
9351296 it.scoped("reuses existing Google provider grants for batch connects and clears orphan state", () =>
9361297 Effect.gen(function* () {
9371298 const runtime = yield* makeRuntime;
packages/platform/control-plane/src/runtime/local/source-artifacts.ts+27−4
@@ -23,7 +23,7 @@ import { · 22 unmodified lines
2323 contentHash,
2424 snapshotFromSourceCatalogSyncResult,
2525 type SourceCatalogSyncResult,
26−} from "../sources/catalog-sync-result";
26+} from "@executor/source-core";
2727 import type { ResolvedLocalWorkspaceContext } from "./config";
2828 import {
2929 LocalFileSystemError,
@@ -34,8 +34,18 @@ import { · 4 unmodified lines
3434 createSourceCatalogRevisionRecord,
3535 stableSourceCatalogId,
3636 } from "../sources/source-definitions";
37+const LEGACY_LOCAL_SOURCE_ARTIFACT_VERSION = 3 as const;
3738 const LOCAL_SOURCE_ARTIFACT_VERSION = 4 as const;
3839
40+const LegacyLocalSourceArtifactSchema = Schema.Struct({
41+ version: Schema.Literal(LEGACY_LOCAL_SOURCE_ARTIFACT_VERSION),
42+ sourceId: SourceIdSchema,
43+ catalogId: SourceCatalogIdSchema,
44+ generatedAt: TimestampMsSchema,
45+ revision: StoredSourceCatalogRevisionRecordSchema,
46+ snapshot: CatalogSnapshotV1Schema,
47+});
48+
3949 export const LocalSourceArtifactSchema = Schema.Struct({
4050 version: Schema.Literal(LOCAL_SOURCE_ARTIFACT_VERSION),
4151 sourceId: SourceIdSchema,
@@ -46,8 +56,21 @@ export const LocalSourceArtifactSchema = Schema.Struct({ · 4 unmodified lines
4656 });
4757
4858 export type LocalSourceArtifact = typeof LocalSourceArtifactSchema.Type;
49−
50−const decodeLocalSourceArtifact = Schema.decodeUnknownSync(LocalSourceArtifactSchema);
59+type LegacyLocalSourceArtifact = typeof LegacyLocalSourceArtifactSchema.Type;
60+
61+const decodeLocalSourceArtifact = Schema.decodeUnknownSync(
62+ Schema.Union(LocalSourceArtifactSchema, LegacyLocalSourceArtifactSchema),
63+);
64+
65+const normalizeLocalSourceArtifact = (
66+ artifact: LocalSourceArtifact | LegacyLocalSourceArtifact,
67+): LocalSourceArtifact =>
68+ artifact.version === LOCAL_SOURCE_ARTIFACT_VERSION
69+ ? artifact
70+ : {
71+ ...artifact,
72+ version: LOCAL_SOURCE_ARTIFACT_VERSION,
73+ };
5174
5275 const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>
5376 new LocalFileSystemError({
@@ -212,7 +235,7 @@ export const readLocalSourceArtifact = (input: { · 158 unmodified lines
212235 path: readPath,
213236 details: unknownLocalErrorDetails(cause),
214237 }),
215− });
238+ }).pipe(Effect.map(normalizeLocalSourceArtifact));
216239
217240 const rawDocuments: Record<string, NativeBlob> = {};
218241 for (const documentId of Object.keys(artifact.snapshot.catalog.documents)) {
packages/platform/control-plane/src/runtime/local/storage.ts+1−1
@@ -32,7 +32,7 @@ import { · 31 unmodified lines
3232 loadLocalWorkspaceState,
3333 writeLocalWorkspaceState,
3434 } from "./workspace-state";
35−import type { SourceCatalogSyncResult } from "../sources/catalog-sync-result";
35+import type { SourceCatalogSyncResult } from "@executor/source-core";
3636 import type { Source } from "#schema";
3737
3838 export type InstallationStoreShape = {
packages/platform/control-plane/src/runtime/sources/source-adapters/index.ts+28−46
@@ -1,66 +1,48 @@
11 import * as Schema from "effect/Schema";
22
33 import {
4− createSourceAdapterRegistry,
4+ createSourceAdapterComposition,
55 type SourceAdapter,
66 } from "@executor/source-core";
7−import { googleDiscoverySourceAdapter } from "@executor/source-google-discovery";
8−import { graphqlSourceAdapter } from "@executor/source-graphql";
9−import { mcpSourceAdapter } from "@executor/source-mcp";
10−import { openApiSourceAdapter } from "@executor/source-openapi";
7+import { externalSourceAdapters } from "@executor/source-builtins";
118
129 import { internalSourceAdapter } from "./internal";
1310
1411 export type * from "@executor/source-core";
1512
1613 export const builtInSourceAdapters = [
17− openApiSourceAdapter,
18− graphqlSourceAdapter,
19− googleDiscoverySourceAdapter,
20− mcpSourceAdapter,
14+ ...externalSourceAdapters,
2115 internalSourceAdapter,
2216 ] as const satisfies readonly SourceAdapter[];
23−
24−export const connectableSourceAdapters = [
25− mcpSourceAdapter,
26− openApiSourceAdapter,
27− graphqlSourceAdapter,
28− googleDiscoverySourceAdapter,
29−] as const;
30−
31−export const ConnectSourcePayloadSchema = Schema.Union(
32− mcpSourceAdapter.connectPayloadSchema!,
33− openApiSourceAdapter.connectPayloadSchema!,
34− graphqlSourceAdapter.connectPayloadSchema!,
35− googleDiscoverySourceAdapter.connectPayloadSchema!,
36−);
37−
17+const composition = createSourceAdapterComposition(builtInSourceAdapters);
18+
19+export const connectableSourceAdapters = composition.connectableSourceAdapters;
20+export const ConnectSourcePayloadSchema =
21+ composition.connectPayloadSchema as Schema.Schema<
22+ typeof composition.connectPayloadSchema.Type,
23+ any,
24+ never
25+ >;
3826 export type ConnectSourcePayload = typeof ConnectSourcePayloadSchema.Type;
3927
40−export const executorAddableSourceAdapters = [
41− mcpSourceAdapter,
42− openApiSourceAdapter,
43− graphqlSourceAdapter,
44− googleDiscoverySourceAdapter,
45−] as const;
46−
47−export const ExecutorAddSourceInputSchema = Schema.Union(
48− mcpSourceAdapter.executorAddInputSchema!,
49− openApiSourceAdapter.executorAddInputSchema!,
50− graphqlSourceAdapter.executorAddInputSchema!,
51− googleDiscoverySourceAdapter.executorAddInputSchema!,
52−);
53−
28+export const executorAddableSourceAdapters = composition.executorAddableSourceAdapters;
29+export const ExecutorAddSourceInputSchema =
30+ composition.executorAddInputSchema as Schema.Schema<
31+ typeof composition.executorAddInputSchema.Type,
32+ any,
33+ never
34+ >;
5435 export type ExecutorAddSourceInput = typeof ExecutorAddSourceInputSchema.Type;
5536
56−const registry = createSourceAdapterRegistry(builtInSourceAdapters);
37+export const localConfigurableSourceAdapters = composition.localConfigurableSourceAdapters;
5738
58−export const getSourceAdapter = registry.getSourceAdapter;
59−export const getSourceAdapterForSource = registry.getSourceAdapterForSource;
60−export const sourceBindingStateFromSource = registry.sourceBindingStateFromSource;
61−export const sourceAdapterCatalogKind = registry.sourceAdapterCatalogKind;
39+export const getSourceAdapter = composition.getSourceAdapter;
40+export const getSourceAdapterForSource = composition.getSourceAdapterForSource;
41+export const findSourceAdapterByProviderKey = composition.findSourceAdapterByProviderKey;
42+export const sourceBindingStateFromSource = composition.sourceBindingStateFromSource;
43+export const sourceAdapterCatalogKind = composition.sourceAdapterCatalogKind;
6244 export const sourceAdapterRequiresInteractiveConnect =
63− registry.sourceAdapterRequiresInteractiveConnect;
45+ composition.sourceAdapterRequiresInteractiveConnect;
6446 export const sourceAdapterUsesCredentialManagedAuth =
65− registry.sourceAdapterUsesCredentialManagedAuth;
66−export const isInternalSourceAdapter = registry.isInternalSourceAdapter;
47+ composition.sourceAdapterUsesCredentialManagedAuth;
48+export const isInternalSourceAdapter = composition.isInternalSourceAdapter;
packages/platform/control-plane/src/runtime/sources/source-adapters/internal.ts+4−2
@@ -52,7 +52,7 @@ const internalBindingConfigFromSource = ( · 51 unmodified lines
5252 });
5353 });
5454
55−export const internalSourceAdapter: SourceAdapter = {
55+export const internalSourceAdapter = {
5656 key: "internal",
5757 displayName: "Internal",
5858 catalogKind: "internal",
@@ -65,6 +65,8 @@ export const internalSourceAdapter: SourceAdapter = { · 6 unmodified lines
6565 executorAddInputSchema: null,
6666 executorAddHelpText: null,
6767 executorAddInputSignatureWidth: null,
68+ localConfigBindingSchema: null,
69+ localConfigBindingFromSource: () => null,
6870 serializeBindingConfig: (source) =>
6971 encodeBindingConfig({
7072 adapterKey: source.kind,
@@ -124,4 +126,4 @@ export const internalSourceAdapter: SourceAdapter = { · 53 unmodified lines
124126 Effect.fail(
125127 runtimeEffectError("sources/source-adapters/internal", "Internal sources do not support persisted adapter invocation"),
126128 ),
127−};
129+} satisfies SourceAdapter;
packages/platform/control-plane/src/runtime/sources/source-auth-service.ts+236−144
@@ -4,6 +4,10 @@ import { · 3 unmodified lines
44 isMcpStdioTransport,
55 type McpDiscoveryElicitationContext,
66 } from "@executor/source-mcp";
7+import {
8+ isSourceCredentialRequiredError,
9+ type SourceAdapterOauth2SetupConfig,
10+} from "@executor/source-core";
711 import {
812 AccountId,
913 type CredentialSlot,
@@ -61,14 +65,13 @@ import { · 51 unmodified lines
6165 updateSourceFromPayload,
6266 } from "./source-definitions";
6367 import {
68+ findSourceAdapterByProviderKey,
6469 getSourceAdapter,
6570 getSourceAdapterForSource,
6671 sourceAdapterRequiresInteractiveConnect,
6772 sourceAdapterUsesCredentialManagedAuth,
6873 sourceBindingStateFromSource,
6974 } from "./source-adapters";
70−import type { SourceAdapterOauth2SetupConfig } from "./source-adapters/types";
71−import { isSourceCredentialRequiredError } from "./source-adapters/shared";
7275 import {
7376 createDefaultSecretMaterialDeleter,
7477 type ResolveSecretMaterial,
@@ -932,6 +935,24 @@ type ResolvedWorkspaceOauthClient = ResolvedSourceOauthClient & { · 857 unmodified lines
932935 label: string | null;
933936 };
934937
938+type RuntimeProviderAuthTarget = {
939+ source: Source;
940+ requiredScopes: ReadonlyArray<string>;
941+ setupConfig: SourceAdapterOauth2SetupConfig;
942+};
943+
944+type RuntimeProviderAuthResult =
945+ | {
946+ kind: "connected";
947+ sources: ReadonlyArray<Source>;
948+ }
949+ | {
950+ kind: "oauth_required";
951+ sources: ReadonlyArray<Source>;
952+ sessionId: SourceAuthSession["id"];
953+ authorizationUrl: string;
954+ };
955+
935956 const decodeWorkspaceSourceOauthClientMetadataOption = Schema.decodeUnknownOption(
936957 WorkspaceSourceOauthClientMetadataJsonSchema,
937958 );
@@ -1172,6 +1193,84 @@ const providerGrantCoversScopes = ( · 234 unmodified lines
11721193 const normalizeScopes = (scopes: ReadonlyArray<string>): ReadonlyArray<string> =>
11731194 [...new Set(scopes.map((scope) => scope.trim()).filter((scope) => scope.length > 0))];
11741195
1196+const authorizationParamsMatch = (
1197+ left: SourceAdapterOauth2SetupConfig["authorizationParams"],
1198+ right: SourceAdapterOauth2SetupConfig["authorizationParams"],
1199+): boolean => {
1200+ const leftEntries = Object.entries(left ?? {}).sort(([leftKey], [rightKey]) =>
1201+ leftKey.localeCompare(rightKey),
1202+ );
1203+ const rightEntries = Object.entries(right ?? {}).sort(([leftKey], [rightKey]) =>
1204+ leftKey.localeCompare(rightKey),
1205+ );
1206+
1207+ return leftEntries.length === rightEntries.length
1208+ && leftEntries.every(([key, value], index) => {
1209+ const otherEntry = rightEntries[index];
1210+ return otherEntry !== undefined
1211+ && otherEntry[0] === key
1212+ && otherEntry[1] === value;
1213+ });
1214+};
1215+
1216+const providerOauthSetupConfigMatches = (
1217+ left: SourceAdapterOauth2SetupConfig,
1218+ right: SourceAdapterOauth2SetupConfig,
1219+): boolean =>
1220+ left.providerKey === right.providerKey
1221+ && left.authorizationEndpoint === right.authorizationEndpoint
1222+ && left.tokenEndpoint === right.tokenEndpoint
1223+ && left.headerName === right.headerName
1224+ && left.prefix === right.prefix
1225+ && left.clientAuthentication === right.clientAuthentication
1226+ && authorizationParamsMatch(left.authorizationParams, right.authorizationParams);
1227+
1228+const resolveRuntimeProviderAuthTarget = (
1229+ source: Source,
1230+): Effect.Effect<RuntimeProviderAuthTarget | null, Error, never> =>
1231+ Effect.gen(function* () {
1232+ const adapter = getSourceAdapterForSource(source);
1233+ const setupConfig = adapter.getOauth2SetupConfig
1234+ ? yield* adapter.getOauth2SetupConfig({
1235+ source,
1236+ slot: "runtime",
1237+ })
1238+ : null;
1239+ if (setupConfig === null) {
1240+ return null;
1241+ }
1242+
1243+ return {
1244+ source,
1245+ requiredScopes: normalizeScopes(setupConfig.scopes),
1246+ setupConfig,
1247+ } satisfies RuntimeProviderAuthTarget;
1248+ });
1249+
1250+const mergeRuntimeProviderAuthSetupConfig = (
1251+ targets: ReadonlyArray<RuntimeProviderAuthTarget>,
1252+): Effect.Effect<SourceAdapterOauth2SetupConfig, Error, never> =>
1253+ Effect.gen(function* () {
1254+ if (targets.length === 0) {
1255+ return yield* runtimeEffectError("sources/source-auth-service", "Provider auth setup requires at least one target source");
1256+ }
1257+
1258+ const baseSetupConfig = targets[0]!.setupConfig;
1259+ for (const target of targets.slice(1)) {
1260+ if (!providerOauthSetupConfigMatches(baseSetupConfig, target.setupConfig)) {
1261+ return yield* runtimeEffectError(
1262+ "sources/source-auth-service",
1263+ `Provider auth setup for ${baseSetupConfig.providerKey} requires compatible source OAuth configuration`,
1264+ );
1265+ }
1266+ }
1267+
1268+ return {
1269+ ...baseSetupConfig,
1270+ scopes: normalizeScopes(targets.flatMap((target) => [...target.requiredScopes])),
1271+ };
1272+ });
1273+
11751274 const findReusableProviderGrant = (input: {
11761275 rows: ControlPlaneStoreShape;
11771276 workspaceId: WorkspaceId;
@@ -1512,6 +1611,96 @@ const startProviderOauthBatchCredentialSetup = (input: { · 334 unmodified lines
15121611 );
15131612 });
15141613
1614+const connectSourcesWithProviderRuntimeAuth = (input: {
1615+ rows: ControlPlaneStoreShape;
1616+ sourceStore: RuntimeSourceStore;
1617+ sourceCatalogSync: RuntimeSourceCatalogSyncShape;
1618+ workspaceId: WorkspaceId;
1619+ actorAccountId?: AccountId | null;
1620+ executionId?: SourceAuthSession["executionId"];
1621+ interactionId?: SourceAuthSession["interactionId"];
1622+ baseUrl?: string | null;
1623+ getLocalServerBaseUrl?: () => string | undefined;
1624+ workspaceOauthClient: ResolvedWorkspaceOauthClient;
1625+ targets: ReadonlyArray<RuntimeProviderAuthTarget>;
1626+}): Effect.Effect<RuntimeProviderAuthResult, Error, WorkspaceStorageServices> =>
1627+ Effect.gen(function* () {
1628+ const setupConfig = yield* mergeRuntimeProviderAuthSetupConfig(input.targets);
1629+ const reusableGrant = yield* findReusableProviderGrant({
1630+ rows: input.rows,
1631+ workspaceId: input.workspaceId,
1632+ actorAccountId: input.actorAccountId,
1633+ providerKey: setupConfig.providerKey,
1634+ oauthClientId: input.workspaceOauthClient.id,
1635+ requiredScopes: setupConfig.scopes,
1636+ });
1637+
1638+ if (Option.isSome(reusableGrant)) {
1639+ const connectedSources = yield* attachProviderGrantToSources({
1640+ sourceStore: input.sourceStore,
1641+ sourceCatalogSync: input.sourceCatalogSync,
1642+ actorAccountId: input.actorAccountId,
1643+ grantId: reusableGrant.value.id,
1644+ providerKey: reusableGrant.value.providerKey,
1645+ headerName: reusableGrant.value.headerName,
1646+ prefix: reusableGrant.value.prefix,
1647+ targets: input.targets.map((target) => ({
1648+ source: target.source,
1649+ requiredScopes: target.requiredScopes,
1650+ })),
1651+ });
1652+
1653+ return {
1654+ kind: "connected",
1655+ sources: connectedSources,
1656+ } satisfies RuntimeProviderAuthResult;
1657+ }
1658+
1659+ const requestBaseUrl = trimOrNull(input.baseUrl);
1660+ const baseUrl = requestBaseUrl ?? input.getLocalServerBaseUrl?.() ?? null;
1661+ if (baseUrl === null) {
1662+ return yield* runtimeEffectError(
1663+ "sources/source-auth-service",
1664+ `Local executor server base URL is unavailable for ${setupConfig.providerKey} OAuth setup`,
1665+ );
1666+ }
1667+
1668+ const oauthRequired = yield* startProviderOauthBatchCredentialSetup({
1669+ rows: input.rows,
1670+ sourceStore: input.sourceStore,
1671+ workspaceId: input.workspaceId,
1672+ actorAccountId: input.actorAccountId,
1673+ executionId: input.executionId,
1674+ interactionId: input.interactionId,
1675+ baseUrl,
1676+ redirectModeOverride: requestBaseUrl ? "app_callback" : undefined,
1677+ workspaceOauthClient: input.workspaceOauthClient,
1678+ setupConfig,
1679+ targetSources: input.targets.map((target) => ({
1680+ source: target.source,
1681+ requiredScopes: target.requiredScopes,
1682+ })),
1683+ });
1684+
1685+ const pendingSources = yield* Effect.forEach(
1686+ input.targets,
1687+ (target) =>
1688+ input.sourceStore.loadSourceById({
1689+ workspaceId: target.source.workspaceId,
1690+ sourceId: target.source.id,
1691+ actorAccountId: input.actorAccountId,
1692+ }),
1693+ { discard: false },
1694+ );
1695+
1696+ return {
1697+ kind: "oauth_required",
1698+ sources: pendingSources,
1699+ sessionId: oauthRequired.sessionId,
1700+ authorizationUrl: oauthRequired.authorizationUrl,
1701+ } satisfies RuntimeProviderAuthResult;
1702+ });
1703+
15151704 const attachProviderGrantToSources = (input: {
15161705 sourceStore: RuntimeSourceStore;
15171706 sourceCatalogSync: RuntimeSourceCatalogSyncShape;
@@ -2235,15 +2424,10 @@ const addExecutorGoogleDiscoverySource = (input: { · 717 unmodified lines
22352424 actorAccountId: input.sourceInput.actorAccountId,
22362425 });
22372426 const googleAdapter = getSourceAdapterForSource(persistedDraft);
2238− const googleSetupConfig = googleAdapter.getOauth2SetupConfig
2239− ? yield* googleAdapter.getOauth2SetupConfig({
2240− source: persistedDraft,
2241− slot: "runtime",
2242− })
2243− : null;
2427+ const providerAuthTarget = yield* resolveRuntimeProviderAuthTarget(persistedDraft);
22442428
22452429 if (
2246− googleSetupConfig !== null
2430+ providerAuthTarget !== null
22472431 && input.sourceInput.auth === undefined
22482432 && persistedDraft.auth.kind === "none"
22492433 ) {
@@ -2253,13 +2437,13 @@ const addExecutorGoogleDiscoverySource = (input: { · 3 unmodified lines
22532437 rows: input.rows,
22542438 workspaceId: persistedDraft.workspaceId,
22552439 oauthClientId: input.sourceInput.workspaceOauthClientId,
2256− providerKey: googleSetupConfig.providerKey,
2440+ providerKey: providerAuthTarget.setupConfig.providerKey,
22572441 });
22582442 } else if (input.sourceInput.oauthClient) {
22592443 workspaceOauthClient = yield* createWorkspaceOauthClient({
22602444 rows: input.rows,
22612445 workspaceId: persistedDraft.workspaceId,
2262− providerKey: googleSetupConfig.providerKey,
2446+ providerKey: providerAuthTarget.setupConfig.providerKey,
22632447 oauthClient: input.sourceInput.oauthClient,
22642448 label: `${chosenName} OAuth Client`,
22652449 normalizeOauthClient: googleAdapter.normalizeOauthClientInput,
@@ -2268,69 +2452,37 @@ const addExecutorGoogleDiscoverySource = (input: { · 2 unmodified lines
22682452 }
22692453
22702454 if (workspaceOauthClient === null) {
2271− return yield* runtimeEffectError("sources/source-auth-service", "Google shared auth requires a workspace OAuth client");
2272− }
2273−
2274− const requiredScopes = normalizeScopes(googleSetupConfig.scopes);
2275− const reusableGrant = yield* findReusableProviderGrant({
2276− rows: input.rows,
2277− workspaceId: persistedDraft.workspaceId,
2278− actorAccountId: input.sourceInput.actorAccountId,
2279− providerKey: googleSetupConfig.providerKey,
2280− oauthClientId: workspaceOauthClient.id,
2281− requiredScopes,
2282− });
2283−
2284− if (Option.isSome(reusableGrant)) {
2285− const [connectedSource] = yield* attachProviderGrantToSources({
2286− sourceStore: input.sourceStore,
2287− sourceCatalogSync: input.sourceCatalogSync,
2288− actorAccountId: input.sourceInput.actorAccountId,
2289− grantId: reusableGrant.value.id,
2290− providerKey: reusableGrant.value.providerKey,
2291− headerName: reusableGrant.value.headerName,
2292− prefix: reusableGrant.value.prefix,
2293− targets: [{
2294− source: persistedDraft,
2295− requiredScopes,
2296− }],
2297− });
2298−
2299− return {
2300− kind: "connected",
2301− source: connectedSource!,
2302− } satisfies ExecutorSourceAddResult;
2303− }
2304−
2305− const requestBaseUrl = trimOrNull(input.baseUrl);
2306− const baseUrl = requestBaseUrl ?? input.getLocalServerBaseUrl?.() ?? null;
2307− if (baseUrl === null) {
2308− return yield* runtimeEffectError("sources/source-auth-service", "Local executor server base URL is unavailable for Google OAuth setup");
2455+ return yield* runtimeEffectError(
2456+ "sources/source-auth-service",
2457+ `${providerAuthTarget.setupConfig.providerKey} shared auth requires a workspace OAuth client`,
2458+ );
23092459 }
23102460
2311− const oauthRequired = yield* startProviderOauthBatchCredentialSetup({
2461+ const providerAuthResult = yield* connectSourcesWithProviderRuntimeAuth({
23122462 rows: input.rows,
23132463 sourceStore: input.sourceStore,
2464+ sourceCatalogSync: input.sourceCatalogSync,
23142465 workspaceId: persistedDraft.workspaceId,
23152466 actorAccountId: input.sourceInput.actorAccountId,
23162467 executionId: input.sourceInput.executionId,
23172468 interactionId: input.sourceInput.interactionId,
2318− baseUrl,
2319− redirectModeOverride: requestBaseUrl ? "app_callback" : undefined,
2469+ baseUrl: input.baseUrl,
2470+ getLocalServerBaseUrl: input.getLocalServerBaseUrl,
23202471 workspaceOauthClient,
2321− setupConfig: googleSetupConfig,
2322− targetSources: [{
2323− source: persistedDraft,
2324− requiredScopes,
2325− }],
2472+ targets: [providerAuthTarget],
23262473 });
23272474
2328− return {
2329− kind: "oauth_required",
2330− source: oauthRequired.source,
2331− sessionId: oauthRequired.sessionId,
2332− authorizationUrl: oauthRequired.authorizationUrl,
2333− } satisfies ExecutorSourceAddResult;
2475+ return providerAuthResult.kind === "connected"
2476+ ? {
2477+ kind: "connected",
2478+ source: providerAuthResult.sources[0]!,
2479+ } satisfies ExecutorSourceAddResult
2480+ : {
2481+ kind: "oauth_required",
2482+ source: providerAuthResult.sources[0]!,
2483+ sessionId: providerAuthResult.sessionId,
2484+ authorizationUrl: providerAuthResult.authorizationUrl,
2485+ } satisfies ExecutorSourceAddResult;
23342486 }
23352487
23362488 if (input.sourceInput.oauthClient) {
@@ -2537,66 +2689,43 @@ const connectGoogleDiscoveryBatchInternal = (input: { · 200 unmodified lines
25372689 const persistedDraft = yield* input.sourceStore.persistSource(draftSource, {
25382690 actorAccountId: input.sourceInput.actorAccountId,
25392691 });
2540− const adapter = getSourceAdapterForSource(persistedDraft);
2541− const setupConfig = adapter.getOauth2SetupConfig
2542− ? yield* adapter.getOauth2SetupConfig({
2543− source: persistedDraft,
2544− slot: "runtime",
2545− })
2546− : null;
2547− if (setupConfig === null) {
2692+ const providerAuthTarget = yield* resolveRuntimeProviderAuthTarget(persistedDraft);
2693+ if (providerAuthTarget === null) {
25482694 return yield* runtimeEffectError("sources/source-auth-service", `Source ${persistedDraft.id} does not support Google shared auth`);
25492695 }
25502696
2551− return {
2552− source: persistedDraft,
2553− requiredScopes: normalizeScopes(setupConfig.scopes),
2554− setupConfig,
2555− };
2697+ return providerAuthTarget;
25562698 }),
25572699 { discard: false },
25582700 );
25592701
2560− const baseSetupConfig = persistedTargets[0]!.setupConfig;
2561− const unionScopes = normalizeScopes(
2562− persistedTargets.flatMap((target) => [...target.requiredScopes]),
2563− );
25642702 const workspaceOauthClient = yield* resolveWorkspaceOauthClientById({
25652703 rows: input.rows,
25662704 workspaceId: input.sourceInput.workspaceId,
25672705 oauthClientId: input.sourceInput.workspaceOauthClientId,
2568− providerKey: baseSetupConfig.providerKey,
2706+ providerKey: persistedTargets[0]!.setupConfig.providerKey,
25692707 });
25702708 if (workspaceOauthClient === null) {
25712709 return yield* runtimeEffectError("sources/source-auth-service", `Workspace OAuth client not found: ${input.sourceInput.workspaceOauthClientId}`);
25722710 }
25732711
2574− const reusableGrant = yield* findReusableProviderGrant({
2712+ const providerAuthResult = yield* connectSourcesWithProviderRuntimeAuth({
25752713 rows: input.rows,
2714+ sourceStore: input.sourceStore,
2715+ sourceCatalogSync: input.sourceCatalogSync,
25762716 workspaceId: input.sourceInput.workspaceId,
25772717 actorAccountId: input.sourceInput.actorAccountId,
2578− providerKey: baseSetupConfig.providerKey,
2579− oauthClientId: workspaceOauthClient.id,
2580− requiredScopes: unionScopes,
2718+ executionId: input.sourceInput.executionId,
2719+ interactionId: input.sourceInput.interactionId,
2720+ baseUrl: input.sourceInput.baseUrl,
2721+ getLocalServerBaseUrl: input.getLocalServerBaseUrl,
2722+ workspaceOauthClient,
2723+ targets: persistedTargets,
25812724 });
25822725
2583− if (Option.isSome(reusableGrant)) {
2584− const connectedSources = yield* attachProviderGrantToSources({
2585− sourceStore: input.sourceStore,
2586− sourceCatalogSync: input.sourceCatalogSync,
2587− actorAccountId: input.sourceInput.actorAccountId,
2588− grantId: reusableGrant.value.id,
2589− providerKey: reusableGrant.value.providerKey,
2590− headerName: reusableGrant.value.headerName,
2591− prefix: reusableGrant.value.prefix,
2592− targets: persistedTargets.map((target) => ({
2593− source: target.source,
2594− requiredScopes: target.requiredScopes,
2595− })),
2596− });
2597−
2726+ if (providerAuthResult.kind === "connected") {
25982727 return {
2599− results: connectedSources.map((source) => ({
2728+ results: providerAuthResult.sources.map((source) => ({
26002729 source,
26012730 status: "connected" as const,
26022731 })),
@@ -2604,51 +2733,15 @@ const connectGoogleDiscoveryBatchInternal = (input: { · 1 unmodified line
26042733 };
26052734 }
26062735
2607− const baseUrl = trimOrNull(input.sourceInput.baseUrl) ?? input.getLocalServerBaseUrl?.() ?? null;
2608− if (baseUrl === null) {
2609− return yield* runtimeEffectError("sources/source-auth-service", "Local executor server base URL is unavailable for Google OAuth setup");
2610− }
2611−
2612− const oauthRequired = yield* startProviderOauthBatchCredentialSetup({
2613− rows: input.rows,
2614− sourceStore: input.sourceStore,
2615− workspaceId: input.sourceInput.workspaceId,
2616− actorAccountId: input.sourceInput.actorAccountId,
2617− executionId: input.sourceInput.executionId,
2618− interactionId: input.sourceInput.interactionId,
2619− baseUrl,
2620− redirectModeOverride: trimOrNull(input.sourceInput.baseUrl) ? "app_callback" : undefined,
2621− workspaceOauthClient,
2622− setupConfig: {
2623− ...baseSetupConfig,
2624− scopes: unionScopes,
2625− },
2626− targetSources: persistedTargets.map((target) => ({
2627− source: target.source,
2628− requiredScopes: target.requiredScopes,
2629− })),
2630− });
2631−
2632− const pendingSources = yield* Effect.forEach(
2633− persistedTargets,
2634− (target) =>
2635− input.sourceStore.loadSourceById({
2636− workspaceId: input.sourceInput.workspaceId,
2637− sourceId: target.source.id,
2638− actorAccountId: input.sourceInput.actorAccountId,
2639− }),
2640− { discard: false },
2641− );
2642−
26432736 return {
2644− results: pendingSources.map((source) => ({
2737+ results: providerAuthResult.sources.map((source) => ({
26452738 source,
26462739 status: "pending_oauth" as const,
26472740 })),
26482741 providerOauthSession: {
2649− sessionId: oauthRequired.sessionId,
2650− authorizationUrl: oauthRequired.authorizationUrl,
2651− sourceIds: pendingSources.map((source) => source.id),
2742+ sessionId: providerAuthResult.sessionId,
2743+ authorizationUrl: providerAuthResult.authorizationUrl,
2744+ sourceIds: providerAuthResult.sources.map((source) => source.id),
26522745 },
26532746 };
26542747 });
@@ -2883,15 +2976,14 @@ const createRuntimeSourceConnectionService = ( · 228 unmodified lines
28832976 createWorkspaceOauthClient: ({ workspaceId, providerKey, label, oauthClient }) =>
28842977 provideLocalWorkspace(
28852978 Effect.gen(function* () {
2979+ const sourceAdapter = findSourceAdapterByProviderKey(providerKey);
28862980 const created = yield* createWorkspaceOauthClient({
28872981 rows: input.rows,
28882982 workspaceId,
28892983 providerKey,
28902984 oauthClient,
28912985 label,
2892− normalizeOauthClient: providerKey === "google_workspace"
2893− ? getSourceAdapter("google_discovery").normalizeOauthClientInput
2894− : undefined,
2986+ normalizeOauthClient: sourceAdapter?.normalizeOauthClientInput,
28952987 storeSecretMaterial: input.storeSecretMaterial,
28962988 });
28972989 const stored = yield* input.rows.workspaceOauthClients.getById(created.id);
packages/platform/control-plane/src/runtime/sources/source-discovery.ts+28−51
@@ -2,75 +2,52 @@ import * as Effect from "effect/Effect"; · 1 unmodified line
22
33 import {
44 fallbackSourceDiscoveryResult,
5− looksLikeGraphqlEndpoint,
65 normalizeSourceDiscoveryUrl,
76 probeHeadersFromAuth,
7+ type SourceAdapter,
88 type SourceProbeAuth,
99 type SourceDiscoveryResult,
1010 } from "@executor/source-core";
11−import { detectGoogleDiscoverySource } from "@executor/source-google-discovery";
12−import { detectGraphqlSource } from "@executor/source-graphql";
13−import { detectMcpSource } from "@executor/source-mcp";
14−import { detectOpenApiSource } from "@executor/source-openapi";
11+
12+import { builtInSourceAdapters } from "./source-adapters";
13+
14+type DiscoverableSourceAdapter = SourceAdapter & {
15+ detectSource: NonNullable<SourceAdapter["detectSource"]>;
16+};
17+
18+const sourceAdapters: ReadonlyArray<SourceAdapter> = builtInSourceAdapters;
19+
20+const discoverableSourceAdapters = sourceAdapters.filter(
21+ (adapter): adapter is DiscoverableSourceAdapter =>
22+ adapter.detectSource !== undefined,
23+);
1524
1625 export const discoverSource = (input: {
1726 url: string;
1827 probeAuth?: SourceProbeAuth | null;
1928 }): Effect.Effect<SourceDiscoveryResult, Error, never> =>
2029 Effect.gen(function* () {
21− const normalizedUrl = normalizeSourceDiscoveryUrl(input.url);
30+ const normalizedUrl = yield* Effect.try({
31+ try: () => normalizeSourceDiscoveryUrl(input.url),
32+ catch: (cause) =>
33+ cause instanceof Error ? cause : new Error(String(cause)),
34+ });
2235 const headers = probeHeadersFromAuth(input.probeAuth);
2336
24− if (looksLikeGraphqlEndpoint(normalizedUrl)) {
25− const graphql = yield* detectGraphqlSource({
26− normalizedUrl,
27− headers,
28− });
29− if (graphql) {
30− return graphql;
31− }
37+ const adaptersByPriority = [...discoverableSourceAdapters].sort(
38+ (left, right) =>
39+ (right.discoveryPriority?.({ normalizedUrl }) ?? 0)
40+ - (left.discoveryPriority?.({ normalizedUrl }) ?? 0),
41+ );
3242
33− const mcp = yield* detectMcpSource({
43+ for (const adapter of adaptersByPriority) {
44+ const detected = yield* adapter.detectSource({
3445 normalizedUrl,
3546 headers,
3647 });
37− if (mcp) {
38− return mcp;
48+ if (detected) {
49+ return detected;
3950 }
40−
41− return fallbackSourceDiscoveryResult(normalizedUrl);
42− }
43−
44− const googleDiscovery = yield* detectGoogleDiscoverySource({
45− normalizedUrl,
46− headers,
47− });
48− if (googleDiscovery) {
49− return googleDiscovery;
50− }
51−
52− const openApi = yield* detectOpenApiSource({
53− normalizedUrl,
54− headers,
55− });
56− if (openApi) {
57− return openApi;
58− }
59−
60− const graphql = yield* detectGraphqlSource({
61− normalizedUrl,
62− headers,
63− });
64− if (graphql) {
65− return graphql;
66− }
67−
68− const mcp = yield* detectMcpSource({
69− normalizedUrl,
70− headers,
71− });
72− if (mcp) {
73− return mcp;
7451 }
7552
7653 return fallbackSourceDiscoveryResult(normalizedUrl);
packages/platform/control-plane/src/runtime/sources/source-store/config.ts+13−42
@@ -12,6 +12,7 @@ import { · 11 unmodified lines
1212 fromConfigSecretProviderId,
1313 toConfigSecretProviderId,
1414 } from "../../local/config-secrets";
15+import { getSourceAdapterForSource } from "../source-adapters";
1516 import { slugify } from "../slug";
1617
1718 export const trimOrNull = (value: string | null | undefined): string | null => {
@@ -168,47 +169,17 @@ export const configSourceFromLocalSource = (input: { · 150 unmodified lines
168169 },
169170 };
170171
171− switch (input.source.kind) {
172− case "openapi":
173− return {
174− kind: "openapi",
175− ...common,
176− binding: cloneJson(input.source.binding) as Extract<
177− LocalConfigSource,
178− { kind: "openapi" }
179− >["binding"],
180− };
181− case "graphql":
182− return {
183− kind: "graphql",
184− ...common,
185− binding: cloneJson(input.source.binding) as Extract<
186− LocalConfigSource,
187− { kind: "graphql" }
188− >["binding"],
189− };
190− case "google_discovery":
191− return {
192− kind: "google_discovery",
193− ...common,
194− binding: cloneJson(input.source.binding) as Extract<
195− LocalConfigSource,
196− { kind: "google_discovery" }
197− >["binding"],
198− };
199− case "mcp":
200− return {
201− kind: "mcp",
202− ...common,
203− binding: cloneJson(input.source.binding) as Extract<
204− LocalConfigSource,
205− { kind: "mcp" }
206− >["binding"],
207− };
208− default:
209− throw new LocalUnsupportedSourceKindError({
210− message: `Unsupported source kind for local config: ${input.source.kind}`,
211− kind: input.source.kind,
212− });
172+ const adapter = getSourceAdapterForSource(input.source);
173+ if (adapter.localConfigBindingSchema === null) {
174+ throw new LocalUnsupportedSourceKindError({
175+ message: `Unsupported source kind for local config: ${input.source.kind}`,
176+ kind: input.source.kind,
177+ });
213178 }
179+
180+ return {
181+ kind: input.source.kind as LocalConfigSource["kind"],
182+ ...common,
183+ binding: cloneJson(adapter.localConfigBindingFromSource(input.source)),
184+ } as LocalConfigSource;
214185 };
packages/platform/control-plane/src/schema/models/local-config.ts+36−42
@@ -1,11 +1,6 @@
11 import { Schema } from "effect";
22
3−import {
4− GoogleDiscoveryLocalConfigBindingSchema,
5−} from "@executor/source-google-discovery";
6−import { GraphqlLocalConfigBindingSchema } from "@executor/source-graphql";
7−import { McpLocalConfigBindingSchema } from "@executor/source-mcp";
8−import { OpenApiLocalConfigBindingSchema } from "@executor/source-openapi";
3+import { localConfigurableSourceAdapters } from "../../runtime/sources/source-adapters";
94
105 export const LocalExecutorRuntimeSchema = Schema.Literal(
116 "quickjs",
@@ -73,44 +68,44 @@ const LocalConfigSourceEntryBaseSchema = Schema.Struct({ · 61 unmodified lines
7368 connection: LocalConfigSourceConnectionSchema,
7469 });
7570
76−export const LocalConfigOpenApiSourceSchema = Schema.extend(
77− LocalConfigSourceEntryBaseSchema,
78− Schema.Struct({
79− kind: Schema.Literal("openapi"),
80− binding: OpenApiLocalConfigBindingSchema,
81− }),
82−);
83−
84−export const LocalConfigGraphqlSourceSchema = Schema.extend(
85− LocalConfigSourceEntryBaseSchema,
86− Schema.Struct({
87− kind: Schema.Literal("graphql"),
88− binding: GraphqlLocalConfigBindingSchema,
89− }),
90−);
91−
92−export const LocalConfigMcpSourceSchema = Schema.extend(
93− LocalConfigSourceEntryBaseSchema,
94− Schema.Struct({
95− kind: Schema.Literal("mcp"),
96− binding: McpLocalConfigBindingSchema,
97− }),
98−);
99−
100−export const LocalConfigGoogleDiscoverySourceSchema = Schema.extend(
101− LocalConfigSourceEntryBaseSchema,
102− Schema.Struct({
103− kind: Schema.Literal("google_discovery"),
104− binding: GoogleDiscoveryLocalConfigBindingSchema,
105− }),
71+type SchemaTypeOf<TSchema> = TSchema extends { readonly Type: infer T } ? T : never;
72+type LocalConfigSourceEntryBase = typeof LocalConfigSourceEntryBaseSchema.Type;
73+
74+type LocalConfigSourceFromAdapter<
75+ TAdapter extends (typeof localConfigurableSourceAdapters)[number],
76+> =
77+ TAdapter extends {
78+ key: infer TKey extends string;
79+ localConfigBindingSchema: infer TSchema;
80+ }
81+ ? {
82+ kind: TKey;
83+ binding: SchemaTypeOf<TSchema>;
84+ } & LocalConfigSourceEntryBase
85+ : never;
86+
87+export type LocalConfigSource = LocalConfigSourceFromAdapter<
88+ (typeof localConfigurableSourceAdapters)[number]
89+>;
90+
91+const LocalConfigSourceEntrySchemas = localConfigurableSourceAdapters.map(
92+ (adapter) =>
93+ Schema.extend(
94+ LocalConfigSourceEntryBaseSchema,
95+ Schema.Struct({
96+ kind: Schema.Literal(adapter.key as LocalConfigSource["kind"]),
97+ binding: adapter.localConfigBindingSchema,
98+ }),
99+ ),
106100 );
107101
108102 export const LocalConfigSourceSchema = Schema.Union(
109− LocalConfigOpenApiSourceSchema,
110− LocalConfigGraphqlSourceSchema,
111− LocalConfigMcpSourceSchema,
112− LocalConfigGoogleDiscoverySourceSchema,
113−);
103+ ...((LocalConfigSourceEntrySchemas as unknown) as [
104+ Schema.Schema<any, any, never>,
105+ Schema.Schema<any, any, never>,
106+ ...Array<Schema.Schema<any, any, never>>,
107+ ]),
108+) as Schema.Schema<LocalConfigSource, any, never>;
114109
115110 export const LocalConfigPolicyActionSchema = Schema.Literal("allow", "deny");
116111 export const LocalConfigPolicyApprovalSchema = Schema.Literal("auto", "manual");
@@ -168,7 +163,6 @@ export type LocalConfigSecretProvider = · 51 unmodified lines
168163 export type LocalConfigExplicitSecretRef =
169164 typeof LocalConfigExplicitSecretRefSchema.Type;
170165 export type LocalConfigSecretInput = typeof LocalConfigSecretInputSchema.Type;
171−export type LocalConfigSource = typeof LocalConfigSourceSchema.Type;
172166 export type LocalConfigPolicy = typeof LocalConfigPolicySchema.Type;
173167 export type LocalConfigSecrets = typeof LocalConfigSecretsSchema.Type;
174168 export type LocalExecutorRuntime = typeof LocalExecutorRuntimeSchema.Type;
packages/sources/builtins/package.json+26−0
@@ -0,0 +1,26 @@
1+{
2+ "name": "@executor/source-builtins",
3+ "private": true,
4+ "type": "module",
5+ "exports": {
6+ ".": "./src/index.ts"
7+ },
8+ "files": [
9+ "src"
10+ ],
11+ "scripts": {
12+ "typecheck": "bunx tsc --noEmit -p tsconfig.json"
13+ },
14+ "dependencies": {
15+ "@executor/source-core": "workspace:*",
16+ "@executor/source-google-discovery": "workspace:*",
17+ "@executor/source-graphql": "workspace:*",
18+ "@executor/source-mcp": "workspace:*",
19+ "@executor/source-openapi": "workspace:*"
20+ },
21+ "devDependencies": {
22+ "@types/node": "catalog:",
23+ "bun-types": "catalog:"
24+ },
25+ "version": "0.0.0"
26+}
packages/sources/builtins/src/index.ts+12−0
@@ -0,0 +1,12 @@
1+import type { SourceAdapter } from "@executor/source-core";
2+import { googleDiscoverySourceAdapter } from "@executor/source-google-discovery";
3+import { graphqlSourceAdapter } from "@executor/source-graphql";
4+import { mcpSourceAdapter } from "@executor/source-mcp";
5+import { openApiSourceAdapter } from "@executor/source-openapi";
6+
7+export const externalSourceAdapters = [
8+ openApiSourceAdapter,
9+ graphqlSourceAdapter,
10+ googleDiscoverySourceAdapter,
11+ mcpSourceAdapter,
12+] as const satisfies readonly SourceAdapter[];
packages/sources/builtins/tsconfig.json+15−0
@@ -0,0 +1,15 @@
1+{
2+ "compilerOptions": {
3+ "target": "ES2022",
4+ "module": "ESNext",
5+ "moduleResolution": "Bundler",
6+ "strict": true,
7+ "skipLibCheck": true,
8+ "noEmit": true,
9+ "lib": ["ES2022"],
10+ "types": ["bun-types", "node"]
11+ },
12+ "include": [
13+ "src/**/*.ts"
14+ ]
15+}
packages/sources/core/src/composition.ts+88−0
@@ -0,0 +1,88 @@
1+import * as Schema from "effect/Schema";
2+
3+import { createSourceAdapterRegistry } from "./registry";
4+import type { SourceAdapter, SourceAdapterInputSchema } from "./types";
5+
6+type SchemaTypeOf<TSchema> = TSchema extends { readonly Type: infer T } ? T : never;
7+type AdapterUnion<TAdapters extends readonly SourceAdapter[]> = TAdapters[number];
8+type ConnectPayloadSchema<TAdapters extends readonly SourceAdapter[]> =
9+ NonNullable<AdapterUnion<TAdapters>["connectPayloadSchema"]>;
10+type ExecutorAddInputSchema<TAdapters extends readonly SourceAdapter[]> =
11+ NonNullable<AdapterUnion<TAdapters>["executorAddInputSchema"]>;
12+
13+export type ConnectableSourceAdapter<TAdapter extends SourceAdapter = SourceAdapter> = TAdapter & {
14+ connectPayloadSchema: SourceAdapterInputSchema;
15+};
16+
17+export type ExecutorAddableSourceAdapter<TAdapter extends SourceAdapter = SourceAdapter> = TAdapter & {
18+ executorAddInputSchema: SourceAdapterInputSchema;
19+};
20+
21+export type LocalConfigurableSourceAdapter<TAdapter extends SourceAdapter = SourceAdapter> = TAdapter & {
22+ localConfigBindingSchema: SourceAdapterInputSchema;
23+};
24+
25+const isConnectableSourceAdapter = <TAdapter extends SourceAdapter>(
26+ adapter: TAdapter,
27+): adapter is ConnectableSourceAdapter<TAdapter> =>
28+ adapter.connectPayloadSchema !== null;
29+
30+const isExecutorAddableSourceAdapter = <TAdapter extends SourceAdapter>(
31+ adapter: TAdapter,
32+): adapter is ExecutorAddableSourceAdapter<TAdapter> =>
33+ adapter.executorAddInputSchema !== null;
34+
35+const isLocalConfigurableSourceAdapter = <TAdapter extends SourceAdapter>(
36+ adapter: TAdapter,
37+): adapter is LocalConfigurableSourceAdapter<TAdapter> =>
38+ adapter.localConfigBindingSchema !== null;
39+
40+const asSchemaTuple = (
41+ schemas: ReadonlyArray<Schema.Schema<any, any, never>>,
42+): [
43+ Schema.Schema<any, any, never>,
44+ Schema.Schema<any, any, never>,
45+ ...Array<Schema.Schema<any, any, never>>,
46+] => schemas as [
47+ Schema.Schema<any, any, never>,
48+ Schema.Schema<any, any, never>,
49+ ...Array<Schema.Schema<any, any, never>>,
50+];
51+
52+const createSchemaFromAdapters = <TSchema extends SourceAdapterInputSchema>(
53+ schemas: ReadonlyArray<TSchema>,
54+ label: string,
55+): TSchema =>
56+ (schemas.length === 0
57+ ? (() => {
58+ throw new Error(`Cannot create ${label} without any schemas`);
59+ })()
60+ : schemas.length === 1
61+ ? schemas[0]!
62+ : Schema.Union(...asSchemaTuple(schemas))) as TSchema;
63+
64+export const createSourceAdapterComposition = <const TAdapters extends readonly SourceAdapter[]>(
65+ adapters: TAdapters,
66+) => {
67+ const connectableSourceAdapters = adapters.filter(isConnectableSourceAdapter);
68+ const executorAddableSourceAdapters = adapters.filter(isExecutorAddableSourceAdapter);
69+ const localConfigurableSourceAdapters = adapters.filter(isLocalConfigurableSourceAdapter);
70+ const registry = createSourceAdapterRegistry(adapters);
71+
72+ return {
73+ connectableSourceAdapters,
74+ connectPayloadSchema: createSchemaFromAdapters(
75+ connectableSourceAdapters.map((adapter) => adapter.connectPayloadSchema),
76+ "connect payload schema",
77+ ) as Schema.Schema<SchemaTypeOf<ConnectPayloadSchema<TAdapters>>, any, never>,
78+ executorAddableSourceAdapters,
79+ executorAddInputSchema: createSchemaFromAdapters(
80+ executorAddableSourceAdapters.map((adapter) => adapter.executorAddInputSchema),
81+ "executor add input schema",
82+ ) as Schema.Schema<SchemaTypeOf<ExecutorAddInputSchema<TAdapters>>, any, never>,
83+ localConfigurableSourceAdapters,
84+ ...registry,
85+ };
86+};
87+
88+export type SourceAdapterComposition = ReturnType<typeof createSourceAdapterComposition>;
packages/sources/core/src/index.ts+1−0
@@ -1,5 +1,6 @@
11 export * from "./catalog";
22 export * from "./catalog-sync-result";
3+export * from "./composition";
34 export * from "./discovery";
45 export * from "./discovery-models";
56 export * from "./effect-errors";
packages/sources/core/src/registry.ts+4−0
@@ -20,10 +20,14 @@ export const createSourceAdapterRegistry = <const TAdapters extends readonly Sou · 19 unmodified lines
2020 const getSourceAdapterForSource = (source: Pick<Source, "kind">): SourceAdapter =>
2121 getSourceAdapter(source.kind);
2222
23+ const findSourceAdapterByProviderKey = (providerKey: string): SourceAdapter | null =>
24+ adapters.find((adapter) => adapter.providerKey === providerKey) ?? null;
25+
2326 return {
2427 adapters,
2528 getSourceAdapter,
2629 getSourceAdapterForSource,
30+ findSourceAdapterByProviderKey,
2731 sourceBindingStateFromSource: (source: Source) =>
2832 getSourceAdapterForSource(source).bindingStateFromSource(source),
2933 sourceAdapterCatalogKind: (key: string): SourceCatalogKind =>
packages/sources/core/src/types.ts+14−0
@@ -9,6 +9,7 @@ import type { Schema } from "effect"; · 8 unmodified lines
99
1010 import type { OAuth2ClientAuthenticationMethod } from "@executor/auth-oauth2";
1111 import type { SourceCatalogSyncResult } from "./catalog-sync-result";
12+import type { SourceDiscoveryResult } from "./discovery-models";
1213 import type {
1314 CredentialSlot,
1415 SecretRef,
@@ -128,6 +129,11 @@ export type SourceAdapterInvokeInput = { · 113 unmodified lines
128129 context?: Record<string, unknown>;
129130 };
130131
132+export type SourceAdapterDiscoveryInput = {
133+ normalizedUrl: string;
134+ headers: Readonly<Record<string, string>>;
135+};
136+
131137 export type SourceAdapter = {
132138 key: string;
133139 displayName: string;
@@ -141,6 +147,8 @@ export type SourceAdapter = { · 7 unmodified lines
141147 executorAddInputSchema: SourceAdapterInputSchema | null;
142148 executorAddHelpText: readonly string[] | null;
143149 executorAddInputSignatureWidth: number | null;
150+ localConfigBindingSchema: SourceAdapterInputSchema | null;
151+ localConfigBindingFromSource: (source: Source) => unknown;
144152 serializeBindingConfig: (source: Source) => string;
145153 deserializeBindingConfig: (
146154 input: Pick<StoredSourceRecord, "id" | "bindingConfigJson">,
@@ -149,6 +157,12 @@ export type SourceAdapter = { · 2 unmodified lines
149157 sourceConfigFromSource: (source: Source) => Record<string, unknown>;
150158 validateSource: (source: Source) => Effect.Effect<Source, Error, never>;
151159 shouldAutoProbe: (source: Source) => boolean;
160+ discoveryPriority?: (input: {
161+ normalizedUrl: string;
162+ }) => number;
163+ detectSource?: (
164+ input: SourceAdapterDiscoveryInput,
165+ ) => Effect.Effect<SourceDiscoveryResult | null, Error, never>;
152166 syncCatalog: (
153167 input: SourceAdapterSyncInput,
154168 ) => Effect.Effect<SourceCatalogSyncResult, Error, never>;
packages/sources/google-discovery/src/adapter.ts+27−2
@@ -12,11 +12,13 @@ import { · 11 unmodified lines
1212 compileGoogleDiscoveryToolDefinitions,
1313 extractGoogleDiscoveryManifest,
1414 } from "./document";
15+import { detectGoogleDiscoverySource } from "./discovery";
1516 import { buildGoogleDiscoveryToolPresentation } from "./tools";
1617 import {
1718 createGoogleDiscoveryCatalogFragment,
1819 type GoogleDiscoveryCatalogOperationInput,
1920 } from "./catalog";
21+import { GoogleDiscoveryLocalConfigBindingSchema } from "./local-config";
2022 import {
2123 GoogleDiscoveryToolProviderDataSchema,
2224 type GoogleDiscoveryToolManifest,
@@ -424,7 +426,7 @@ const googleDiscoveryOauth2SetupConfig = (source: Source) => · 401 unmodified lines
424426 };
425427 });
426428
427−export const googleDiscoverySourceAdapter: SourceAdapter = {
429+export const googleDiscoverySourceAdapter = {
428430 key: "google_discovery",
429431 displayName: "Google Discovery",
430432 catalogKind: "imported",
@@ -439,6 +441,20 @@ export const googleDiscoverySourceAdapter: SourceAdapter = { · 8 unmodified lines
439441 "service is the Discovery service name, e.g. sheets or drive. version is the API version, e.g. v4 or v3.",
440442 ],
441443 executorAddInputSignatureWidth: 420,
444+ localConfigBindingSchema: GoogleDiscoveryLocalConfigBindingSchema,
445+ localConfigBindingFromSource: (source) =>
446+ Effect.runSync(
447+ Effect.map(
448+ googleDiscoveryBindingConfigFromSource(source),
449+ (bindingConfig) => ({
450+ service: bindingConfig.service,
451+ version: bindingConfig.version,
452+ discoveryUrl: bindingConfig.discoveryUrl,
453+ defaultHeaders: bindingConfig.defaultHeaders ?? null,
454+ scopes: bindingConfig.scopes,
455+ }),
456+ ),
457+ ),
442458 serializeBindingConfig: (source) =>
443459 encodeBindingConfig({
444460 adapterKey: "google_discovery",
@@ -508,6 +524,15 @@ export const googleDiscoverySourceAdapter: SourceAdapter = { · 63 unmodified lines
508524 shouldAutoProbe: (source) =>
509525 source.enabled &&
510526 (source.status === "draft" || source.status === "probing"),
527+ discoveryPriority: ({ normalizedUrl }) =>
528+ normalizedUrl.includes("$discovery/rest") || normalizedUrl.includes("/discovery/v1/apis/")
529+ ? 500
530+ : 300,
531+ detectSource: ({ normalizedUrl, headers }) =>
532+ detectGoogleDiscoverySource({
533+ normalizedUrl,
534+ headers,
535+ }),
511536 syncCatalog: ({ source, resolveAuthMaterialForSlot }) =>
512537 Effect.gen(function* () {
513538 const bindingConfig =
@@ -684,4 +709,4 @@ export const googleDiscoverySourceAdapter: SourceAdapter = { · 170 unmodified lines
684709 catch: (cause) =>
685710 cause instanceof Error ? cause : new Error(String(cause)),
686711 }),
687−};
712+} satisfies SourceAdapter;
packages/sources/graphql/src/adapter.ts+19−2
@@ -7,6 +7,7 @@ import { · 6 unmodified lines
77 GraphqlToolProviderDataSchema,
88 type GraphqlToolProviderData,
99 } from "./provider-data";
10+import { detectGraphqlSource } from "./discovery";
1011 import {
1112 createGraphqlCatalogFragment,
1213 type GraphqlCatalogOperationInput,
@@ -17,6 +18,7 @@ import { · 4 unmodified lines
1718 extractGraphqlManifest,
1819 GRAPHQL_INTROSPECTION_QUERY,
1920 } from "./graphql-tools";
21+import { GraphqlLocalConfigBindingSchema } from "./local-config";
2022 import {
2123 ConnectHttpAuthSchema,
2224 ConnectHttpImportAuthSchema,
@@ -27,6 +29,7 @@ import { · 4 unmodified lines
2729 emptySourceBindingState,
2830 encodeBindingConfig,
2931 isSourceCredentialRequiredError,
32+ looksLikeGraphqlEndpoint,
3033 OptionalNullableStringSchema,
3134 SourceCredentialRequiredError,
3235 SourceConnectCommonFieldsSchema,
@@ -255,7 +258,7 @@ const withoutUndefinedEntries = ( · 222 unmodified lines
255258 Object.entries(record).filter(([, value]) => value !== undefined),
256259 );
257260
258−export const graphqlSourceAdapter: SourceAdapter = {
261+export const graphqlSourceAdapter = {
259262 key: "graphql",
260263 displayName: "GraphQL",
261264 catalogKind: "imported",
@@ -268,6 +271,13 @@ export const graphqlSourceAdapter: SourceAdapter = { · 6 unmodified lines
268271 executorAddInputSchema: GraphqlExecutorAddInputSchema,
269272 executorAddHelpText: ["endpoint is the GraphQL HTTP endpoint."],
270273 executorAddInputSignatureWidth: 320,
274+ localConfigBindingSchema: GraphqlLocalConfigBindingSchema,
275+ localConfigBindingFromSource: (source) =>
276+ Effect.runSync(
277+ Effect.map(graphqlBindingConfigFromSource(source), (bindingConfig) => ({
278+ defaultHeaders: bindingConfig.defaultHeaders,
279+ })),
280+ ),
271281 serializeBindingConfig: (source) =>
272282 encodeBindingConfig({
273283 adapterKey: "graphql",
@@ -318,6 +328,13 @@ export const graphqlSourceAdapter: SourceAdapter = { · 44 unmodified lines
318328 shouldAutoProbe: (source) =>
319329 source.enabled &&
320330 (source.status === "draft" || source.status === "probing"),
331+ discoveryPriority: ({ normalizedUrl }) =>
332+ looksLikeGraphqlEndpoint(normalizedUrl) ? 400 : 150,
333+ detectSource: ({ normalizedUrl, headers }) =>
334+ detectGraphqlSource({
335+ normalizedUrl,
336+ headers,
337+ }),
321338 syncCatalog: ({ source, resolveAuthMaterialForSlot }) =>
322339 Effect.gen(function* () {
323340 const bindingConfig = yield* graphqlBindingConfigFromSource(source);
@@ -519,4 +536,4 @@ export const graphqlSourceAdapter: SourceAdapter = { · 195 unmodified lines
519536 catch: (cause) =>
520537 cause instanceof Error ? cause : new Error(String(cause)),
521538 }),
522−};
539+} satisfies SourceAdapter;
packages/sources/mcp/src/adapter.ts+25−2
@@ -12,10 +12,12 @@ import { · 11 unmodified lines
1212 type McpToolManifest,
1313 type McpToolManifestEntry,
1414 } from "./tools";
15+import { detectMcpSource } from "./discovery";
1516 import {
1617 createMcpCatalogFragment,
1718 type McpCatalogOperationInput,
1819 } from "./catalog";
20+import { McpLocalConfigBindingSchema } from "./local-config";
1921 import type { McpServerMetadata } from "./manifest";
2022 import {
2123 contentHash,
@@ -25,6 +27,7 @@ import { · 3 unmodified lines
2527 decodeSourceBindingPayload,
2628 emptySourceBindingState,
2729 encodeBindingConfig,
30+ looksLikeGraphqlEndpoint,
2831 McpConnectFieldsSchema,
2932 OptionalNullableStringSchema,
3033 SourceTransportSchema,
@@ -326,7 +329,7 @@ export const catalogSyncResultFromMcpManifest = (input: { · 295 unmodified lines
326329 });
327330 };
328331
329−export const mcpSourceAdapter: SourceAdapter = {
332+export const mcpSourceAdapter = {
330333 key: "mcp",
331334 displayName: "MCP",
332335 catalogKind: "imported",
@@ -342,6 +345,19 @@ export const mcpSourceAdapter: SourceAdapter = { · 9 unmodified lines
342345 'For local servers, set transport: "stdio" and provide command plus optional args/env/cwd.',
343346 ],
344347 executorAddInputSignatureWidth: 240,
348+ localConfigBindingSchema: McpLocalConfigBindingSchema,
349+ localConfigBindingFromSource: (source) =>
350+ Effect.runSync(
351+ Effect.map(mcpBindingConfigFromSource(source), (bindingConfig) => ({
352+ transport: bindingConfig.transport,
353+ queryParams: bindingConfig.queryParams,
354+ headers: bindingConfig.headers,
355+ command: bindingConfig.command,
356+ args: bindingConfig.args,
357+ env: bindingConfig.env,
358+ cwd: bindingConfig.cwd,
359+ })),
360+ ),
345361 serializeBindingConfig: (source) =>
346362 encodeBindingConfig({
347363 adapterKey: "mcp",
@@ -408,6 +424,13 @@ export const mcpSourceAdapter: SourceAdapter = { · 60 unmodified lines
408424 };
409425 }),
410426 shouldAutoProbe: () => false,
427+ discoveryPriority: ({ normalizedUrl }) =>
428+ looksLikeGraphqlEndpoint(normalizedUrl) ? 350 : 125,
429+ detectSource: ({ normalizedUrl, headers }) =>
430+ detectMcpSource({
431+ normalizedUrl,
432+ headers,
433+ }),
411434 syncCatalog: ({ source, resolveAuthMaterialForSlot }) =>
412435 Effect.gen(function* () {
413436 const bindingConfig = yield* mcpBindingConfigFromSource(source);
@@ -599,4 +622,4 @@ export const mcpSourceAdapter: SourceAdapter = { · 185 unmodified lines
599622 status: null,
600623 };
601624 }),
602−};
625+} satisfies SourceAdapter;
packages/sources/openapi/src/adapter.ts+20−2
@@ -10,12 +10,14 @@ import { · 9 unmodified lines
1010 } from "@executor/codemode-core";
1111 import { buildOpenApiToolPresentation } from "./tool-presentation";
1212 import { compileOpenApiToolDefinitions } from "./definitions";
13+import { detectOpenApiSource } from "./discovery";
1314 import {
1415 httpBodyModeFromContentType,
1516 serializeOpenApiParameterValue,
1617 serializeOpenApiRequestBody,
1718 withSerializedQueryEntries,
1819 } from "./http-serialization";
20+import { OpenApiLocalConfigBindingSchema } from "./local-config";
1921 import {
2022 OpenApiToolProviderDataSchema,
2123 type OpenApiRefHintTable,
@@ -36,6 +38,7 @@ import { · 14 unmodified lines
3638 emptySourceBindingState,
3739 encodeBindingConfig,
3840 isSourceCredentialRequiredError,
41+ looksLikeGraphqlEndpoint,
3942 OptionalNullableStringSchema,
4043 SourceCredentialRequiredError,
4144 SourceConnectCommonFieldsSchema,
@@ -402,7 +405,7 @@ const openApiCatalogOperationFromDefinition = (input: { · 360 unmodified lines
402405 };
403406 };
404407
405−export const openApiSourceAdapter: SourceAdapter = {
408+export const openApiSourceAdapter = {
406409 key: "openapi",
407410 displayName: "OpenAPI",
408411 catalogKind: "imported",
@@ -417,6 +420,14 @@ export const openApiSourceAdapter: SourceAdapter = { · 8 unmodified lines
417420 "endpoint is the base API URL. specUrl is the OpenAPI document URL.",
418421 ],
419422 executorAddInputSignatureWidth: 420,
423+ localConfigBindingSchema: OpenApiLocalConfigBindingSchema,
424+ localConfigBindingFromSource: (source) =>
425+ Effect.runSync(
426+ Effect.map(openApiBindingConfigFromSource(source), (bindingConfig) => ({
427+ specUrl: bindingConfig.specUrl,
428+ defaultHeaders: bindingConfig.defaultHeaders,
429+ })),
430+ ),
420431 serializeBindingConfig: (source) =>
421432 encodeBindingConfig({
422433 adapterKey: "openapi",
@@ -472,6 +483,13 @@ export const openApiSourceAdapter: SourceAdapter = { · 49 unmodified lines
472483 shouldAutoProbe: (source) =>
473484 source.enabled &&
474485 (source.status === "draft" || source.status === "probing"),
486+ discoveryPriority: ({ normalizedUrl }) =>
487+ looksLikeGraphqlEndpoint(normalizedUrl) ? 50 : 250,
488+ detectSource: ({ normalizedUrl, headers }) =>
489+ detectOpenApiSource({
490+ normalizedUrl,
491+ headers,
492+ }),
475493 syncCatalog: ({ source, resolveAuthMaterialForSlot }) =>
476494 Effect.gen(function* () {
477495 const bindingConfig = yield* openApiBindingConfigFromSource(source);
@@ -663,4 +681,4 @@ export const openApiSourceAdapter: SourceAdapter = { · 185 unmodified lines
663681 catch: (cause) =>
664682 cause instanceof Error ? cause : new Error(String(cause)),
665683 }),
666−};
684+} satisfies SourceAdapter;