History

add unified source connect and discover API endpoints

Introduce POST /sources/connect that handles MCP, OpenAPI, and GraphQL connection in a single endpoint (replacing the old MCP-only connect). Add POST /sources/discover for URL-based source type detection. Update sources-operations with auto-probe logic for draft sources.

Rhys Sullivan committed 8 months ago

Commit af725aa1 parent ff7c45f3 files · +266 −90Browse files

Unified · Split

3 files
  1. packages/control-plane/src/api/sources/api.ts +105 −38
  2. packages/control-plane/src/api/sources/http.ts +119 −12
  3. packages/control-plane/src/runtime/sources-operations.ts +42 −40
packages/control-plane/src/api/sources/api.ts+105−38
@@ -1,14 +1,17 @@
1import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "@effect/platform";1import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "@effect/platform";
2import {2import {
3 ExecutionInteractionIdSchema,3 ExecutionInteractionIdSchema,
4 SecretRefSchema,
4 SourceAuthSchema,5 SourceAuthSchema,
5 SourceAuthSessionIdSchema,6 SourceAuthSessionIdSchema,
7 SourceDiscoveryResultSchema,
6 SourceIdSchema,8 SourceIdSchema,
7 SourceInspectionDiscoverPayloadSchema,9 SourceInspectionDiscoverPayloadSchema,
8 SourceInspectionDiscoverResultSchema,10 SourceInspectionDiscoverResultSchema,
9 SourceInspectionSchema,11 SourceInspectionSchema,
10 SourceInspectionToolDetailSchema,12 SourceInspectionToolDetailSchema,
11 SourceKindSchema,13 SourceKindSchema,
14 SourceProbeAuthSchema,
12 SourceSchema,15 SourceSchema,
13 SourceStatusSchema,16 SourceStatusSchema,
14 SourceTransportSchema,17 SourceTransportSchema,
@@ -75,34 +78,6 @@ export const UpdateSourcePayloadSchema = Schema.Struct({ · 60 unmodified lines
75 78
76export type UpdateSourcePayload = typeof UpdateSourcePayloadSchema.Type;79export type UpdateSourcePayload = typeof UpdateSourcePayloadSchema.Type;
77 80
78export const ConnectMcpSourcePayloadSchema = Schema.Struct({
79 sourceId: Schema.optional(SourceIdSchema),
80 name: Schema.optional(Schema.NullOr(Schema.String)),
81 endpoint: TrimmedNonEmptyStringSchema,
82 namespace: Schema.optional(Schema.NullOr(Schema.String)),
83 enabled: Schema.optional(Schema.Boolean),
84 transport: Schema.optional(SourceTransportSchema),
85 queryParams: Schema.optional(Schema.NullOr(StringMapSchema)),
86 headers: Schema.optional(Schema.NullOr(StringMapSchema)),
87});
88
89export type ConnectMcpSourcePayload = typeof ConnectMcpSourcePayloadSchema.Type;
90
91export const ConnectMcpSourceResultSchema = Schema.Union(
92 Schema.Struct({
93 kind: Schema.Literal("connected"),
94 source: SourceSchema,
95 }),
96 Schema.Struct({
97 kind: Schema.Literal("oauth_required"),
98 source: SourceSchema,
99 sessionId: SourceAuthSessionIdSchema,
100 authorizationUrl: Schema.String,
101 }),
102);
103
104export type ConnectMcpSourceResult = typeof ConnectMcpSourceResultSchema.Type;
105
106const workspaceIdParam = HttpApiSchema.param("workspaceId", WorkspaceIdSchema);81const workspaceIdParam = HttpApiSchema.param("workspaceId", WorkspaceIdSchema);
107const sourceIdParam = HttpApiSchema.param("sourceId", SourceIdSchema);82const sourceIdParam = HttpApiSchema.param("sourceId", SourceIdSchema);
108const toolPathParam = HttpApiSchema.param("toolPath", Schema.String);83const toolPathParam = HttpApiSchema.param("toolPath", Schema.String);
@@ -123,11 +98,113 @@ const CredentialOauthCompleteUrlParamsSchema = Schema.Struct({ · 14 unmodified lines
123 error_description: Schema.optional(Schema.String),98 error_description: Schema.optional(Schema.String),
124});99});
125 100
101const DiscoverSourcePayloadSchema = Schema.Struct({
102 url: TrimmedNonEmptyStringSchema,
103 probeAuth: Schema.optional(SourceProbeAuthSchema),
104});
105
106export type DiscoverSourcePayload = typeof DiscoverSourcePayloadSchema.Type;
107
108const ConnectBearerAuthSchema = Schema.Struct({
109 kind: Schema.Literal("bearer"),
110 headerName: Schema.optional(Schema.NullOr(Schema.String)),
111 prefix: Schema.optional(Schema.NullOr(Schema.String)),
112 token: Schema.optional(Schema.NullOr(Schema.String)),
113 tokenRef: Schema.optional(Schema.NullOr(SecretRefSchema)),
114});
115
116const ConnectOAuth2AuthSchema = Schema.Struct({
117 kind: Schema.Literal("oauth2"),
118 headerName: Schema.optional(Schema.NullOr(Schema.String)),
119 prefix: Schema.optional(Schema.NullOr(Schema.String)),
120 accessToken: Schema.optional(Schema.NullOr(Schema.String)),
121 accessTokenRef: Schema.optional(Schema.NullOr(SecretRefSchema)),
122 refreshToken: Schema.optional(Schema.NullOr(Schema.String)),
123 refreshTokenRef: Schema.optional(Schema.NullOr(SecretRefSchema)),
124});
125
126const ConnectHttpAuthSchema = Schema.Union(
127 Schema.Struct({ kind: Schema.Literal("none") }),
128 ConnectBearerAuthSchema,
129 ConnectOAuth2AuthSchema,
130);
131
132const ConnectSourcePayloadSchema = Schema.Union(
133 Schema.Struct({
134 kind: Schema.Literal("openapi"),
135 endpoint: TrimmedNonEmptyStringSchema,
136 specUrl: TrimmedNonEmptyStringSchema,
137 name: Schema.optional(Schema.NullOr(Schema.String)),
138 namespace: Schema.optional(Schema.NullOr(Schema.String)),
139 auth: Schema.optional(ConnectHttpAuthSchema),
140 }),
141 Schema.Struct({
142 kind: Schema.Literal("graphql"),
143 endpoint: TrimmedNonEmptyStringSchema,
144 name: Schema.optional(Schema.NullOr(Schema.String)),
145 namespace: Schema.optional(Schema.NullOr(Schema.String)),
146 auth: Schema.optional(ConnectHttpAuthSchema),
147 }),
148 Schema.Struct({
149 kind: Schema.Literal("mcp"),
150 endpoint: TrimmedNonEmptyStringSchema,
151 name: Schema.optional(Schema.NullOr(Schema.String)),
152 namespace: Schema.optional(Schema.NullOr(Schema.String)),
153 transport: Schema.optional(Schema.NullOr(SourceTransportSchema)),
154 queryParams: Schema.optional(Schema.NullOr(StringMapSchema)),
155 headers: Schema.optional(Schema.NullOr(StringMapSchema)),
156 }),
157);
158
159export type ConnectSourcePayload = typeof ConnectSourcePayloadSchema.Type;
160
161const ConnectSourceResultSchema = Schema.Union(
162 Schema.Struct({
163 kind: Schema.Literal("connected"),
164 source: SourceSchema,
165 }),
166 Schema.Struct({
167 kind: Schema.Literal("credential_required"),
168 source: SourceSchema,
169 }),
170 Schema.Struct({
171 kind: Schema.Literal("oauth_required"),
172 source: SourceSchema,
173 sessionId: SourceAuthSessionIdSchema,
174 authorizationUrl: Schema.String,
175 }),
176);
177
178export type ConnectSourceResult = typeof ConnectSourceResultSchema.Type;
179
180export {
181 ConnectSourcePayloadSchema,
182 ConnectSourceResultSchema,
183 DiscoverSourcePayloadSchema,
184};
185
126const HtmlSchema = HttpApiSchema.Text({186const HtmlSchema = HttpApiSchema.Text({
127 contentType: "text/html",187 contentType: "text/html",
128});188});
129 189
130export class SourcesApi extends HttpApiGroup.make("sources")190export class SourcesApi extends HttpApiGroup.make("sources")
191 .add(
192 HttpApiEndpoint.post("discover")`/sources/discover`
193 .setPayload(DiscoverSourcePayloadSchema)
194 .addSuccess(SourceDiscoveryResultSchema)
195 .addError(ControlPlaneBadRequestError)
196 .addError(ControlPlaneUnauthorizedError)
197 .addError(ControlPlaneForbiddenError),
198 )
199 .add(
200 HttpApiEndpoint.post("connect")`/workspaces/${workspaceIdParam}/sources/connect`
201 .setPayload(ConnectSourcePayloadSchema)
202 .addSuccess(ConnectSourceResultSchema)
203 .addError(ControlPlaneBadRequestError)
204 .addError(ControlPlaneUnauthorizedError)
205 .addError(ControlPlaneForbiddenError)
206 .addError(ControlPlaneStorageError),
207 )
131 .add(208 .add(
132 HttpApiEndpoint.get("list")`/workspaces/${workspaceIdParam}/sources`209 HttpApiEndpoint.get("list")`/workspaces/${workspaceIdParam}/sources`
133 .addSuccess(Schema.Array(SourceSchema))210 .addSuccess(Schema.Array(SourceSchema))
@@ -164,16 +241,6 @@ export class SourcesApi extends HttpApiGroup.make("sources") · 30 unmodified lines
164 .addError(ControlPlaneNotFoundError)241 .addError(ControlPlaneNotFoundError)
165 .addError(ControlPlaneStorageError),242 .addError(ControlPlaneStorageError),
166 )243 )
167 .add(
168 HttpApiEndpoint.post("connectMcp")`/workspaces/${workspaceIdParam}/sources/mcp/oauth/start`
169 .setPayload(ConnectMcpSourcePayloadSchema)
170 .addSuccess(ConnectMcpSourceResultSchema)
171 .addError(ControlPlaneBadRequestError)
172 .addError(ControlPlaneUnauthorizedError)
173 .addError(ControlPlaneForbiddenError)
174 .addError(ControlPlaneNotFoundError)
175 .addError(ControlPlaneStorageError),
176 )
177 .add(244 .add(
178 HttpApiEndpoint.del("remove")`/workspaces/${workspaceIdParam}/sources/${sourceIdParam}`245 HttpApiEndpoint.del("remove")`/workspaces/${workspaceIdParam}/sources/${sourceIdParam}`
179 .addSuccess(Schema.Struct({ removed: Schema.Boolean }))246 .addSuccess(Schema.Struct({ removed: Schema.Boolean }))
packages/control-plane/src/api/sources/http.ts+119−12
@@ -1,5 +1,6 @@
1import {1import {
2 HttpApiBuilder,2 HttpApiBuilder,
3 HttpServerRequest,
3 HttpServerResponse,4 HttpServerResponse,
4} from "@effect/platform";5} from "@effect/platform";
5import type {6import type {
@@ -11,7 +12,6 @@ import * as Effect from "effect/Effect"; · 5 unmodified lines
11 12
12import { requirePermission, withPolicy } from "#domain";13import { requirePermission, withPolicy } from "#domain";
13import {14import {
14 connectMcpSource,
15 createSource,15 createSource,
16 getSource,16 getSource,
17 listSources,17 listSources,
@@ -28,6 +28,8 @@ import { · 10 unmodified lines
28 getSourceCredentialInteraction,28 getSourceCredentialInteraction,
29 submitSourceCredentialInteraction,29 submitSourceCredentialInteraction,
30} from "../../runtime/local-operations";30} from "../../runtime/local-operations";
31import { discoverSource } from "../../runtime/source-discovery";
32import { RuntimeSourceAuthServiceTag } from "../../runtime/source-auth-service";
31 33
32import {34import {
33 ControlPlaneBadRequestError,35 ControlPlaneBadRequestError,
@@ -35,7 +37,7 @@ import { · 1 unmodified line
35 ControlPlaneStorageError,37 ControlPlaneStorageError,
36} from "../errors";38} from "../errors";
37import { ControlPlaneApi } from "../api";39import { ControlPlaneApi } from "../api";
38import { withWorkspaceRequestActor } from "../http-auth";40import { withRequestActor, withWorkspaceRequestActor } from "../http-auth";
39 41
40const requireReadSources = (workspaceId: WorkspaceId) =>42const requireReadSources = (workspaceId: WorkspaceId) =>
41 requirePermission({43 requirePermission({
@@ -49,6 +51,50 @@ const requireWriteSources = (workspaceId: WorkspaceId) => · 7 unmodified lines
49 workspaceId,51 workspaceId,
50 });52 });
51 53
54const readHeader = (headers: unknown, name: string): string | null => {
55 if (headers == null || typeof headers !== "object") {
56 return null;
57 }
58
59 const record = headers as Record<string, unknown>;
60 const exact = record[name];
61 if (typeof exact === "string" && exact.length > 0) {
62 return exact;
63 }
64
65 const lower = record[name.toLowerCase()];
66 if (typeof lower === "string" && lower.length > 0) {
67 return lower;
68 }
69
70 return null;
71};
72
73const resolveRequestOrigin = (request: { url: string; headers: unknown }): string | null => {
74 try {
75 return new URL(request.url).origin;
76 } catch {
77 const forwardedHost = readHeader(request.headers, "x-forwarded-host");
78 const host = forwardedHost ?? readHeader(request.headers, "host");
79 if (!host) {
80 return null;
81 }
82
83 const forwardedProto = readHeader(request.headers, "x-forwarded-proto");
84 const protocol = forwardedProto && forwardedProto.length > 0 ? forwardedProto : "http";
85 return `${protocol}://${host}`;
86 }
87};
88
89const toBadRequestError = (operation: string, cause: unknown) => {
90 const message = cause instanceof Error ? cause.message : String(cause);
91 return new ControlPlaneBadRequestError({
92 operation,
93 message,
94 details: message,
95 });
96};
97
52const escapeHtml = (value: string): string =>98const escapeHtml = (value: string): string =>
53 value99 value
54 .replaceAll("&", "&amp;")100 .replaceAll("&", "&amp;")
@@ -616,6 +662,77 @@ export const ControlPlaneSourcesLive = HttpApiBuilder.group( · 561 unmodified lines
616 "sources",662 "sources",
617 (handlers) =>663 (handlers) =>
618 handlers664 handlers
665 .handle("discover", ({ payload }) =>
666 withRequestActor("sources.discover", () =>
667 discoverSource({
668 url: payload.url,
669 probeAuth: payload.probeAuth,
670 }).pipe(
671 Effect.catchAll((cause) =>
672 Effect.fail(toBadRequestError("sources.discover", cause)),
673 ),
674 ),
675 ),
676 )
677 .handle("connect", ({ path, payload }) =>
678 withWorkspaceRequestActor("sources.connect", path.workspaceId, () =>
679 withPolicy(requireWriteSources(path.workspaceId))(
680 Effect.gen(function* () {
681 const request = yield* HttpServerRequest.HttpServerRequest;
682 const sourceAuthService = yield* RuntimeSourceAuthServiceTag;
683 const baseUrl = resolveRequestOrigin(request);
684
685 if (payload.kind === "mcp") {
686 return yield* sourceAuthService.connectMcpSource({
687 workspaceId: path.workspaceId,
688 endpoint: payload.endpoint,
689 name: payload.name,
690 namespace: payload.namespace,
691 transport: payload.transport,
692 queryParams: payload.queryParams,
693 headers: payload.headers,
694 baseUrl,
695 });
696 }
697
698 if (payload.kind === "openapi") {
699 return yield* sourceAuthService.addExecutorSource(
700 {
701 workspaceId: path.workspaceId,
702 executionId: null,
703 interactionId: null,
704 kind: "openapi",
705 endpoint: payload.endpoint,
706 specUrl: payload.specUrl,
707 name: payload.name,
708 namespace: payload.namespace,
709 auth: payload.auth,
710 },
711 { baseUrl },
712 );
713 }
714
715 return yield* sourceAuthService.addExecutorSource(
716 {
717 workspaceId: path.workspaceId,
718 executionId: null,
719 interactionId: null,
720 kind: "graphql",
721 endpoint: payload.endpoint,
722 name: payload.name,
723 namespace: payload.namespace,
724 auth: payload.auth,
725 },
726 { baseUrl },
727 );
728 }).pipe(
729 Effect.catchAll((cause) =>
730 Effect.fail(toBadRequestError("sources.connect", cause)),
731 ),
732 ),
733 ),
734 ),
735 )
619 .handle("list", ({ path }) =>736 .handle("list", ({ path }) =>
620 withWorkspaceRequestActor("sources.list", path.workspaceId, () =>737 withWorkspaceRequestActor("sources.list", path.workspaceId, () =>
621 withPolicy(requireReadSources(path.workspaceId))(738 withPolicy(requireReadSources(path.workspaceId))(
@@ -683,16 +800,6 @@ export const ControlPlaneSourcesLive = HttpApiBuilder.group( · 61 unmodified lines
683 ),800 ),
684 ),801 ),
685 )802 )
686 .handle("connectMcp", ({ path, payload }) =>
687 withWorkspaceRequestActor("sources.connectMcp", path.workspaceId, () =>
688 withPolicy(requireWriteSources(path.workspaceId))(
689 connectMcpSource({
690 workspaceId: path.workspaceId,
691 payload,
692 }),
693 ),
694 ),
695 )
696 .handle("remove", ({ path }) =>803 .handle("remove", ({ path }) =>
697 withWorkspaceRequestActor("sources.remove", path.workspaceId, () =>804 withWorkspaceRequestActor("sources.remove", path.workspaceId, () =>
698 withPolicy(requireWriteSources(path.workspaceId))(805 withPolicy(requireWriteSources(path.workspaceId))(
packages/control-plane/src/runtime/sources-operations.ts+42−40
@@ -1,5 +1,4 @@
1import type {1import type {
2 ConnectMcpSourcePayload,
3 CreateSourcePayload,2 CreateSourcePayload,
4 UpdateSourcePayload,3 UpdateSourcePayload,
5} from "../api/sources/api";4} from "../api/sources/api";
@@ -24,7 +23,6 @@ import { · 18 unmodified lines
24 operationErrors,23 operationErrors,
25} from "./operation-errors";24} from "./operation-errors";
26import { createDefaultSecretMaterialResolver } from "./secret-material-providers";25import { createDefaultSecretMaterialResolver } from "./secret-material-providers";
27import { RuntimeSourceAuthServiceTag } from "./source-auth-service";
28import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";26import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";
29import { syncSourceToolArtifacts } from "./tool-artifacts";27import { syncSourceToolArtifacts } from "./tool-artifacts";
30import {28import {
@@ -39,10 +37,17 @@ const sourceOps = { · 8 unmodified lines
39 create: operationErrors("sources.create"),37 create: operationErrors("sources.create"),
40 get: operationErrors("sources.get"),38 get: operationErrors("sources.get"),
41 update: operationErrors("sources.update"),39 update: operationErrors("sources.update"),
42 connectMcp: operationErrors("sources.connectMcp"),
43 remove: operationErrors("sources.remove"),40 remove: operationErrors("sources.remove"),
44} as const;41} as const;
45 42
43const shouldAutoProbeSource = (source: Source): boolean =>
44 source.enabled
45 && (
46 (source.kind === "openapi" && !!source.specUrl)
47 || source.kind === "graphql"
48 )
49 && (source.status === "draft" || source.status === "probing");
50
46const syncArtifactsForSource = (input: {51const syncArtifactsForSource = (input: {
47 store: ControlPlaneStoreShape;52 store: ControlPlaneStoreShape;
48 source: Source;53 source: Source;
@@ -55,19 +60,50 @@ const syncArtifactsForSource = (input: { · 6 unmodified lines
55 rows: input.store,60 rows: input.store,
56 });61 });
57 62
63 // For HTTP-backed source kinds that can validate themselves from a remote
64 // document, automatically attempt to probe and connect. This mirrors the
65 // addExecutorSource flow by overriding status to "connected" so the sync
66 // guard passes.
67 const autoProbe = shouldAutoProbeSource(input.source);
68 const sourceForSync = autoProbe
69 ? { ...input.source, status: "connected" as const }
70 : input.source;
71
58 const synced = yield* Effect.either(72 const synced = yield* Effect.either(
59 syncSourceToolArtifacts({73 syncSourceToolArtifacts({
60 rows: input.store,74 rows: input.store,
61 source: input.source,75 source: sourceForSync,
62 resolveSecretMaterial,76 resolveSecretMaterial,
63 }),77 }),
64 );78 );
65 79
66 return yield* Either.match(synced, {80 return yield* Either.match(synced, {
67 onRight: () => Effect.succeed(input.source),81 onRight: () =>
82 Effect.gen(function* () {
83 if (autoProbe) {
84 const connectedSource = yield* updateSourceFromPayload({
85 source: input.source,
86 payload: { status: "connected", lastError: null },
87 now: Date.now(),
88 }).pipe(
89 Effect.mapError((cause) =>
90 input.operation.badRequest(
91 "Failed updating source status",
92 cause instanceof Error ? cause.message : String(cause),
93 ),
94 ),
95 );
96 yield* mapPersistenceError(
97 input.operation.child("source_connected"),
98 persistSource(input.store, connectedSource),
99 );
100 return connectedSource;
101 }
102 return input.source;
103 }),
68 onLeft: (error) =>104 onLeft: (error) =>
69 Effect.gen(function* () {105 Effect.gen(function* () {
70 if (input.source.enabled && input.source.status === "connected") {106 if (autoProbe || (input.source.enabled && input.source.status === "connected")) {
71 const erroredSource = yield* updateSourceFromPayload({107 const erroredSource = yield* updateSourceFromPayload({
72 source: input.source,108 source: input.source,
73 payload: {109 payload: {
@@ -213,40 +249,6 @@ export const updateSource = (input: { · 139 unmodified lines
213 });249 });
214 }));250 }));
215 251
216export const connectMcpSource = (input: {
217 workspaceId: WorkspaceId;
218 payload: ConnectMcpSourcePayload;
219}) =>
220 Effect.gen(function* () {
221 const sourceAuthService = yield* RuntimeSourceAuthServiceTag;
222
223 return yield* sourceAuthService.connectMcpSource({
224 workspaceId: input.workspaceId,
225 sourceId: input.payload.sourceId,
226 endpoint: input.payload.endpoint,
227 name: input.payload.name,
228 namespace: input.payload.namespace,
229 enabled: input.payload.enabled,
230 transport: input.payload.transport,
231 queryParams: input.payload.queryParams,
232 headers: input.payload.headers,
233 }).pipe(
234 Effect.mapError((cause) => {
235 if (cause instanceof Error && cause.message.startsWith("Source not found:")) {
236 return sourceOps.connectMcp.notFound(
237 "Source not found",
238 `workspaceId=${input.workspaceId} sourceId=${input.payload.sourceId}`,
239 );
240 }
241
242 return sourceOps.connectMcp.unknownStorage(
243 cause,
244 "Failed preparing MCP source OAuth flow",
245 );
246 }),
247 );
248 });
249
250export const removeSource = (input: {252export const removeSource = (input: {
251 workspaceId: WorkspaceId;253 workspaceId: WorkspaceId;
252 sourceId: SourceId;254 sourceId: SourceId;