History

Refactor control-plane runtime services

Rhys Sullivan committed 8 months ago

Commit 88b34681 parent 0cf170c33 files · +1602 −784Browse files

Unified · Split

33 files
  1. apps/executor/src/server/server.test.ts +1 −1
  2. packages/control-plane/src/api/local/http.ts +21 −10
  3. packages/control-plane/src/runtime/control-plane-runtime.test.ts +53 −30
  4. packages/control-plane/src/runtime/execution-service.ts +15 −5
  5. packages/control-plane/src/runtime/executor-tools.ts +14 −12
  6. packages/control-plane/src/runtime/index.ts +90 −75
  7. packages/control-plane/src/runtime/live-execution.ts +5 −0
  8. packages/control-plane/src/runtime/local-config.test.ts +4 −3
  9. packages/control-plane/src/runtime/local-config.ts +22 −23
  10. packages/control-plane/src/runtime/local-control-plane-store.ts +4 −42
  11. packages/control-plane/src/runtime/local-installation.test.ts +2 −1
  12. packages/control-plane/src/runtime/local-runtime-context.ts +13 −0
  13. packages/control-plane/src/runtime/local-source-artifacts.ts +10 −19
  14. packages/control-plane/src/runtime/local-storage.ts +85 −14
  15. packages/control-plane/src/runtime/local-tools.ts +88 −65
  16. packages/control-plane/src/runtime/local-workspace-state.ts +10 −16
  17. packages/control-plane/src/runtime/local-workspace-sync.ts +3 −2
  18. packages/control-plane/src/runtime/secret-material-providers.ts +108 −0
  19. packages/control-plane/src/runtime/source-adapters/google-discovery.ts +8 −0
  20. packages/control-plane/src/runtime/source-adapters/graphql.ts +2 −0
  21. packages/control-plane/src/runtime/source-adapters/mcp.ts +2 −0
  22. packages/control-plane/src/runtime/source-adapters/openapi.ts +2 −0
  23. packages/control-plane/src/runtime/source-adapters/types.ts +0 −2
  24. packages/control-plane/src/runtime/source-auth-material.test.ts +94 −88
  25. packages/control-plane/src/runtime/source-auth-material.ts +43 −2
  26. packages/control-plane/src/runtime/source-auth-service.ts +87 −90
  27. packages/control-plane/src/runtime/source-inspection.ts +5 −9
  28. packages/control-plane/src/runtime/source-materialization.ts +165 −43
  29. packages/control-plane/src/runtime/source-recipes-runtime.ts +255 −67
  30. packages/control-plane/src/runtime/source-store.ts +306 −55
  31. packages/control-plane/src/runtime/sources-operations.ts +22 −20
  32. packages/control-plane/src/runtime/store.ts +0 −5
  33. packages/control-plane/src/runtime/workspace-execution-environment.ts +63 −85
apps/executor/src/server/server.test.ts+1−1
@@ -171,7 +171,7 @@ const writeConfiguredLocalMcpSource = (input: { · 170 unmodified lines
171 materialization,171 materialization,
172 }),172 }),
173 });173 });
174 });174 }).pipe(Effect.provide(NodeFileSystem.layer));
175 175
176const makeServer = createIsolatedLocalExecutorServer({176const makeServer = createIsolatedLocalExecutorServer({
177 port: 0,177 port: 0,
packages/control-plane/src/api/local/http.ts+21−10
@@ -9,12 +9,14 @@ import { SecretMaterialIdSchema } from "#schema"; · 8 unmodified lines
9import {9import {
10 getLocalInstallation,10 getLocalInstallation,
11} from "../../runtime/local-operations";11} from "../../runtime/local-operations";
12import { requireRuntimeLocalWorkspace } from "../../runtime/local-runtime-context";
12import {13import {
13 ENV_SECRET_PROVIDER_ID,14 ENV_SECRET_PROVIDER_ID,
14 KEYCHAIN_SECRET_PROVIDER_ID,15 KEYCHAIN_SECRET_PROVIDER_ID,
15 LOCAL_SECRET_PROVIDER_ID,16 LOCAL_SECRET_PROVIDER_ID,
16 parseSecretStoreProviderId,17 parseSecretStoreProviderId,
17} from "../../runtime/secret-material-providers";18} from "../../runtime/secret-material-providers";
19import { RuntimeSourceStoreService } from "../../runtime/source-store";
18import { ControlPlaneStore } from "../../runtime/store";20import { ControlPlaneStore } from "../../runtime/store";
19import type {21import type {
20 CreateSecretResult,22 CreateSecretResult,
@@ -84,16 +86,25 @@ export const ControlPlaneLocalLive = HttpApiBuilder.group( · 63 unmodified lines
84 .handle("config", () =>86 .handle("config", () =>
85 getInstanceConfig(),87 getInstanceConfig(),
86 )88 )
87 .handle("listSecrets", () =>89 .handle("listSecrets", () =>
88 Effect.gen(function* () {90 Effect.gen(function* () {
89 const store = yield* ControlPlaneStore;91 const store = yield* ControlPlaneStore;
90 const rows = yield* store.secretMaterials.listAll().pipe(92 const sourceStore = yield* RuntimeSourceStoreService;
91 Effect.mapError(() => storageError("Failed listing secrets.")),93 const runtimeLocalWorkspace = yield* requireRuntimeLocalWorkspace().pipe(
92 );94 Effect.mapError(() => storageError("Failed resolving local workspace.")),
93 const linkedSourcesMap = yield* store.secretMaterials.listLinkedSources().pipe(95 );
94 Effect.mapError(() => storageError("Failed loading linked sources.")),96 const rows = yield* store.secretMaterials.listAll().pipe(
95 );97 Effect.mapError(() => storageError("Failed listing secrets.")),
96 return rows.map((row) => ({98 );
99 const linkedSourcesMap = yield* sourceStore.listLinkedSecretSourcesInWorkspace(
100 runtimeLocalWorkspace.installation.workspaceId,
101 {
102 actorAccountId: runtimeLocalWorkspace.installation.accountId,
103 },
104 ).pipe(
105 Effect.mapError(() => storageError("Failed loading linked sources.")),
106 );
107 return rows.map((row) => ({
97 ...row,108 ...row,
98 linkedSources: linkedSourcesMap.get(row.id) ?? [],109 linkedSources: linkedSourcesMap.get(row.id) ?? [],
99 }));110 }));
packages/control-plane/src/runtime/control-plane-runtime.test.ts+53−30
@@ -1,8 +1,9 @@
1import { createServer } from "node:http";1import { createServer } from "node:http";
2import { mkdtempSync, readFileSync } from "node:fs";2import { readFileSync } from "node:fs";
3import { tmpdir } from "node:os";
4import { join } from "node:path";3import { join } from "node:path";
5 4
5import { FileSystem } from "@effect/platform";
6import { NodeFileSystem } from "@effect/platform-node";
6import { describe, expect, it } from "@effect/vitest";7import { describe, expect, it } from "@effect/vitest";
7import { assertTrue } from "@effect/vitest/utils";8import { assertTrue } from "@effect/vitest/utils";
8import * as Effect from "effect/Effect";9import * as Effect from "effect/Effect";
@@ -13,6 +14,7 @@ import type { AccountId } from "#schema"; · 4 unmodified lines
13import {14import {
14 ExecutionIdSchema,15 ExecutionIdSchema,
15 ExecutionInteractionIdSchema,16 ExecutionInteractionIdSchema,
17 type ExecutionInteraction,
16 SecretMaterialIdSchema,18 SecretMaterialIdSchema,
17 SourceIdSchema,19 SourceIdSchema,
18} from "#schema";20} from "#schema";
@@ -21,19 +23,24 @@ import type { ToolPath } from "@executor/codemode-core"; · 2 unmodified lines
21import {23import {
22 createControlPlaneRuntime,24 createControlPlaneRuntime,
23 LiveExecutionManagerService,25 LiveExecutionManagerService,
26 provideControlPlaneRuntime,
24} from "./index";27} from "./index";
25import { createSourceFromPayload } from "./source-definitions";28import { createSourceFromPayload } from "./source-definitions";
26import { decodeSourceCredentialSelectionContent } from "./source-credential-interactions";29import { decodeSourceCredentialSelectionContent } from "./source-credential-interactions";
27import { persistSource } from "./source-store";30import { persistSource } from "./source-store";
28import { withControlPlaneClient } from "./test-http-client";31import { withControlPlaneClient } from "./test-http-client";
29 32
30const makeRuntime = Effect.acquireRelease(33const makeRuntime = Effect.gen(function* () {
31 createControlPlaneRuntime({34 const fs = yield* FileSystem.FileSystem;
32 localDataDir: ":memory:",35 const workspaceRoot = yield* fs.makeTempDirectoryScoped({
33 workspaceRoot: mkdtempSync(join(tmpdir(), "executor-control-plane-runtime-")),36 prefix: "executor-control-plane-runtime-",
34 }),37 });
35 (runtime) => Effect.promise(() => runtime.close()).pipe(Effect.orDie),38
36);39 return yield* Effect.acquireRelease(
40 createControlPlaneRuntime({ workspaceRoot }),
41 (runtime) => Effect.promise(() => runtime.close()).pipe(Effect.orDie),
42 );
43}).pipe(Effect.provide(NodeFileSystem.layer));
37 44
38type OpenApiSpecServer = {45type OpenApiSpecServer = {
39 baseUrl: string;46 baseUrl: string;
@@ -121,10 +128,12 @@ const expectLeft = <A, E>(effect: Effect.Effect<A, E, never>) => · 81 unmodified lines
121describe("control-plane-runtime", () => {128describe("control-plane-runtime", () => {
122 it.scoped("writes local source changes through executor.jsonc", () =>129 it.scoped("writes local source changes through executor.jsonc", () =>
123 Effect.gen(function* () {130 Effect.gen(function* () {
124 const workspaceRoot = mkdtempSync(join(tmpdir(), "executor-local-config-runtime-"));131 const fs = yield* FileSystem.FileSystem;
132 const workspaceRoot = yield* fs.makeTempDirectoryScoped({
133 prefix: "executor-local-config-runtime-",
134 });
125 const runtime = yield* Effect.acquireRelease(135 const runtime = yield* Effect.acquireRelease(
126 createControlPlaneRuntime({136 createControlPlaneRuntime({
127 localDataDir: ":memory:",
128 workspaceRoot,137 workspaceRoot,
129 }),138 }),
130 (createdRuntime) => Effect.promise(() => createdRuntime.close()).pipe(Effect.orDie),139 (createdRuntime) => Effect.promise(() => createdRuntime.close()).pipe(Effect.orDie),
@@ -174,7 +183,7 @@ describe("control-plane-runtime", () => { · 43 unmodified lines
174 sources?: Record<string, unknown>;183 sources?: Record<string, unknown>;
175 };184 };
176 expect(removedConfig.sources?.github).toBeUndefined();185 expect(removedConfig.sources?.github).toBeUndefined();
177 }),186 }).pipe(Effect.provide(NodeFileSystem.layer)),
178 60_000,187 60_000,
179 );188 );
180 189
@@ -228,10 +237,7 @@ describe("control-plane-runtime", () => { · 47 unmodified lines
228 }).pipe(Effect.orDie);237 }).pipe(Effect.orDie);
229 yield* persistSource(runtime.persistence.rows, localSource, {238 yield* persistSource(runtime.persistence.rows, localSource, {
230 actorAccountId: installation.accountId,239 actorAccountId: installation.accountId,
231 }).pipe(240 }).pipe((effect) => provideControlPlaneRuntime(effect, runtime), Effect.orDie);
232 Effect.provide(runtime.runtimeLayer),
233 Effect.orDie,
234 );
235 241
236 const interactionFiber = yield* Effect.gen(function* () {242 const interactionFiber = yield* Effect.gen(function* () {
237 const liveExecutionManager = yield* LiveExecutionManagerService;243 const liveExecutionManager = yield* LiveExecutionManagerService;
@@ -260,14 +266,24 @@ describe("control-plane-runtime", () => { · 22 unmodified lines
260 },266 },
261 });267 });
262 }).pipe(268 }).pipe(
263 Effect.provide(runtime.runtimeLayer),269 (effect) => provideControlPlaneRuntime(effect, runtime),
264 Effect.fork,270 Effect.fork,
265 );271 );
266 272
267 yield* Effect.yieldNow();273 const waitForPendingInteraction = (
268 274 remaining: number,
269 const pendingInteraction = yield* runtime.persistence.rows.executionInteractions275 ): Effect.Effect<Option.Option<ExecutionInteraction>, Error> =>
270 .getPendingByExecutionId(executionId);276 runtime.persistence.rows.executionInteractions.getPendingByExecutionId(executionId).pipe(
277 Effect.flatMap((pendingInteraction) =>
278 Option.isSome(pendingInteraction) || remaining <= 0
279 ? Effect.succeed(pendingInteraction)
280 : Effect.yieldNow().pipe(
281 Effect.zipRight(waitForPendingInteraction(remaining - 1)),
282 ),
283 ),
284 );
285
286 const pendingInteraction = yield* waitForPendingInteraction(20);
271 assertTrue(Option.isSome(pendingInteraction));287 assertTrue(Option.isSome(pendingInteraction));
272 expect(pendingInteraction.value.id).toBe(interactionId);288 expect(pendingInteraction.value.id).toBe(interactionId);
273 289
@@ -391,10 +407,7 @@ describe("control-plane-runtime", () => { · 117 unmodified lines
391 }).pipe(Effect.orDie);407 }).pipe(Effect.orDie);
392 yield* persistSource(runtime.persistence.rows, localSource, {408 yield* persistSource(runtime.persistence.rows, localSource, {
393 actorAccountId: installation.accountId,409 actorAccountId: installation.accountId,
394 }).pipe(410 }).pipe((effect) => provideControlPlaneRuntime(effect, runtime), Effect.orDie);
395 Effect.provide(runtime.runtimeLayer),
396 Effect.orDie,
397 );
398 411
399 const interactionFiber = yield* Effect.gen(function* () {412 const interactionFiber = yield* Effect.gen(function* () {
400 const liveExecutionManager = yield* LiveExecutionManagerService;413 const liveExecutionManager = yield* LiveExecutionManagerService;
@@ -423,14 +436,24 @@ describe("control-plane-runtime", () => { · 22 unmodified lines
423 },436 },
424 });437 });
425 }).pipe(438 }).pipe(
426 Effect.provide(runtime.runtimeLayer),439 (effect) => provideControlPlaneRuntime(effect, runtime),
427 Effect.fork,440 Effect.fork,
428 );441 );
429 442
430 yield* Effect.yieldNow();443 const waitForPendingInteraction = (
431 444 remaining: number,
432 const pendingInteraction = yield* runtime.persistence.rows.executionInteractions445 ): Effect.Effect<Option.Option<ExecutionInteraction>, Error> =>
433 .getPendingByExecutionId(executionId);446 runtime.persistence.rows.executionInteractions.getPendingByExecutionId(executionId).pipe(
447 Effect.flatMap((pendingInteraction) =>
448 Option.isSome(pendingInteraction) || remaining <= 0
449 ? Effect.succeed(pendingInteraction)
450 : Effect.yieldNow().pipe(
451 Effect.zipRight(waitForPendingInteraction(remaining - 1)),
452 ),
453 ),
454 );
455
456 const pendingInteraction = yield* waitForPendingInteraction(20);
434 assertTrue(Option.isSome(pendingInteraction));457 assertTrue(Option.isSome(pendingInteraction));
435 458
436 const submittedPage = yield* withControlPlaneClient(459 const submittedPage = yield* withControlPlaneClient(
packages/control-plane/src/runtime/execution-service.ts+15−5
@@ -34,6 +34,10 @@ import { · 33 unmodified lines
34 sanitizePersistedElicitationResponse,34 sanitizePersistedElicitationResponse,
35 type LiveExecutionManager,35 type LiveExecutionManager,
36} from "./live-execution";36} from "./live-execution";
37import {
38 getRuntimeLocalWorkspaceOption,
39 provideOptionalRuntimeLocalWorkspace,
40} from "./local-runtime-context";
37import {41import {
38 asOperationErrors,42 asOperationErrors,
39 operationErrors,43 operationErrors,
@@ -658,16 +662,22 @@ const forkExecutionAttemptWithDependencies = ( · 618 unmodified lines
658 execution: Execution,662 execution: Execution,
659 interactionMode: InteractionMode,663 interactionMode: InteractionMode,
660) =>664) =>
661 Effect.sync(() => {665 Effect.gen(function* () {
662 Effect.runFork(666 const runtimeLocalWorkspace = yield* getRuntimeLocalWorkspaceOption();
663 runExecutionAttemptWithDependencies(667
668 yield* Effect.sync(() => {
669 const effect = runExecutionAttemptWithDependencies(
664 store,670 store,
665 executionResolver,671 executionResolver,
666 liveExecutionManager,672 liveExecutionManager,
667 execution,673 execution,
668 interactionMode,674 interactionMode,
669 ),675 );
670 );676
677 Effect.runFork(
678 provideOptionalRuntimeLocalWorkspace(effect, runtimeLocalWorkspace),
679 );
680 });
671 });681 });
672 682
673const submitExecutionInteractionResponseWithDependencies = (683const submitExecutionInteractionResponseWithDependencies = (
packages/control-plane/src/runtime/executor-tools.ts+14−12
@@ -18,33 +18,35 @@ import { · 17 unmodified lines
18import * as Effect from "effect/Effect";18import * as Effect from "effect/Effect";
19import * as Cause from "effect/Cause";19import * as Cause from "effect/Cause";
20import * as Exit from "effect/Exit";20import * as Exit from "effect/Exit";
21import * as Layer from "effect/Layer";
22import * as Schema from "effect/Schema";21import * as Schema from "effect/Schema";
23import type { RuntimeLocalWorkspaceState } from "./local-runtime-context";22import type { RuntimeLocalWorkspaceState } from "./local-runtime-context";
24import {23import {
25 InstallationStore,
26 type LocalStorageServices,24 type LocalStorageServices,
27 LocalInstallationStore,25 LocalInstallationStore,
28 LocalSourceArtifactStore,26 LocalSourceArtifactStore,
29 LocalWorkspaceConfigStore,27 LocalWorkspaceConfigStore,
30 LocalWorkspaceStateStore,28 LocalWorkspaceStateStore,
31 SourceArtifactStore,29 makeLocalStorageLayer,
32 WorkspaceConfigStore,
33 WorkspaceStateStore,
34} from "./local-storage";30} from "./local-storage";
31import { provideOptionalRuntimeLocalWorkspace } from "./local-runtime-context";
35 32
36/** Run an Effect as a Promise, preserving the original error (not FiberFailure). */33/** Run an Effect as a Promise, preserving the original error (not FiberFailure). */
37const runEffect = async <A>(34const runEffect = async <A>(
38 effect: Effect.Effect<A, unknown, LocalStorageServices>,35 effect: Effect.Effect<A, unknown, LocalStorageServices>,
39 _runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null = null,36 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null = null,
40): Promise<A> => {37): Promise<A> => {
41 const baseLayer = Layer.mergeAll(38 const baseLayer = makeLocalStorageLayer({
42 Layer.succeed(InstallationStore, LocalInstallationStore),39 installationStore: LocalInstallationStore,
43 Layer.succeed(WorkspaceConfigStore, LocalWorkspaceConfigStore),40 workspaceConfigStore: LocalWorkspaceConfigStore,
44 Layer.succeed(WorkspaceStateStore, LocalWorkspaceStateStore),41 workspaceStateStore: LocalWorkspaceStateStore,
45 Layer.succeed(SourceArtifactStore, LocalSourceArtifactStore),42 sourceArtifactStore: LocalSourceArtifactStore,
43 });
44 const exit = await Effect.runPromiseExit(
45 provideOptionalRuntimeLocalWorkspace(
46 effect.pipe(Effect.provide(baseLayer)),
47 runtimeLocalWorkspace,
48 ),
46 );49 );
47 const exit = await Effect.runPromiseExit(effect.pipe(Effect.provide(baseLayer)));
48 if (Exit.isSuccess(exit)) return exit.value;50 if (Exit.isSuccess(exit)) return exit.value;
49 throw Cause.squash(exit.cause);51 throw Cause.squash(exit.cause);
50};52};
packages/control-plane/src/runtime/index.ts+90−75
@@ -1,7 +1,9 @@
1import * as Effect from "effect/Effect";1import * as Effect from "effect/Effect";
2import * as Exit from "effect/Exit";2import * as Exit from "effect/Exit";
3import * as Layer from "effect/Layer";3import * as Layer from "effect/Layer";
4import * as ManagedRuntime from "effect/ManagedRuntime";
4import * as Scope from "effect/Scope";5import * as Scope from "effect/Scope";
6import { NodeFileSystem } from "@effect/platform-node";
5 7
6import { type ControlPlaneApiRuntimeContext } from "#api";8import { type ControlPlaneApiRuntimeContext } from "#api";
7import type { LocalInstallation } from "#schema";9import type { LocalInstallation } from "#schema";
@@ -9,7 +11,6 @@ import type { LocalInstallation } from "#schema"; · 1 unmodified line
9import { type ResolveExecutionEnvironment } from "./execution-state";11import { type ResolveExecutionEnvironment } from "./execution-state";
10import {12import {
11 createLiveExecutionManager,13 createLiveExecutionManager,
12 LiveExecutionManagerLive,
13 LiveExecutionManagerService,14 LiveExecutionManagerService,
14} from "./live-execution";15} from "./live-execution";
15import {16import {
@@ -20,36 +21,28 @@ import { · 4 unmodified lines
20 resolveLocalWorkspaceContext,21 resolveLocalWorkspaceContext,
21} from "./local-config";22} from "./local-config";
22import {23import {
23 InstallationStore,24 LocalStorageLive,
24 LocalInstallationStore,25 LocalInstallationStore,
25 LocalInstallationStoreLive,
26 LocalSourceArtifactStore,
27 LocalSourceArtifactStoreLive,
28 SourceArtifactStore,
29 LocalWorkspaceConfigStore,26 LocalWorkspaceConfigStore,
30 LocalWorkspaceConfigStoreLive,
31 LocalWorkspaceStateStore,
32 LocalWorkspaceStateStoreLive,
33 WorkspaceConfigStore,
34 WorkspaceStateStore,
35} from "./local-storage";27} from "./local-storage";
36import {28import {
37 type RuntimeLocalWorkspaceState,29 type RuntimeLocalWorkspaceState,
38 RuntimeLocalWorkspaceService,30 RuntimeLocalWorkspaceLive,
39} from "./local-runtime-context";31} from "./local-runtime-context";
32import { LocalToolRuntimeLoaderLive } from "./local-tools";
40import { synchronizeLocalWorkspaceState } from "./local-workspace-sync";33import { synchronizeLocalWorkspaceState } from "./local-workspace-sync";
41import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";34import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";
35import { RuntimeSourceStoreLive } from "./source-store";
36import { RuntimeSourceRecipeStoreLive } from "./source-recipes-runtime";
37import { RuntimeSourceAuthMaterialLive } from "./source-auth-material";
38import { RuntimeSourceMaterializationLive } from "./source-materialization";
42import {39import {
43 createRuntimeSourceAuthService,
44 RuntimeSourceAuthServiceLive,40 RuntimeSourceAuthServiceLive,
45 RuntimeSourceAuthServiceTag,
46} from "./source-auth-service";41} from "./source-auth-service";
47import type { ResolveSecretMaterial } from "./secret-material-providers";42import type { ResolveSecretMaterial } from "./secret-material-providers";
48import { createDefaultSecretMaterialResolver } from "./secret-material-providers";43import { SecretMaterialLive } from "./secret-material-providers";
49import {44import {
50 createWorkspaceExecutionEnvironmentResolver,
51 RuntimeExecutionResolverLive,45 RuntimeExecutionResolverLive,
52 RuntimeExecutionResolverService,
53} from "./workspace-execution-environment";46} from "./workspace-execution-environment";
54 47
55export * from "./execution-state";48export * from "./execution-state";
@@ -101,34 +94,84 @@ export const createRuntimeControlPlaneLayer = ( · 45 unmodified lines
101 input: RuntimeControlPlaneOptions & {94 input: RuntimeControlPlaneOptions & {
102 store: ControlPlaneStoreShape;95 store: ControlPlaneStoreShape;
103 localWorkspaceState: RuntimeLocalWorkspaceState;96 localWorkspaceState: RuntimeLocalWorkspaceState;
97 liveExecutionManager: ReturnType<typeof createLiveExecutionManager>;
104 },98 },
105) => {99) => {
100 const platformLayer = NodeFileSystem.layer;
101 const storageLayer = LocalStorageLive.pipe(
102 Layer.provide(platformLayer),
103 );
104 const localToolRuntimeLayer = LocalToolRuntimeLoaderLive.pipe(
105 Layer.provide(platformLayer),
106 );
107
106 const baseLayer = Layer.mergeAll(108 const baseLayer = Layer.mergeAll(
109 platformLayer,
107 Layer.succeed(ControlPlaneStore, input.store),110 Layer.succeed(ControlPlaneStore, input.store),
108 Layer.succeed(RuntimeLocalWorkspaceService, input.localWorkspaceState),111 RuntimeLocalWorkspaceLive(input.localWorkspaceState),
109 LocalInstallationStoreLive,112 storageLayer,
110 LocalWorkspaceConfigStoreLive,113 Layer.succeed(LiveExecutionManagerService, input.liveExecutionManager),
111 LocalWorkspaceStateStoreLive,114 );
112 LocalSourceArtifactStoreLive,115
113 LiveExecutionManagerLive,116 const secretMaterialLayer = SecretMaterialLive({
117 resolveSecretMaterial: input.resolveSecretMaterial,
118 }).pipe(Layer.provide(baseLayer));
119
120 const sourceStoreLayer = RuntimeSourceStoreLive.pipe(
121 Layer.provide(baseLayer),
122 );
123
124 const sourceRecipeStoreLayer = RuntimeSourceRecipeStoreLive.pipe(
125 Layer.provide(Layer.mergeAll(baseLayer, sourceStoreLayer)),
126 );
127
128 const sourceAuthMaterialLayer = RuntimeSourceAuthMaterialLive.pipe(
129 Layer.provide(Layer.mergeAll(baseLayer, secretMaterialLayer)),
130 );
131
132 const sourceMaterializationLayer = RuntimeSourceMaterializationLive.pipe(
133 Layer.provide(
134 Layer.mergeAll(baseLayer, secretMaterialLayer, sourceAuthMaterialLayer),
135 ),
114 );136 );
115 137
116 const sourceAuthLayer = RuntimeSourceAuthServiceLive({138 const sourceAuthLayer = RuntimeSourceAuthServiceLive({
117 getLocalServerBaseUrl: input.getLocalServerBaseUrl,139 getLocalServerBaseUrl: input.getLocalServerBaseUrl,
118 localConfig: input.localWorkspaceState.loadedConfig.config,140 }).pipe(
119 workspaceRoot: input.localWorkspaceState.context.workspaceRoot,141 Layer.provide(
120 localWorkspaceState: input.localWorkspaceState,142 Layer.mergeAll(
121 }).pipe(Layer.provide(baseLayer));143 baseLayer,
144 secretMaterialLayer,
145 sourceStoreLayer,
146 sourceMaterializationLayer,
147 ),
148 ),
149 );
122 150
123 const executionResolverLayer = RuntimeExecutionResolverLive({151 const executionResolverLayer = RuntimeExecutionResolverLive({
124 executionResolver: input.executionResolver,152 executionResolver: input.executionResolver,
125 resolveSecretMaterial: input.resolveSecretMaterial,
126 }).pipe(153 }).pipe(
127 Layer.provide(Layer.mergeAll(baseLayer, sourceAuthLayer)),154 Layer.provide(
155 Layer.mergeAll(
156 baseLayer,
157 secretMaterialLayer,
158 sourceAuthMaterialLayer,
159 sourceMaterializationLayer,
160 sourceAuthLayer,
161 sourceRecipeStoreLayer,
162 localToolRuntimeLayer,
163 ),
164 ),
128 );165 );
129 166
130 return Layer.mergeAll(167 return Layer.mergeAll(
131 baseLayer,168 baseLayer,
169 secretMaterialLayer,
170 sourceStoreLayer,
171 sourceAuthMaterialLayer,
172 sourceMaterializationLayer,
173 sourceRecipeStoreLayer,
174 localToolRuntimeLayer,
132 sourceAuthLayer,175 sourceAuthLayer,
133 executionResolverLayer,176 executionResolverLayer,
134 ) as RuntimeControlPlaneLayer;177 ) as RuntimeControlPlaneLayer;
@@ -137,12 +180,19 @@ export const createRuntimeControlPlaneLayer = ( · 2 unmodified lines
137export type ControlPlaneRuntime = {180export type ControlPlaneRuntime = {
138 persistence: LocalControlPlanePersistence;181 persistence: LocalControlPlanePersistence;
139 localInstallation: LocalInstallation;182 localInstallation: LocalInstallation;
183 managedRuntime: ManagedRuntime.ManagedRuntime<ControlPlaneApiRuntimeContext, never>;
140 runtimeLayer: RuntimeControlPlaneLayer;184 runtimeLayer: RuntimeControlPlaneLayer;
141 close: () => Promise<void>;185 close: () => Promise<void>;
142};186};
143 187
144export type CreateControlPlaneRuntimeOptions = RuntimeControlPlaneOptions;188export type CreateControlPlaneRuntimeOptions = RuntimeControlPlaneOptions;
145 189
190export const provideControlPlaneRuntime = <A, E, R>(
191 effect: Effect.Effect<A, E, R>,
192 runtime: ControlPlaneRuntime,
193): Effect.Effect<A, E | never, Exclude<R, ControlPlaneApiRuntimeContext>> =>
194 effect.pipe(Effect.provide(runtime.managedRuntime));
195
146export const createControlPlaneRuntime = (196export const createControlPlaneRuntime = (
147 options: CreateControlPlaneRuntimeOptions,197 options: CreateControlPlaneRuntimeOptions,
148): Effect.Effect<ControlPlaneRuntime, Error> =>198): Effect.Effect<ControlPlaneRuntime, Error> =>
@@ -160,8 +210,6 @@ export const createControlPlaneRuntime = ( · 11 unmodified lines
160 210
161 const installationStore = LocalInstallationStore;211 const installationStore = LocalInstallationStore;
162 const workspaceConfigStore = LocalWorkspaceConfigStore;212 const workspaceConfigStore = LocalWorkspaceConfigStore;
163 const workspaceStateStore = LocalWorkspaceStateStore;
164 const sourceArtifactStore = LocalSourceArtifactStore;
165 213
166 const localInstallation = yield* installationStore.getOrProvision({214 const localInstallation = yield* installationStore.getOrProvision({
167 context: localWorkspaceContext,215 context: localWorkspaceContext,
@@ -194,34 +242,6 @@ export const createControlPlaneRuntime = ( · 26 unmodified lines
194 ),242 ),
195 );243 );
196 244
197 const resolveSecretMaterial =
198 options.resolveSecretMaterial ??
199 createDefaultSecretMaterialResolver({
200 rows,
201 localConfig: effectiveLocalConfig,
202 workspaceRoot: localWorkspaceContext.workspaceRoot,
203 });
204
205 const liveExecutionManager = createLiveExecutionManager();
206 const sourceAuthService = createRuntimeSourceAuthService({
207 rows,
208 liveExecutionManager,
209 getLocalServerBaseUrl: options.getLocalServerBaseUrl,
210 localConfig: effectiveLocalConfig,
211 workspaceRoot: localWorkspaceContext.workspaceRoot,
212 localWorkspaceState: {
213 context: localWorkspaceContext,
214 installation: {
215 workspaceId: localInstallation.workspaceId,
216 accountId: localInstallation.accountId,
217 },
218 loadedConfig: {
219 ...loadedLocalConfig,
220 config: effectiveLocalConfig,
221 },
222 },
223 });
224
225 const runtimeLocalWorkspaceState: RuntimeLocalWorkspaceState = {245 const runtimeLocalWorkspaceState: RuntimeLocalWorkspaceState = {
226 context: localWorkspaceContext,246 context: localWorkspaceContext,
227 installation: {247 installation: {
@@ -233,30 +253,25 @@ export const createControlPlaneRuntime = ( · 5 unmodified lines
233 config: effectiveLocalConfig,253 config: effectiveLocalConfig,
234 },254 },
235 };255 };
236 256 const liveExecutionManager = createLiveExecutionManager();
237 const executionResolver =
238 options.executionResolver ??
239 createWorkspaceExecutionEnvironmentResolver({
240 rows,
241 sourceAuthService,
242 resolveSecretMaterial,
243 workspaceConfigStore,
244 workspaceStateStore,
245 sourceArtifactStore,
246 });
247 257
248 const concreteRuntimeLayer = createRuntimeControlPlaneLayer({258 const concreteRuntimeLayer = createRuntimeControlPlaneLayer({
249 ...options,259 ...options,
250 store: rows,260 store: rows,
251 localWorkspaceState: runtimeLocalWorkspaceState,261 localWorkspaceState: runtimeLocalWorkspaceState,
252 resolveSecretMaterial,262 liveExecutionManager,
253 executionResolver,
254 });263 });
264 const managedRuntime = ManagedRuntime.make(concreteRuntimeLayer);
265 const runtime = yield* managedRuntime.runtimeEffect;
266 const runtimeLayer = Layer.succeedContext(
267 runtime.context,
268 ) as RuntimeControlPlaneLayer;
255 269
256 return {270 return {
257 persistence,271 persistence,
258 localInstallation,272 localInstallation,
259 runtimeLayer: concreteRuntimeLayer,273 managedRuntime,
260 close: () => Effect.runPromise(Scope.close(scope, Exit.void)).catch(() => {}),274 runtimeLayer: runtimeLayer as RuntimeControlPlaneLayer,
275 close: () => managedRuntime.dispose().catch(() => {}),
261 };276 };
262 });277 }).pipe(Effect.provide(NodeFileSystem.layer));
packages/control-plane/src/runtime/live-execution.ts+5−0
@@ -15,6 +15,7 @@ import * as Layer from "effect/Layer"; · 14 unmodified lines
15import type { ControlPlaneStoreShape } from "./store";15import type { ControlPlaneStoreShape } from "./store";
16 16
17type VisibleExecutionState =17type VisibleExecutionState =
18 | "running"
18 | "waiting_for_interaction"19 | "waiting_for_interaction"
19 | "completed"20 | "completed"
20 | "failed";21 | "failed";
@@ -209,6 +210,10 @@ export const createLiveExecutionManager = () => { · 188 unmodified lines
209 status: "running",210 status: "running",
210 updatedAt: resolvedAt,211 updatedAt: resolvedAt,
211 });212 });
213 yield* publishState({
214 executionId,
215 state: "running",
216 });
212 217
213 run.currentInteraction = null;218 run.currentInteraction = null;
214 return resolved;219 return resolved;
packages/control-plane/src/runtime/local-config.test.ts+4−3
@@ -2,6 +2,7 @@ import { mkdtempSync } from "node:fs"; · 1 unmodified line
2import { mkdir, writeFile } from "node:fs/promises";2import { mkdir, writeFile } from "node:fs/promises";
3import { tmpdir } from "node:os";3import { tmpdir } from "node:os";
4import { join } from "node:path";4import { join } from "node:path";
5import { NodeFileSystem } from "@effect/platform-node";
5import { describe, expect, it } from "@effect/vitest";6import { describe, expect, it } from "@effect/vitest";
6import * as Effect from "effect/Effect";7import * as Effect from "effect/Effect";
7 8
@@ -50,7 +51,7 @@ describe("local-config", () => { · 42 unmodified lines
50 expect(loaded.config?.sources?.github?.connection.endpoint).toBe(51 expect(loaded.config?.sources?.github?.connection.endpoint).toBe(
51 "https://api.github.com",52 "https://api.github.com",
52 );53 );
53 }),54 }).pipe(Effect.provide(NodeFileSystem.layer)),
54 );55 );
55 56
56 it.effect("reports jsonc syntax errors with line and column details", () =>57 it.effect("reports jsonc syntax errors with line and column details", () =>
@@ -81,7 +82,7 @@ describe("local-config", () => { · 24 unmodified lines
81 82
82 expect(failure.message).toContain("Invalid executor config");83 expect(failure.message).toContain("Invalid executor config");
83 expect(failure.message).toContain("line 5, column 7");84 expect(failure.message).toContain("line 5, column 7");
84 }),85 }).pipe(Effect.provide(NodeFileSystem.layer)),
85 );86 );
86 87
87 it("uses platform-standard home config candidates", () => {88 it("uses platform-standard home config candidates", () => {
@@ -123,6 +124,6 @@ describe("local-config", () => { · 35 unmodified lines
123 });124 });
124 125
125 expect(resolvedPath).toBe(join(legacyConfigDirectory, "executor.jsonc"));126 expect(resolvedPath).toBe(join(legacyConfigDirectory, "executor.jsonc"));
126 }),127 }).pipe(Effect.provide(NodeFileSystem.layer)),
127 );128 );
128});129});
packages/control-plane/src/runtime/local-config.ts+22−23
@@ -1,7 +1,6 @@
1import { homedir } from "node:os";1import { homedir } from "node:os";
2import { basename, dirname, isAbsolute, join, resolve } from "node:path";2import { basename, dirname, isAbsolute, join, resolve } from "node:path";
3import { FileSystem } from "@effect/platform";3import { FileSystem } from "@effect/platform";
4import { NodeFileSystem } from "@effect/platform-node";
5import {4import {
6 type ParseError as JsoncParseError,5 type ParseError as JsoncParseError,
7 parse as parseJsoncDocument,6 parse as parseJsoncDocument,
@@ -30,15 +29,6 @@ const PROJECT_CONFIG_FALLBACK_BASENAME = "executor.json"; · 22 unmodified lines
30const PROJECT_CONFIG_DIRECTORY = ".executor";29const PROJECT_CONFIG_DIRECTORY = ".executor";
31const EXECUTOR_CONFIG_DIR_ENV = "EXECUTOR_CONFIG_DIR";30const EXECUTOR_CONFIG_DIR_ENV = "EXECUTOR_CONFIG_DIR";
32 31
33const provideNodeFileSystem = <A, E, R>(
34 effect: Effect.Effect<A, E, R | FileSystem.FileSystem>,
35): Effect.Effect<A, E, Exclude<R, FileSystem.FileSystem>> =>
36 effect.pipe(Effect.provide(NodeFileSystem.layer)) as Effect.Effect<
37 A,
38 E,
39 Exclude<R, FileSystem.FileSystem>
40 >;
41
42const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>32const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>
43 new LocalFileSystemError({33 new LocalFileSystemError({
44 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,34 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,
@@ -114,8 +104,8 @@ export const resolveHomeConfigPath = (input: { · 69 unmodified lines
114 env?: NodeJS.ProcessEnv;104 env?: NodeJS.ProcessEnv;
115 platform?: NodeJS.Platform;105 platform?: NodeJS.Platform;
116 homeDirectory?: string;106 homeDirectory?: string;
117} = {}): Effect.Effect<string, LocalFileSystemError> =>107} = {}): Effect.Effect<string, LocalFileSystemError, FileSystem.FileSystem> =>
118 provideNodeFileSystem(Effect.gen(function* () {108 Effect.gen(function* () {
119 const fs = yield* FileSystem.FileSystem;109 const fs = yield* FileSystem.FileSystem;
120 const candidates = resolveDefaultHomeConfigCandidates(input);110 const candidates = resolveDefaultHomeConfigCandidates(input);
121 111
@@ -129,7 +119,7 @@ export const resolveHomeConfigPath = (input: { · 7 unmodified lines
129 }119 }
130 120
131 return candidates[0]!;121 return candidates[0]!;
132 }));122 });
133 123
134const formatJsoncParseErrors = (content: string, errors: readonly JsoncParseError[]): string => {124const formatJsoncParseErrors = (content: string, errors: readonly JsoncParseError[]): string => {
135 const lines = content.split("\n");125 const lines = content.split("\n");
@@ -338,8 +328,12 @@ export type LoadedLocalExecutorConfig = { · 202 unmodified lines
338export const resolveLocalWorkspaceContext = (input: {328export const resolveLocalWorkspaceContext = (input: {
339 cwd?: string;329 cwd?: string;
340 workspaceRoot?: string;330 workspaceRoot?: string;
341} = {}): Effect.Effect<ResolvedLocalWorkspaceContext, LocalFileSystemError> =>331} = {}): Effect.Effect<
342 provideNodeFileSystem(Effect.gen(function* () {332 ResolvedLocalWorkspaceContext,
333 LocalFileSystemError,
334 FileSystem.FileSystem
335> =>
336 Effect.gen(function* () {
343 const cwd = resolve(input.cwd ?? process.cwd());337 const cwd = resolve(input.cwd ?? process.cwd());
344 const workspaceRoot = resolve(338 const workspaceRoot = resolve(
345 input.workspaceRoot ?? (yield* resolveWorkspaceRootFromCwdEffect(cwd)),339 input.workspaceRoot ?? (yield* resolveWorkspaceRootFromCwdEffect(cwd)),
@@ -358,12 +352,16 @@ export const resolveLocalWorkspaceContext = (input: { · 12 unmodified lines
358 artifactsDirectory: join(workspaceRoot, PROJECT_CONFIG_DIRECTORY, "artifacts"),352 artifactsDirectory: join(workspaceRoot, PROJECT_CONFIG_DIRECTORY, "artifacts"),
359 stateDirectory: join(workspaceRoot, PROJECT_CONFIG_DIRECTORY, "state"),353 stateDirectory: join(workspaceRoot, PROJECT_CONFIG_DIRECTORY, "state"),
360 };354 };
361 }));355 });
362 356
363export const readOptionalLocalExecutorConfig = (357export const readOptionalLocalExecutorConfig = (
364 path: string,358 path: string,
365): Effect.Effect<LocalExecutorConfig | null, LocalFileSystemError | LocalExecutorConfigDecodeError> =>359): Effect.Effect<
366 provideNodeFileSystem(Effect.gen(function* () {360 LocalExecutorConfig | null,
361 LocalFileSystemError | LocalExecutorConfigDecodeError,
362 FileSystem.FileSystem
363> =>
364 Effect.gen(function* () {
367 const fs = yield* FileSystem.FileSystem;365 const fs = yield* FileSystem.FileSystem;
368 const exists = yield* fs.exists(path).pipe(366 const exists = yield* fs.exists(path).pipe(
369 Effect.mapError(mapFileSystemError(path, "check config path")),367 Effect.mapError(mapFileSystemError(path, "check config path")),
@@ -386,13 +384,14 @@ export const readOptionalLocalExecutorConfig = ( · 16 unmodified lines
386 details: unknownLocalErrorDetails(cause),384 details: unknownLocalErrorDetails(cause),
387 }),385 }),
388 });386 });
389 }));387 });
390 388
391export const loadLocalExecutorConfig = (389export const loadLocalExecutorConfig = (
392 context: ResolvedLocalWorkspaceContext,390 context: ResolvedLocalWorkspaceContext,
393): Effect.Effect<391): Effect.Effect<
394 LoadedLocalExecutorConfig,392 LoadedLocalExecutorConfig,
395 LocalFileSystemError | LocalExecutorConfigDecodeError393 LocalFileSystemError | LocalExecutorConfigDecodeError,
394 FileSystem.FileSystem
396> =>395> =>
397 Effect.gen(function* () {396 Effect.gen(function* () {
398 const [homeConfig, projectConfig] = yield* Effect.all([397 const [homeConfig, projectConfig] = yield* Effect.all([
@@ -415,8 +414,8 @@ export const encodeLocalExecutorConfig = (config: LocalExecutorConfig): string = · 16 unmodified lines
415export const writeProjectLocalExecutorConfig = (input: {414export const writeProjectLocalExecutorConfig = (input: {
416 context: ResolvedLocalWorkspaceContext;415 context: ResolvedLocalWorkspaceContext;
417 config: LocalExecutorConfig;416 config: LocalExecutorConfig;
418}): Effect.Effect<void, LocalFileSystemError> =>417}): Effect.Effect<void, LocalFileSystemError, FileSystem.FileSystem> =>
419 provideNodeFileSystem(Effect.gen(function* () {418 Effect.gen(function* () {
420 const fs = yield* FileSystem.FileSystem;419 const fs = yield* FileSystem.FileSystem;
421 yield* fs.makeDirectory(input.context.configDirectory, { recursive: true }).pipe(420 yield* fs.makeDirectory(input.context.configDirectory, { recursive: true }).pipe(
422 Effect.mapError(mapFileSystemError(input.context.configDirectory, "create config directory")),421 Effect.mapError(mapFileSystemError(input.context.configDirectory, "create config directory")),
@@ -427,7 +426,7 @@ export const writeProjectLocalExecutorConfig = (input: { · 4 unmodified lines
427 ).pipe(426 ).pipe(
428 Effect.mapError(mapFileSystemError(input.context.projectConfigPath, "write config")),427 Effect.mapError(mapFileSystemError(input.context.projectConfigPath, "write config")),
429 );428 );
430 }));429 });
431 430
432export const resolveConfigRelativePath = (input: {431export const resolveConfigRelativePath = (input: {
433 path: string;432 path: string;
packages/control-plane/src/runtime/local-control-plane-store.ts+4−42
@@ -25,9 +25,7 @@ import * as Effect from "effect/Effect"; · 24 unmodified lines
25import * as Option from "effect/Option";25import * as Option from "effect/Option";
26import * as Schema from "effect/Schema";26import * as Schema from "effect/Schema";
27 27
28import { authArtifactSecretMaterialRefs } from "./auth-artifacts";
29import type { ResolvedLocalWorkspaceContext } from "./local-config";28import type { ResolvedLocalWorkspaceContext } from "./local-config";
30import { loadLocalExecutorConfig } from "./local-config";
31import {29import {
32 LocalFileSystemError,30 LocalFileSystemError,
33 unknownLocalErrorDetails,31 unknownLocalErrorDetails,
@@ -303,7 +301,10 @@ const createStateManager = (context: ResolvedLocalWorkspaceContext) => { · 269 unmodified lines
303 operation: (state: LocalControlPlaneState) => A | Promise<A>,301 operation: (state: LocalControlPlaneState) => A | Promise<A>,
304 ): Effect.Effect<A, LocalFileSystemError> =>302 ): Effect.Effect<A, LocalFileSystemError> =>
305 Effect.tryPromise({303 Effect.tryPromise({
306 try: async () => operation(cloneValue(await ensureLoaded())),304 try: async () => {
305 await mutationQueue;
306 return operation(cloneValue(await ensureLoaded()));
307 },
307 catch: mapFileSystemError(308 catch: mapFileSystemError(
308 localControlPlaneStatePath(context),309 localControlPlaneStatePath(context),
309 "read control plane state",310 "read control plane state",
@@ -772,45 +773,6 @@ export const createLocalControlPlaneStore = ( · 462 unmodified lines
772 value: nextMaterials.length !== state.secretMaterials.length,773 value: nextMaterials.length !== state.secretMaterials.length,
773 } satisfies StateMutationResult<boolean>;774 } satisfies StateMutationResult<boolean>;
774 }),775 }),
775
776 listLinkedSources: () =>
777 Effect.gen(function* () {
778 const [state, loadedConfig] = yield* Effect.all([
779 stateManager.read((current) => current),
780 loadLocalExecutorConfig(context),
781 ]);
782
783 const sourceNames = new Map<string, string>();
784 for (const [sourceId, source] of Object.entries(
785 loadedConfig.config?.sources ?? {},
786 )) {
787 sourceNames.set(sourceId, source.name ?? sourceId);
788 }
789
790 const materialIds = new Set(
791 state.secretMaterials.map((material) => String(material.id)),
792 );
793 const linkedSources = new Map<string, Array<{ sourceId: string; sourceName: string }>>();
794
795 for (const artifact of state.authArtifacts) {
796 for (const ref of authArtifactSecretMaterialRefs(artifact)) {
797 if (!materialIds.has(ref.handle)) {
798 continue;
799 }
800
801 const existing = linkedSources.get(ref.handle) ?? [];
802 if (!existing.some((link) => link.sourceId === artifact.sourceId)) {
803 existing.push({
804 sourceId: artifact.sourceId,
805 sourceName: sourceNames.get(artifact.sourceId) ?? artifact.sourceId,
806 });
807 linkedSources.set(ref.handle, existing);
808 }
809 }
810 }
811
812 return linkedSources;
813 }),
814 },776 },
815 777
816 executions: {778 executions: {
packages/control-plane/src/runtime/local-installation.test.ts+2−1
@@ -1,6 +1,7 @@
1import { existsSync, mkdtempSync } from "node:fs";1import { existsSync, mkdtempSync } from "node:fs";
2import { tmpdir } from "node:os";2import { tmpdir } from "node:os";
3import { join } from "node:path";3import { join } from "node:path";
4import { NodeFileSystem } from "@effect/platform-node";
4import { describe, expect, it } from "@effect/vitest";5import { describe, expect, it } from "@effect/vitest";
5import * as Effect from "effect/Effect";6import * as Effect from "effect/Effect";
6 7
@@ -45,6 +46,6 @@ describe("local-installation", () => { · 38 unmodified lines
45 46
46 expect(second.accountId).toBe(first.accountId);47 expect(second.accountId).toBe(first.accountId);
47 expect(second.workspaceId).toBe(first.workspaceId);48 expect(second.workspaceId).toBe(first.workspaceId);
48 }),49 }).pipe(Effect.provide(NodeFileSystem.layer)),
49 );50 );
50});51});
packages/control-plane/src/runtime/local-runtime-context.ts+13−0
@@ -1,5 +1,6 @@
1import * as Context from "effect/Context";1import * as Context from "effect/Context";
2import * as Effect from "effect/Effect";2import * as Effect from "effect/Effect";
3import * as Layer from "effect/Layer";
3import * as Option from "effect/Option";4import * as Option from "effect/Option";
4 5
5import type { AccountId, WorkspaceId } from "#schema";6import type { AccountId, WorkspaceId } from "#schema";
@@ -25,6 +26,18 @@ export class RuntimeLocalWorkspaceService extends Context.Tag( · 19 unmodified lines
25 "#runtime/RuntimeLocalWorkspaceService",26 "#runtime/RuntimeLocalWorkspaceService",
26)<RuntimeLocalWorkspaceService, RuntimeLocalWorkspaceState>() {}27)<RuntimeLocalWorkspaceService, RuntimeLocalWorkspaceState>() {}
27 28
29export const RuntimeLocalWorkspaceLive = (
30 runtimeLocalWorkspace: RuntimeLocalWorkspaceState,
31) => Layer.succeed(RuntimeLocalWorkspaceService, runtimeLocalWorkspace);
32
33export const provideOptionalRuntimeLocalWorkspace = <A, E, R>(
34 effect: Effect.Effect<A, E, R>,
35 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null | undefined,
36): Effect.Effect<A, E, R> =>
37 runtimeLocalWorkspace === null || runtimeLocalWorkspace === undefined
38 ? effect
39 : effect.pipe(Effect.provide(RuntimeLocalWorkspaceLive(runtimeLocalWorkspace)));
40
28export const getRuntimeLocalWorkspaceOption = () =>41export const getRuntimeLocalWorkspaceOption = () =>
29 Effect.contextWith((context) =>42 Effect.contextWith((context) =>
30 Context.getOption(context, RuntimeLocalWorkspaceService),43 Context.getOption(context, RuntimeLocalWorkspaceService),
packages/control-plane/src/runtime/local-source-artifacts.ts+10−19
@@ -1,6 +1,5 @@
1import { join } from "node:path";1import { join } from "node:path";
2import { FileSystem } from "@effect/platform";2import { FileSystem } from "@effect/platform";
3import { NodeFileSystem } from "@effect/platform-node";
4 3
5import {4import {
6 SourceIdSchema,5 SourceIdSchema,
@@ -48,15 +47,6 @@ export type LocalSourceArtifact = typeof LocalSourceArtifactSchema.Type; · 41 unmodified lines
48 47
49const decodeLocalSourceArtifact = Schema.decodeUnknownSync(LocalSourceArtifactSchema);48const decodeLocalSourceArtifact = Schema.decodeUnknownSync(LocalSourceArtifactSchema);
50 49
51const provideNodeFileSystem = <A, E, R>(
52 effect: Effect.Effect<A, E, R | FileSystem.FileSystem>,
53): Effect.Effect<A, E, Exclude<R, FileSystem.FileSystem>> =>
54 effect.pipe(Effect.provide(NodeFileSystem.layer)) as Effect.Effect<
55 A,
56 E,
57 Exclude<R, FileSystem.FileSystem>
58 >;
59
60const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>50const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>
61 new LocalFileSystemError({51 new LocalFileSystemError({
62 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,52 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,
@@ -167,9 +157,10 @@ export const readLocalSourceArtifact = (input: { · 104 unmodified lines
167 sourceId: string;157 sourceId: string;
168}): Effect.Effect<158}): Effect.Effect<
169 LocalSourceArtifact | null,159 LocalSourceArtifact | null,
170 LocalFileSystemError | LocalSourceArtifactDecodeError160 LocalFileSystemError | LocalSourceArtifactDecodeError,
161 FileSystem.FileSystem
171> =>162> =>
172 provideNodeFileSystem(Effect.gen(function* () {163 Effect.gen(function* () {
173 const fs = yield* FileSystem.FileSystem;164 const fs = yield* FileSystem.FileSystem;
174 const path = localSourceArtifactPath(input);165 const path = localSourceArtifactPath(input);
175 const exists = yield* fs.exists(path).pipe(166 const exists = yield* fs.exists(path).pipe(
@@ -192,14 +183,14 @@ export const readLocalSourceArtifact = (input: { · 16 unmodified lines
192 });183 });
193 },184 },
194 });185 });
195 }));186 });
196 187
197export const writeLocalSourceArtifact = (input: {188export const writeLocalSourceArtifact = (input: {
198 context: ResolvedLocalWorkspaceContext;189 context: ResolvedLocalWorkspaceContext;
199 sourceId: string;190 sourceId: string;
200 artifact: LocalSourceArtifact;191 artifact: LocalSourceArtifact;
201}): Effect.Effect<void, LocalFileSystemError> =>192}): Effect.Effect<void, LocalFileSystemError, FileSystem.FileSystem> =>
202 provideNodeFileSystem(Effect.gen(function* () {193 Effect.gen(function* () {
203 const fs = yield* FileSystem.FileSystem;194 const fs = yield* FileSystem.FileSystem;
204 const directory = join(input.context.artifactsDirectory, "sources");195 const directory = join(input.context.artifactsDirectory, "sources");
205 const path = localSourceArtifactPath(input);196 const path = localSourceArtifactPath(input);
@@ -209,13 +200,13 @@ export const writeLocalSourceArtifact = (input: { · 3 unmodified lines
209 yield* fs.writeFileString(path, `${JSON.stringify(input.artifact, null, 2)}\n`).pipe(200 yield* fs.writeFileString(path, `${JSON.stringify(input.artifact, null, 2)}\n`).pipe(
210 Effect.mapError(mapFileSystemError(path, "write source artifact")),201 Effect.mapError(mapFileSystemError(path, "write source artifact")),
211 );202 );
212 }));203 });
213 204
214export const removeLocalSourceArtifact = (input: {205export const removeLocalSourceArtifact = (input: {
215 context: ResolvedLocalWorkspaceContext;206 context: ResolvedLocalWorkspaceContext;
216 sourceId: string;207 sourceId: string;
217}): Effect.Effect<void, LocalFileSystemError> =>208}): Effect.Effect<void, LocalFileSystemError, FileSystem.FileSystem> =>
218 provideNodeFileSystem(Effect.gen(function* () {209 Effect.gen(function* () {
219 const fs = yield* FileSystem.FileSystem;210 const fs = yield* FileSystem.FileSystem;
220 const path = localSourceArtifactPath(input);211 const path = localSourceArtifactPath(input);
221 const exists = yield* fs.exists(path).pipe(212 const exists = yield* fs.exists(path).pipe(
@@ -227,4 +218,4 @@ export const removeLocalSourceArtifact = (input: { · 5 unmodified lines
227 yield* fs.remove(path).pipe(218 yield* fs.remove(path).pipe(
228 Effect.mapError(mapFileSystemError(path, "remove source artifact")),219 Effect.mapError(mapFileSystemError(path, "remove source artifact")),
229 );220 );
230 }));221 });
packages/control-plane/src/runtime/local-storage.ts+85−14
@@ -1,6 +1,8 @@
1import { FileSystem } from "@effect/platform";
2import { NodeFileSystem } from "@effect/platform-node";
1import type { LocalInstallation, LocalExecutorConfig } from "#schema";3import type { LocalInstallation, LocalExecutorConfig } from "#schema";
2import * as Context from "effect/Context";4import * as Context from "effect/Context";
3import type * as Effect from "effect/Effect";5import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";6import * as Layer from "effect/Layer";
5 7
6import type {8import type {
@@ -120,35 +122,104 @@ export const LocalInstallationStoreLive = Layer.succeed( · 113 unmodified lines
120 LocalInstallationStore,122 LocalInstallationStore,
121);123);
122 124
125const bindFileSystem = <A, E>(
126 fileSystem: FileSystem.FileSystem,
127 effect: Effect.Effect<A, E, FileSystem.FileSystem>,
128): Effect.Effect<A, E, never> =>
129 effect.pipe(Effect.provideService(FileSystem.FileSystem, fileSystem));
130
131const bindNodeFileSystem = <A, E>(
132 effect: Effect.Effect<A, E, FileSystem.FileSystem>,
133): Effect.Effect<A, E, never> =>
134 effect.pipe(Effect.provide(NodeFileSystem.layer));
135
123export const LocalWorkspaceConfigStore: WorkspaceConfigStoreShape = {136export const LocalWorkspaceConfigStore: WorkspaceConfigStoreShape = {
124 load: loadLocalExecutorConfig,137 load: (context) => bindNodeFileSystem(loadLocalExecutorConfig(context)),
125 writeProject: writeProjectLocalExecutorConfig,138 writeProject: (input) => bindNodeFileSystem(writeProjectLocalExecutorConfig(input)),
126 resolveRelativePath: resolveConfigRelativePath,139 resolveRelativePath: resolveConfigRelativePath,
127};140};
128 141
129export const LocalWorkspaceConfigStoreLive = Layer.succeed(142export const LocalWorkspaceConfigStoreLive = Layer.effect(
130 WorkspaceConfigStore,143 WorkspaceConfigStore,
131 LocalWorkspaceConfigStore,144 Effect.gen(function* () {
145 const fileSystem = yield* FileSystem.FileSystem;
146
147 return WorkspaceConfigStore.of({
148 load: (context) => bindFileSystem(fileSystem, loadLocalExecutorConfig(context)),
149 writeProject: (input) =>
150 bindFileSystem(fileSystem, writeProjectLocalExecutorConfig(input)),
151 resolveRelativePath: resolveConfigRelativePath,
152 });
153 }),
132);154);
133 155
134export const LocalWorkspaceStateStore: WorkspaceStateStoreShape = {156export const LocalWorkspaceStateStore: WorkspaceStateStoreShape = {
135 load: loadLocalWorkspaceState,157 load: (context) => bindNodeFileSystem(loadLocalWorkspaceState(context)),
136 write: writeLocalWorkspaceState,158 write: (input) => bindNodeFileSystem(writeLocalWorkspaceState(input)),
137};159};
138 160
139export const LocalWorkspaceStateStoreLive = Layer.succeed(161export const LocalWorkspaceStateStoreLive = Layer.effect(
140 WorkspaceStateStore,162 WorkspaceStateStore,
141 LocalWorkspaceStateStore,163 Effect.gen(function* () {
164 const fileSystem = yield* FileSystem.FileSystem;
165
166 return WorkspaceStateStore.of({
167 load: (context) => bindFileSystem(fileSystem, loadLocalWorkspaceState(context)),
168 write: (input) => bindFileSystem(fileSystem, writeLocalWorkspaceState(input)),
169 });
170 }),
142);171);
143 172
144export const LocalSourceArtifactStore: SourceArtifactStoreShape = {173export const LocalSourceArtifactStore: SourceArtifactStoreShape = {
145 build: buildLocalSourceArtifact,174 build: buildLocalSourceArtifact,
146 read: readLocalSourceArtifact,175 read: (input) => bindNodeFileSystem(readLocalSourceArtifact(input)),
147 write: writeLocalSourceArtifact,176 write: (input) => bindNodeFileSystem(writeLocalSourceArtifact(input)),
148 remove: removeLocalSourceArtifact,177 remove: (input) => bindNodeFileSystem(removeLocalSourceArtifact(input)),
149};178};
150 179
151export const LocalSourceArtifactStoreLive = Layer.succeed(180export const LocalSourceArtifactStoreLive = Layer.effect(
152 SourceArtifactStore,181 SourceArtifactStore,
153 LocalSourceArtifactStore,182 Effect.gen(function* () {
183 const fileSystem = yield* FileSystem.FileSystem;
184
185 return SourceArtifactStore.of({
186 build: buildLocalSourceArtifact,
187 read: (input) => bindFileSystem(fileSystem, readLocalSourceArtifact(input)),
188 write: (input) => bindFileSystem(fileSystem, writeLocalSourceArtifact(input)),
189 remove: (input) => bindFileSystem(fileSystem, removeLocalSourceArtifact(input)),
190 });
191 }),
192);
193
194export const makeWorkspaceStorageLayer = (input: {
195 workspaceConfigStore: WorkspaceConfigStoreShape;
196 workspaceStateStore: WorkspaceStateStoreShape;
197 sourceArtifactStore: SourceArtifactStoreShape;
198}) =>
199 Layer.mergeAll(
200 Layer.succeed(WorkspaceConfigStore, input.workspaceConfigStore),
201 Layer.succeed(WorkspaceStateStore, input.workspaceStateStore),
202 Layer.succeed(SourceArtifactStore, input.sourceArtifactStore),
203 );
204
205export const makeLocalStorageLayer = (input: {
206 installationStore: InstallationStoreShape;
207 workspaceConfigStore: WorkspaceConfigStoreShape;
208 workspaceStateStore: WorkspaceStateStoreShape;
209 sourceArtifactStore: SourceArtifactStoreShape;
210}) =>
211 Layer.mergeAll(
212 Layer.succeed(InstallationStore, input.installationStore),
213 makeWorkspaceStorageLayer(input),
214 );
215
216export const WorkspaceStorageLive = Layer.mergeAll(
217 LocalWorkspaceConfigStoreLive,
218 LocalWorkspaceStateStoreLive,
219 LocalSourceArtifactStoreLive,
220);
221
222export const LocalStorageLive = Layer.mergeAll(
223 LocalInstallationStoreLive,
224 WorkspaceStorageLive,
154);225);
packages/control-plane/src/runtime/local-tools.ts+88−65
@@ -3,7 +3,6 @@ import { existsSync } from "node:fs"; · 2 unmodified lines
3import { dirname, extname, join, relative, resolve, sep } from "node:path";3import { dirname, extname, join, relative, resolve, sep } from "node:path";
4import { fileURLToPath, pathToFileURL } from "node:url";4import { fileURLToPath, pathToFileURL } from "node:url";
5import { FileSystem } from "@effect/platform";5import { FileSystem } from "@effect/platform";
6import { NodeFileSystem } from "@effect/platform-node";
7import {6import {
8 createToolCatalogFromTools,7 createToolCatalogFromTools,
9 makeToolInvokerFromTools,8 makeToolInvokerFromTools,
@@ -17,7 +16,9 @@ import { · 7 unmodified lines
17 type ToolMetadata,16 type ToolMetadata,
18 type ToolPath,17 type ToolPath,
19} from "@executor/codemode-core";18} from "@executor/codemode-core";
19import * as Context from "effect/Context";
20import * as Effect from "effect/Effect";20import * as Effect from "effect/Effect";
21import * as Layer from "effect/Layer";
21import * as ts from "typescript";22import * as ts from "typescript";
22 23
23import type { ResolvedLocalWorkspaceContext } from "./local-config";24import type { ResolvedLocalWorkspaceContext } from "./local-config";
@@ -96,15 +97,6 @@ const emptyLocalToolRuntime = (): LocalToolRuntime => { · 72 unmodified lines
96 };97 };
97};98};
98 99
99const provideNodeFileSystem = <A, E, R>(
100 effect: Effect.Effect<A, E, R | FileSystem.FileSystem>,
101): Effect.Effect<A, E, Exclude<R, FileSystem.FileSystem>> =>
102 effect.pipe(Effect.provide(NodeFileSystem.layer)) as Effect.Effect<
103 A,
104 E,
105 Exclude<R, FileSystem.FileSystem>
106 >;
107
108const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>100const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>
109 new LocalFileSystemError({101 new LocalFileSystemError({
110 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,102 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,
@@ -589,66 +581,97 @@ export const loadLocalToolRuntime = ( · 478 unmodified lines
589 | LocalToolDefinitionError581 | LocalToolDefinitionError
590 | LocalToolImportError582 | LocalToolImportError
591 | LocalToolPathConflictError583 | LocalToolPathConflictError
592 | LocalToolTranspileError584 | LocalToolTranspileError,
585 FileSystem.FileSystem
593> =>586> =>
594 provideNodeFileSystem(587 Effect.gen(function* () {
595 Effect.gen(function* () {588 const sourceDirectory = localToolsDirectory(context);
596 const sourceDirectory = localToolsDirectory(context);589 const sourceFiles = yield* collectLocalToolSourceFiles(sourceDirectory);
597 const sourceFiles = yield* collectLocalToolSourceFiles(sourceDirectory);
598 590
599 if (sourceFiles.length === 0) {591 if (sourceFiles.length === 0) {
600 return emptyLocalToolRuntime();592 return emptyLocalToolRuntime();
593 }
594
595 const artifactPaths = yield* buildLocalToolArtifacts({
596 context,
597 sourceDirectory,
598 sourceFiles,
599 });
600 const candidateFiles = toolCandidateFiles(sourceDirectory, sourceFiles);
601 const tools: ToolMap = {};
602 const seenToolPaths = new Map<string, string>();
603
604 for (const sourcePath of candidateFiles) {
605 const relativePath = relative(sourceDirectory, sourcePath);
606 const toolPath = yield* toToolPath(relativePath);
607 const existing = seenToolPaths.get(toolPath);
608 if (existing) {
609 return yield* Effect.fail(
610 new LocalToolPathConflictError({
611 message: `Local tool path conflict for ${toolPath}`,
612 path: sourcePath,
613 otherPath: existing,
614 toolPath,
615 }),
616 );
617 }
618
619 const artifactPath = artifactPaths.get(sourcePath);
620 if (!artifactPath) {
621 return yield* Effect.fail(
622 new LocalToolImportError({
623 message: `Missing compiled artifact for local tool ${sourcePath}`,
624 path: sourcePath,
625 details:
626 "Expected a compiled local tool artifact, but none was produced.",
627 }),
628 );
601 }629 }
602 630
603 const artifactPaths = yield* buildLocalToolArtifacts({631 tools[toolPath] = yield* importLocalToolModule({
604 context,632 sourcePath,
605 sourceDirectory,633 artifactPath,
606 sourceFiles,634 toolPath,
607 });635 });
608 const candidateFiles = toolCandidateFiles(sourceDirectory, sourceFiles);636 seenToolPaths.set(toolPath, sourcePath);
609 const tools: ToolMap = {};637 }
610 const seenToolPaths = new Map<string, string>();
611
612 for (const sourcePath of candidateFiles) {
613 const relativePath = relative(sourceDirectory, sourcePath);
614 const toolPath = yield* toToolPath(relativePath);
615 const existing = seenToolPaths.get(toolPath);
616 if (existing) {
617 return yield* Effect.fail(
618 new LocalToolPathConflictError({
619 message: `Local tool path conflict for ${toolPath}`,
620 path: sourcePath,
621 otherPath: existing,
622 toolPath,
623 }),
624 );
625 }
626 638
627 const artifactPath = artifactPaths.get(sourcePath);639 return {
628 if (!artifactPath) {640 tools,
629 return yield* Effect.fail(641 catalog: createToolCatalogFromTools({ tools }),
630 new LocalToolImportError({642 toolInvoker: makeToolInvokerFromTools({ tools }),
631 message: `Missing compiled artifact for local tool ${sourcePath}`,643 toolPaths: new Set(Object.keys(tools)),
632 path: sourcePath,644 } satisfies LocalToolRuntime;
633 details:645 });
634 "Expected a compiled local tool artifact, but none was produced.",
635 }),
636 );
637 }
638 646
639 tools[toolPath] = yield* importLocalToolModule({647export type LocalToolRuntimeLoaderShape = {
640 sourcePath,648 load: (
641 artifactPath,649 context: ResolvedLocalWorkspaceContext,
642 toolPath,650 ) => Effect.Effect<
643 });651 LocalToolRuntime,
644 seenToolPaths.set(toolPath, sourcePath);652 | LocalFileSystemError
645 }653 | LocalToolDefinitionError
654 | LocalToolImportError
655 | LocalToolPathConflictError
656 | LocalToolTranspileError,
657 never
658 >;
659};
646 660
647 return {661export class LocalToolRuntimeLoaderService extends Context.Tag(
648 tools,662 "#runtime/LocalToolRuntimeLoaderService",
649 catalog: createToolCatalogFromTools({ tools }),663)<LocalToolRuntimeLoaderService, LocalToolRuntimeLoaderShape>() {}
650 toolInvoker: makeToolInvokerFromTools({ tools }),664
651 toolPaths: new Set(Object.keys(tools)),665export const LocalToolRuntimeLoaderLive = Layer.effect(
652 } satisfies LocalToolRuntime;666 LocalToolRuntimeLoaderService,
653 }),667 Effect.gen(function* () {
654 );668 const fileSystem = yield* FileSystem.FileSystem;
669
670 return LocalToolRuntimeLoaderService.of({
671 load: (context) =>
672 loadLocalToolRuntime(context).pipe(
673 Effect.provideService(FileSystem.FileSystem, fileSystem),
674 ),
675 });
676 }),
677);
packages/control-plane/src/runtime/local-workspace-state.ts+10−16
@@ -1,6 +1,5 @@
1import { join } from "node:path";1import { join } from "node:path";
2import { FileSystem } from "@effect/platform";2import { FileSystem } from "@effect/platform";
3import { NodeFileSystem } from "@effect/platform-node";
4 3
5import {4import {
6 PolicyIdSchema,5 PolicyIdSchema,
@@ -51,15 +50,6 @@ export type LocalWorkspaceState = typeof LocalWorkspaceStateSchema.Type; · 44 unmodified lines
51 50
52const decodeLocalWorkspaceState = Schema.decodeUnknownSync(LocalWorkspaceStateSchema);51const decodeLocalWorkspaceState = Schema.decodeUnknownSync(LocalWorkspaceStateSchema);
53 52
54const provideNodeFileSystem = <A, E, R>(
55 effect: Effect.Effect<A, E, R | FileSystem.FileSystem>,
56): Effect.Effect<A, E, Exclude<R, FileSystem.FileSystem>> =>
57 effect.pipe(Effect.provide(NodeFileSystem.layer)) as Effect.Effect<
58 A,
59 E,
60 Exclude<R, FileSystem.FileSystem>
61 >;
62
63const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>53const mapFileSystemError = (path: string, action: string) => (cause: unknown) =>
64 new LocalFileSystemError({54 new LocalFileSystemError({
65 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,55 message: `Failed to ${action} ${path}: ${unknownLocalErrorDetails(cause)}`,
@@ -80,8 +70,12 @@ export const localWorkspaceStatePath = ( · 14 unmodified lines
80 70
81export const loadLocalWorkspaceState = (71export const loadLocalWorkspaceState = (
82 context: ResolvedLocalWorkspaceContext,72 context: ResolvedLocalWorkspaceContext,
83): Effect.Effect<LocalWorkspaceState, LocalFileSystemError | LocalWorkspaceStateDecodeError> =>73): Effect.Effect<
84 provideNodeFileSystem(Effect.gen(function* () {74 LocalWorkspaceState,
75 LocalFileSystemError | LocalWorkspaceStateDecodeError,
76 FileSystem.FileSystem
77> =>
78 Effect.gen(function* () {
85 const fs = yield* FileSystem.FileSystem;79 const fs = yield* FileSystem.FileSystem;
86 const path = localWorkspaceStatePath(context);80 const path = localWorkspaceStatePath(context);
87 const exists = yield* fs.exists(path).pipe(81 const exists = yield* fs.exists(path).pipe(
@@ -104,13 +98,13 @@ export const loadLocalWorkspaceState = ( · 16 unmodified lines
104 });98 });
105 },99 },
106 });100 });
107 }));101 });
108 102
109export const writeLocalWorkspaceState = (input: {103export const writeLocalWorkspaceState = (input: {
110 context: ResolvedLocalWorkspaceContext;104 context: ResolvedLocalWorkspaceContext;
111 state: LocalWorkspaceState;105 state: LocalWorkspaceState;
112}): Effect.Effect<void, LocalFileSystemError> =>106}): Effect.Effect<void, LocalFileSystemError, FileSystem.FileSystem> =>
113 provideNodeFileSystem(Effect.gen(function* () {107 Effect.gen(function* () {
114 const fs = yield* FileSystem.FileSystem;108 const fs = yield* FileSystem.FileSystem;
115 yield* fs.makeDirectory(input.context.stateDirectory, { recursive: true }).pipe(109 yield* fs.makeDirectory(input.context.stateDirectory, { recursive: true }).pipe(
116 Effect.mapError(mapFileSystemError(input.context.stateDirectory, "create state directory")),110 Effect.mapError(mapFileSystemError(input.context.stateDirectory, "create state directory")),
@@ -121,4 +115,4 @@ export const writeLocalWorkspaceState = (input: { · 4 unmodified lines
121 ).pipe(115 ).pipe(
122 Effect.mapError(mapFileSystemError(localWorkspaceStatePath(input.context), "write workspace state")),116 Effect.mapError(mapFileSystemError(localWorkspaceStatePath(input.context), "write workspace state")),
123 );117 );
124 }));118 });
packages/control-plane/src/runtime/local-workspace-sync.ts+3−2
@@ -1,3 +1,4 @@
1import { FileSystem } from "@effect/platform";
1import * as Effect from "effect/Effect";2import * as Effect from "effect/Effect";
2 3
3import {4import {
@@ -42,7 +43,7 @@ export const derivePolicyConfigKey = ( · 38 unmodified lines
42const pruneLocalWorkspaceState = (input: {43const pruneLocalWorkspaceState = (input: {
43 context: ResolvedLocalWorkspaceContext;44 context: ResolvedLocalWorkspaceContext;
44 loadedConfig: LoadedLocalExecutorConfig;45 loadedConfig: LoadedLocalExecutorConfig;
45}): Effect.Effect<LocalWorkspaceState, Error, never> =>46}): Effect.Effect<LocalWorkspaceState, Error, FileSystem.FileSystem> =>
46 Effect.gen(function* () {47 Effect.gen(function* () {
47 const currentState = yield* loadLocalWorkspaceState(input.context);48 const currentState = yield* loadLocalWorkspaceState(input.context);
48 49
@@ -82,7 +83,7 @@ const pruneLocalWorkspaceState = (input: { · 33 unmodified lines
82export const synchronizeLocalWorkspaceState = (input: {83export const synchronizeLocalWorkspaceState = (input: {
83 context: ResolvedLocalWorkspaceContext;84 context: ResolvedLocalWorkspaceContext;
84 loadedConfig: LoadedLocalExecutorConfig;85 loadedConfig: LoadedLocalExecutorConfig;
85}): Effect.Effect<LoadedLocalExecutorConfig["config"], Error, never> =>86}): Effect.Effect<LoadedLocalExecutorConfig["config"], Error, FileSystem.FileSystem> =>
86 Effect.gen(function* () {87 Effect.gen(function* () {
87 yield* pruneLocalWorkspaceState({88 yield* pruneLocalWorkspaceState({
88 context: input.context,89 context: input.context,
packages/control-plane/src/runtime/secret-material-providers.ts+108−0
@@ -11,11 +11,15 @@ import { · 10 unmodified lines
11 SecretMaterialIdSchema,11 SecretMaterialIdSchema,
12 type SecretRef,12 type SecretRef,
13} from "#schema";13} from "#schema";
14import * as Context from "effect/Context";
14import * as Effect from "effect/Effect";15import * as Effect from "effect/Effect";
16import * as Layer from "effect/Layer";
15import * as Option from "effect/Option";17import * as Option from "effect/Option";
16 18
17import { resolveConfigRelativePath } from "./local-config";19import { resolveConfigRelativePath } from "./local-config";
18import { fromConfigSecretProviderId } from "./local-config-secrets";20import { fromConfigSecretProviderId } from "./local-config-secrets";
21import { getRuntimeLocalWorkspaceOption } from "./local-runtime-context";
22import { ControlPlaneStore } from "./store";
19import type { ControlPlaneStoreShape } from "./store";23import type { ControlPlaneStoreShape } from "./store";
20 24
21export const ENV_SECRET_PROVIDER_ID = "env";25export const ENV_SECRET_PROVIDER_ID = "env";
@@ -46,6 +50,18 @@ export type DeleteSecretMaterial = ( · 24 unmodified lines
46 ref: SecretRef,50 ref: SecretRef,
47) => Effect.Effect<boolean, Error, never>;51) => Effect.Effect<boolean, Error, never>;
48 52
53export class SecretMaterialResolverService extends Context.Tag(
54 "#runtime/SecretMaterialResolverService",
55)<SecretMaterialResolverService, ResolveSecretMaterial>() {}
56
57export class SecretMaterialStorerService extends Context.Tag(
58 "#runtime/SecretMaterialStorerService",
59)<SecretMaterialStorerService, StoreSecretMaterial>() {}
60
61export class SecretMaterialDeleterService extends Context.Tag(
62 "#runtime/SecretMaterialDeleterService",
63)<SecretMaterialDeleterService, DeleteSecretMaterial>() {}
64
49type SecretMaterialProviderRuntime = {65type SecretMaterialProviderRuntime = {
50 rows: ControlPlaneStoreShape;66 rows: ControlPlaneStoreShape;
51 env: NodeJS.ProcessEnv;67 env: NodeJS.ProcessEnv;
@@ -740,3 +756,95 @@ export const createDefaultSecretMaterialDeleter = (input: { · 688 unmodified lines
740 });756 });
741 });757 });
742};758};
759
760const resolveRuntimeSecretMaterialConfig = (input: {
761 localConfig?: LocalExecutorConfig | null;
762 workspaceRoot?: string | null;
763}) =>
764 Effect.gen(function* () {
765 const runtimeLocalWorkspace = yield* getRuntimeLocalWorkspaceOption();
766
767 return {
768 localConfig: input.localConfig
769 ?? runtimeLocalWorkspace?.loadedConfig.config
770 ?? null,
771 workspaceRoot: input.workspaceRoot
772 ?? runtimeLocalWorkspace?.context.workspaceRoot
773 ?? null,
774 };
775 });
776
777export const SecretMaterialResolverLive = (input: {
778 resolveSecretMaterial?: ResolveSecretMaterial;
779 dangerouslyAllowEnvSecrets?: boolean;
780 keychainServiceName?: string;
781 localConfig?: LocalExecutorConfig | null;
782 workspaceRoot?: string | null;
783} = {}) =>
784 input.resolveSecretMaterial
785 ? Layer.succeed(SecretMaterialResolverService, input.resolveSecretMaterial)
786 : Layer.effect(
787 SecretMaterialResolverService,
788 Effect.gen(function* () {
789 const rows = yield* ControlPlaneStore;
790 const runtimeConfig = yield* resolveRuntimeSecretMaterialConfig(input);
791
792 return createDefaultSecretMaterialResolver({
793 rows,
794 dangerouslyAllowEnvSecrets: input.dangerouslyAllowEnvSecrets,
795 keychainServiceName: input.keychainServiceName,
796 localConfig: runtimeConfig.localConfig,
797 workspaceRoot: runtimeConfig.workspaceRoot,
798 });
799 }),
800 );
801
802export const SecretMaterialStorerLive = (input: {
803 storeProviderId?: SecretStoreProviderId;
804 dangerouslyAllowEnvSecrets?: boolean;
805 keychainServiceName?: string;
806} = {}) =>
807 Layer.effect(
808 SecretMaterialStorerService,
809 Effect.gen(function* () {
810 const rows = yield* ControlPlaneStore;
811
812 return createDefaultSecretMaterialStorer({
813 rows,
814 storeProviderId: input.storeProviderId,
815 dangerouslyAllowEnvSecrets: input.dangerouslyAllowEnvSecrets,
816 keychainServiceName: input.keychainServiceName,
817 });
818 }),
819 );
820
821export const SecretMaterialDeleterLive = (input: {
822 dangerouslyAllowEnvSecrets?: boolean;
823 keychainServiceName?: string;
824} = {}) =>
825 Layer.effect(
826 SecretMaterialDeleterService,
827 Effect.gen(function* () {
828 const rows = yield* ControlPlaneStore;
829
830 return createDefaultSecretMaterialDeleter({
831 rows,
832 dangerouslyAllowEnvSecrets: input.dangerouslyAllowEnvSecrets,
833 keychainServiceName: input.keychainServiceName,
834 });
835 }),
836 );
837
838export const SecretMaterialLive = (input: {
839 resolveSecretMaterial?: ResolveSecretMaterial;
840 storeProviderId?: SecretStoreProviderId;
841 dangerouslyAllowEnvSecrets?: boolean;
842 keychainServiceName?: string;
843 localConfig?: LocalExecutorConfig | null;
844 workspaceRoot?: string | null;
845} = {}) =>
846 Layer.mergeAll(
847 SecretMaterialResolverLive(input),
848 SecretMaterialStorerLive(input),
849 SecretMaterialDeleterLive(input),
850 );
packages/control-plane/src/runtime/source-adapters/google-discovery.ts+8−0
@@ -4,6 +4,7 @@ import { · 3 unmodified lines
4 HttpClientRequest,4 HttpClientRequest,
5} from "@effect/platform";5} from "@effect/platform";
6import {6import {
7 allowAllToolInteractions,
7 makeToolInvokerFromTools,8 makeToolInvokerFromTools,
8 typeSignatureFromSchemaJson,9 typeSignatureFromSchemaJson,
9} from "@executor/codemode-core";10} from "@executor/codemode-core";
@@ -616,6 +617,13 @@ export const googleDiscoverySourceAdapter: SourceAdapter = { · 606 unmodified lines
616 tools: {617 tools: {
617 [path]: tool,618 [path]: tool,
618 },619 },
620 // Authorization was already handled by authorizePersistedToolInvocation
621 // at the workspace level, so skip the redundant tool-level interaction
622 // gate. Without this, the default "required" interaction on write tools
623 // would re-elicit approval and then merge the approval form content
624 // (e.g. { approve: true }) into the tool args, breaking schemas that
625 // use additionalProperties: false.
626 onToolInteraction: allowAllToolInteractions,
619 onElicitation,627 onElicitation,
620 }).invoke({628 }).invoke({
621 path,629 path,
packages/control-plane/src/runtime/source-adapters/graphql.ts+2−0
@@ -1,4 +1,5 @@
1import {1import {
2 allowAllToolInteractions,
2 applyCookiePlacementsToHeaders,3 applyCookiePlacementsToHeaders,
3 applyHttpQueryPlacementsToUrl,4 applyHttpQueryPlacementsToUrl,
4 applyJsonBodyPlacements,5 applyJsonBodyPlacements,
@@ -524,6 +525,7 @@ export const graphqlSourceAdapter: SourceAdapter = { · 519 unmodified lines
524 tools: {525 tools: {
525 [path]: tool,526 [path]: tool,
526 },527 },
528 onToolInteraction: allowAllToolInteractions,
527 onElicitation,529 onElicitation,
528 }).invoke({530 }).invoke({
529 path,531 path,
packages/control-plane/src/runtime/source-adapters/mcp.ts+2−0
@@ -1,4 +1,5 @@
1import {1import {
2 allowAllToolInteractions,
2 applyCookiePlacementsToHeaders,3 applyCookiePlacementsToHeaders,
3 makeToolInvokerFromTools,4 makeToolInvokerFromTools,
4} from "@executor/codemode-core";5} from "@executor/codemode-core";
@@ -409,6 +410,7 @@ export const mcpSourceAdapter: SourceAdapter = { · 404 unmodified lines
409 410
410 return yield* makeToolInvokerFromTools({411 return yield* makeToolInvokerFromTools({
411 tools,412 tools,
413 onToolInteraction: allowAllToolInteractions,
412 onElicitation,414 onElicitation,
413 }).invoke({415 }).invoke({
414 path,416 path,
packages/control-plane/src/runtime/source-adapters/openapi.ts+2−0
@@ -4,6 +4,7 @@ import { · 3 unmodified lines
4 HttpClientRequest,4 HttpClientRequest,
5} from "@effect/platform";5} from "@effect/platform";
6import {6import {
7 allowAllToolInteractions,
7 applyCookiePlacementsToHeaders,8 applyCookiePlacementsToHeaders,
8 applyHttpQueryPlacementsToUrl,9 applyHttpQueryPlacementsToUrl,
9 makeToolInvokerFromTools,10 makeToolInvokerFromTools,
@@ -563,6 +564,7 @@ export const openApiSourceAdapter: SourceAdapter = { · 553 unmodified lines
563 tools: {564 tools: {
564 [path]: tool,565 [path]: tool,
565 },566 },
567 onToolInteraction: allowAllToolInteractions,
566 onElicitation,568 onElicitation,
567 }).invoke({569 }).invoke({
568 path,570 path,
packages/control-plane/src/runtime/source-adapters/types.ts+0−2
@@ -24,7 +24,6 @@ import * as Schema from "effect/Schema"; · 23 unmodified lines
24 24
25import type { ResolvedSourceAuthMaterial } from "../source-auth-material";25import type { ResolvedSourceAuthMaterial } from "../source-auth-material";
26import type { ResolveSecretMaterial } from "../secret-material-providers";26import type { ResolveSecretMaterial } from "../secret-material-providers";
27import type { ControlPlaneStoreShape } from "../store";
28 27
29export type SourceAdapterFamily = "http_api" | "mcp" | "internal";28export type SourceAdapterFamily = "http_api" | "mcp" | "internal";
30export type SourceAdapterInputSchema = Schema.Schema<any, any, never>;29export type SourceAdapterInputSchema = Schema.Schema<any, any, never>;
@@ -82,7 +81,6 @@ export type SourceAdapterOauth2SetupConfig = { · 51 unmodified lines
82};81};
83 82
84export type SourceAdapterInvokePersistedToolInput = {83export type SourceAdapterInvokePersistedToolInput = {
85 rows: ControlPlaneStoreShape;
86 workspaceId: Source["workspaceId"];84 workspaceId: Source["workspaceId"];
87 accountId: AccountId;85 accountId: AccountId;
88 source: Source;86 source: Source;
packages/control-plane/src/runtime/source-auth-material.test.ts+94−88
@@ -7,7 +7,7 @@ import { · 6 unmodified lines
7 type Source,7 type Source,
8} from "#schema";8} from "#schema";
9 9
10import { resolveSourceAuthMaterial } from "./source-auth-material";10import { resolveSourceAuthMaterialWithDeps } from "./source-auth-material";
11 11
12const makeSource = (overrides: Partial<Source> = {}): Source => ({12const makeSource = (overrides: Partial<Source> = {}): Source => ({
13 id: SourceIdSchema.make("src_tool_artifacts"),13 id: SourceIdSchema.make("src_tool_artifacts"),
@@ -36,113 +36,119 @@ const makeSource = (overrides: Partial<Source> = {}): Source => ({ · 22 unmodified lines
36describe("source-auth-material", () => {36describe("source-auth-material", () => {
37 describe("resolveSourceAuthMaterial", () => {37 describe("resolveSourceAuthMaterial", () => {
38 it("returns empty headers for auth.kind none", async () => {38 it("returns empty headers for auth.kind none", async () => {
39 const auth = await Effect.runPromise(resolveSourceAuthMaterial({39 await Effect.runPromise(Effect.gen(function* () {
40 source: makeSource({40 const auth = yield* resolveSourceAuthMaterialWithDeps({
41 auth: { kind: "none" },41 source: makeSource({
42 }),42 auth: { kind: "none" },
43 resolveSecretMaterial: () => Effect.die("should not be called"),43 }),
44 }));44 resolveSecretMaterial: () => Effect.die("should not be called"),
45 });
45 46
46 expect(auth).toEqual({47 expect(auth).toEqual({
47 placements: [],48 placements: [],
48 headers: {},49 headers: {},
49 queryParams: {},50 queryParams: {},
50 cookies: {},51 cookies: {},
51 bodyValues: {},52 bodyValues: {},
52 expiresAt: null,53 expiresAt: null,
53 refreshAfter: null,54 refreshAfter: null,
54 });55 });
56 }));
55 });57 });
56 58
57 it("resolves bearer auth headers from the configured token ref", async () => {59 it("resolves bearer auth headers from the configured token ref", async () => {
58 const calls: string[] = [];60 const calls: string[] = [];
59 61
60 const auth = await Effect.runPromise(resolveSourceAuthMaterial({62 await Effect.runPromise(Effect.gen(function* () {
61 source: makeSource({63 const auth = yield* resolveSourceAuthMaterialWithDeps({
62 auth: {64 source: makeSource({
63 kind: "bearer",65 auth: {
64 headerName: "X-Api-Key",66 kind: "bearer",
65 prefix: "Token ",67 headerName: "X-Api-Key",
66 token: {68 prefix: "Token ",
67 providerId: "local",69 token: {
68 handle: "sec_bearer",70 providerId: "local",
71 handle: "sec_bearer",
72 },
69 },73 },
74 }),
75 resolveSecretMaterial: ({ ref }) => {
76 calls.push(`${ref.providerId}:${ref.handle}`);
77 return Effect.succeed("resolved-bearer-token");
70 },78 },
71 }),79 });
72 resolveSecretMaterial: ({ ref }) => {
73 calls.push(`${ref.providerId}:${ref.handle}`);
74 return Effect.succeed("resolved-bearer-token");
75 },
76 }));
77 80
78 expect(calls).toEqual(["local:sec_bearer"]);81 expect(calls).toEqual(["local:sec_bearer"]);
79 expect(auth).toEqual({82 expect(auth).toEqual({
80 placements: [83 placements: [
81 {84 {
82 location: "header",85 location: "header",
83 name: "X-Api-Key",86 name: "X-Api-Key",
84 value: "Token resolved-bearer-token",87 value: "Token resolved-bearer-token",
88 },
89 ],
90 headers: {
91 "X-Api-Key": "Token resolved-bearer-token",
85 },92 },
86 ],93 queryParams: {},
87 headers: {94 cookies: {},
88 "X-Api-Key": "Token resolved-bearer-token",95 bodyValues: {},
89 },96 expiresAt: null,
90 queryParams: {},97 refreshAfter: null,
91 cookies: {},98 });
92 bodyValues: {},99 }));
93 expiresAt: null,
94 refreshAfter: null,
95 });
96 });100 });
97 101
98 it("uses the oauth access token ref and ignores the refresh token", async () => {102 it("uses the oauth access token ref and ignores the refresh token", async () => {
99 const calls: string[] = [];103 const calls: string[] = [];
100 104
101 const auth = await Effect.runPromise(resolveSourceAuthMaterial({105 await Effect.runPromise(Effect.gen(function* () {
102 source: makeSource({106 const auth = yield* resolveSourceAuthMaterialWithDeps({
103 kind: "graphql",107 source: makeSource({
104 endpoint: "https://example.com/graphql",108 kind: "graphql",
105 binding: {109 endpoint: "https://example.com/graphql",
106 defaultHeaders: null,110 binding: {
107 },111 defaultHeaders: null,
108 auth: {
109 kind: "oauth2",
110 headerName: "Authorization",
111 prefix: "Bearer ",
112 accessToken: {
113 providerId: "local",
114 handle: "sec_access",
115 },112 },
116 refreshToken: {113 auth: {
117 providerId: "local",114 kind: "oauth2",
118 handle: "sec_refresh",115 headerName: "Authorization",
116 prefix: "Bearer ",
117 accessToken: {
118 providerId: "local",
119 handle: "sec_access",
120 },
121 refreshToken: {
122 providerId: "local",
123 handle: "sec_refresh",
124 },
119 },125 },
126 }),
127 resolveSecretMaterial: ({ ref }) => {
128 calls.push(`${ref.providerId}:${ref.handle}`);
129 return Effect.succeed("resolved-access-token");
120 },130 },
121 }),131 });
122 resolveSecretMaterial: ({ ref }) => {
123 calls.push(`${ref.providerId}:${ref.handle}`);
124 return Effect.succeed("resolved-access-token");
125 },
126 }));
127 132
128 expect(calls).toEqual(["local:sec_access"]);133 expect(calls).toEqual(["local:sec_access"]);
129 expect(auth).toEqual({134 expect(auth).toEqual({
130 placements: [135 placements: [
131 {136 {
132 location: "header",137 location: "header",
133 name: "Authorization",138 name: "Authorization",
134 value: "Bearer resolved-access-token",139 value: "Bearer resolved-access-token",
140 },
141 ],
142 headers: {
143 Authorization: "Bearer resolved-access-token",
135 },144 },
136 ],145 queryParams: {},
137 headers: {146 cookies: {},
138 Authorization: "Bearer resolved-access-token",147 bodyValues: {},
139 },148 expiresAt: null,
140 queryParams: {},149 refreshAfter: null,
141 cookies: {},150 });
142 bodyValues: {},151 }));
143 expiresAt: null,
144 refreshAfter: null,
145 });
146 });152 });
147 });153 });
148});154});
packages/control-plane/src/runtime/source-auth-material.ts+43−2
@@ -4,10 +4,12 @@ import type { · 3 unmodified lines
4 Source,4 Source,
5 SourceAuth,5 SourceAuth,
6} from "#schema";6} from "#schema";
7import * as Context from "effect/Context";
7import * as Effect from "effect/Effect";8import * as Effect from "effect/Effect";
9import * as Layer from "effect/Layer";
8import * as Option from "effect/Option";10import * as Option from "effect/Option";
9 11
10import type { ControlPlaneStoreShape } from "./store";12import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";
11import {13import {
12 authArtifactFromSourceAuth,14 authArtifactFromSourceAuth,
13 resolveAuthArtifactMaterial,15 resolveAuthArtifactMaterial,
@@ -18,6 +20,7 @@ import type { · 4 unmodified lines
18 ResolveSecretMaterial,20 ResolveSecretMaterial,
19 SecretMaterialResolveContext,21 SecretMaterialResolveContext,
20} from "./secret-material-providers";22} from "./secret-material-providers";
23import { SecretMaterialResolverService } from "./secret-material-providers";
21 24
22const authForSlot = (input: {25const authForSlot = (input: {
23 source: Source;26 source: Source;
@@ -38,7 +41,20 @@ const authForSlot = (input: { · 14 unmodified lines
38 return input.source.importAuth;41 return input.source.importAuth;
39};42};
40 43
41export const resolveSourceAuthMaterial = (input: {44export type RuntimeSourceAuthMaterialShape = {
45 resolve: (input: {
46 source: Source;
47 slot?: CredentialSlot;
48 actorAccountId?: AccountId | null;
49 context?: SecretMaterialResolveContext;
50 }) => Effect.Effect<ResolvedSourceAuthMaterial, Error, never>;
51};
52
53export class RuntimeSourceAuthMaterialService extends Context.Tag(
54 "#runtime/RuntimeSourceAuthMaterialService",
55)<RuntimeSourceAuthMaterialService, RuntimeSourceAuthMaterialShape>() {}
56
57export const resolveSourceAuthMaterialWithDeps = (input: {
42 source: Source;58 source: Source;
43 slot?: CredentialSlot;59 slot?: CredentialSlot;
44 actorAccountId?: AccountId | null;60 actorAccountId?: AccountId | null;
@@ -106,4 +122,29 @@ export const resolveSourceAuthMaterial = (input: { · 61 unmodified lines
106 });122 });
107 });123 });
108 124
125export const resolveSourceAuthMaterial = (input: {
126 source: Source;
127 slot?: CredentialSlot;
128 actorAccountId?: AccountId | null;
129 context?: SecretMaterialResolveContext;
130}): Effect.Effect<ResolvedSourceAuthMaterial, Error, RuntimeSourceAuthMaterialService> =>
131 Effect.flatMap(RuntimeSourceAuthMaterialService, (service) => service.resolve(input));
132
133export const RuntimeSourceAuthMaterialLive = Layer.effect(
134 RuntimeSourceAuthMaterialService,
135 Effect.gen(function* () {
136 const rows = yield* ControlPlaneStore;
137 const resolveSecretMaterial = yield* SecretMaterialResolverService;
138
139 return RuntimeSourceAuthMaterialService.of({
140 resolve: (input) =>
141 resolveSourceAuthMaterialWithDeps({
142 ...input,
143 rows,
144 resolveSecretMaterial,
145 }),
146 });
147 }),
148);
149
109export type { ResolvedSourceAuthMaterial } from "./auth-artifacts";150export type { ResolvedSourceAuthMaterial } from "./auth-artifacts";
packages/control-plane/src/runtime/source-auth-service.ts+87−90
@@ -7,7 +7,6 @@ import { · 6 unmodified lines
7 AccountId,7 AccountId,
8 type CredentialSlot,8 type CredentialSlot,
9 ExecutionIdSchema,9 ExecutionIdSchema,
10 type LocalExecutorConfig,
11 McpSourceAuthSessionDataJsonSchema,10 McpSourceAuthSessionDataJsonSchema,
12 type McpSourceAuthSessionData,11 type McpSourceAuthSessionData,
13 OAuth2PkceSourceAuthSessionDataJsonSchema,12 OAuth2PkceSourceAuthSessionDataJsonSchema,
@@ -42,7 +41,8 @@ import { · 28 unmodified lines
42 type LiveExecutionManager,41 type LiveExecutionManager,
43} from "./live-execution";42} from "./live-execution";
44import {43import {
45 RuntimeLocalWorkspaceService,44 getRuntimeLocalWorkspaceOption,
45 provideOptionalRuntimeLocalWorkspace,
46 type RuntimeLocalWorkspaceState,46 type RuntimeLocalWorkspaceState,
47} from "./local-runtime-context";47} from "./local-runtime-context";
48import {48import {
@@ -61,15 +61,15 @@ import { · 12 unmodified lines
61import { isSourceCredentialRequiredError } from "./source-adapters/shared";61import { isSourceCredentialRequiredError } from "./source-adapters/shared";
62import {62import {
63 createDefaultSecretMaterialDeleter,63 createDefaultSecretMaterialDeleter,
64 createDefaultSecretMaterialResolver,
65 createDefaultSecretMaterialStorer,
66 type ResolveSecretMaterial,64 type ResolveSecretMaterial,
65 SecretMaterialResolverService,
66 SecretMaterialStorerService,
67 type StoreSecretMaterial,67 type StoreSecretMaterial,
68} from "./secret-material-providers";68} from "./secret-material-providers";
69import { upsertOauth2AuthorizedUserLeaseFromTokenResponse } from "./auth-leases";69import { upsertOauth2AuthorizedUserLeaseFromTokenResponse } from "./auth-leases";
70import {70import {
71 persistMcpRecipeMaterializationFromManifest,71 RuntimeSourceMaterializationService,
72 syncSourceMaterialization,72 type RuntimeSourceMaterializationShape,
73} from "./source-materialization";73} from "./source-materialization";
74import {74import {
75 buildOAuth2AuthorizationUrl,75 buildOAuth2AuthorizationUrl,
@@ -81,6 +81,8 @@ import { · 5 unmodified lines
81 loadSourceById,81 loadSourceById,
82 loadSourcesInWorkspace,82 loadSourcesInWorkspace,
83 persistSource,83 persistSource,
84 type RuntimeSourceStore,
85 RuntimeSourceStoreService,
84} from "./source-store";86} from "./source-store";
85import type { WorkspaceStorageServices } from "./local-storage";87import type { WorkspaceStorageServices } from "./local-storage";
86import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";88import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";
@@ -358,7 +360,7 @@ const serializeJson = (value: unknown): string | null => { · 271 unmodified lines
358 return JSON.stringify(value);360 return JSON.stringify(value);
359};361};
360 362
361const updateSourceStatus = (rows: ControlPlaneStoreShape, source: Source, input: {363const updateSourceStatus = (sourceStore: RuntimeSourceStore, source: Source, input: {
362 actorAccountId?: AccountId | null;364 actorAccountId?: AccountId | null;
363 status: Source["status"];365 status: Source["status"];
364 lastError?: string | null;366 lastError?: string | null;
@@ -366,13 +368,13 @@ const updateSourceStatus = (rows: ControlPlaneStoreShape, source: Source, input: · 1 unmodified line
366 importAuth?: Source["importAuth"];368 importAuth?: Source["importAuth"];
367}) =>369}) =>
368 Effect.gen(function* () {370 Effect.gen(function* () {
369 const latest = yield* loadSourceById(rows, {371 const latest = yield* sourceStore.loadSourceById({
370 workspaceId: source.workspaceId,372 workspaceId: source.workspaceId,
371 sourceId: source.id,373 sourceId: source.id,
372 actorAccountId: input.actorAccountId,374 actorAccountId: input.actorAccountId,
373 });375 });
374 376
375 return yield* persistSource(rows, {377 return yield* sourceStore.persistSource({
376 ...latest,378 ...latest,
377 status: input.status,379 status: input.status,
378 lastError: input.lastError ?? null,380 lastError: input.lastError ?? null,
@@ -857,6 +859,7 @@ const resolveExistingSourceOauthClient = (input: { · 478 unmodified lines
857 859
858const startOauth2PkceSourceCredentialSetup = (input: {860const startOauth2PkceSourceCredentialSetup = (input: {
859 rows: ControlPlaneStoreShape;861 rows: ControlPlaneStoreShape;
862 sourceStore: RuntimeSourceStore;
860 source: Source;863 source: Source;
861 actorAccountId?: AccountId | null;864 actorAccountId?: AccountId | null;
862 executionId?: SourceAuthSession["executionId"];865 executionId?: SourceAuthSession["executionId"];
@@ -951,7 +954,7 @@ const startOauth2PkceSourceCredentialSetup = (input: { · 88 unmodified lines
951 updatedAt: now,954 updatedAt: now,
952 });955 });
953 956
954 const authRequiredSource = yield* updateSourceStatus(input.rows, input.source, {957 const authRequiredSource = yield* updateSourceStatus(input.sourceStore, input.source, {
955 actorAccountId: input.actorAccountId,958 actorAccountId: input.actorAccountId,
956 status: "auth_required",959 status: "auth_required",
957 lastError: null,960 lastError: null,
@@ -974,6 +977,8 @@ const startOauth2PkceSourceCredentialSetup = (input: { · 16 unmodified lines
974 977
975const connectMcpSourceInternal = (input: {978const connectMcpSourceInternal = (input: {
976 rows: ControlPlaneStoreShape;979 rows: ControlPlaneStoreShape;
980 sourceStore: RuntimeSourceStore;
981 sourceMaterialization: RuntimeSourceMaterializationShape;
977 getLocalServerBaseUrl?: () => string | undefined;982 getLocalServerBaseUrl?: () => string | undefined;
978 baseUrl?: string | null;983 baseUrl?: string | null;
979 workspaceId: WorkspaceId;984 workspaceId: WorkspaceId;
@@ -995,7 +1000,7 @@ const connectMcpSourceInternal = (input: { · 15 unmodified lines
995 const normalizedEndpoint = normalizeEndpoint(input.endpoint);1000 const normalizedEndpoint = normalizeEndpoint(input.endpoint);
996 const existing = yield* (1001 const existing = yield* (
997 input.sourceId1002 input.sourceId
998 ? loadSourceById(input.rows, {1003 ? input.sourceStore.loadSourceById({
999 workspaceId: input.workspaceId,1004 workspaceId: input.workspaceId,
1000 sourceId: input.sourceId,1005 sourceId: input.sourceId,
1001 actorAccountId: input.actorAccountId,1006 actorAccountId: input.actorAccountId,
@@ -1006,7 +1011,7 @@ const connectMcpSourceInternal = (input: { · 4 unmodified lines
1006 : Effect.fail(new Error(`Expected MCP source, received ${source.kind}`)),1011 : Effect.fail(new Error(`Expected MCP source, received ${source.kind}`)),
1007 ),1012 ),
1008 )1013 )
1009 : loadSourcesInWorkspace(input.rows, input.workspaceId, {1014 : input.sourceStore.loadSourcesInWorkspace(input.workspaceId, {
1010 actorAccountId: input.actorAccountId,1015 actorAccountId: input.actorAccountId,
1011 }).pipe(1016 }).pipe(
1012 Effect.map((sources) =>1017 Effect.map((sources) =>
@@ -1079,14 +1084,12 @@ const connectMcpSourceInternal = (input: { · 66 unmodified lines
1079 now,1084 now,
1080 });1085 });
1081 1086
1082 const persistedDraft = yield* persistSource(input.rows, draftSource, {1087 const persistedDraft = yield* input.sourceStore.persistSource(draftSource, {
1083 actorAccountId: input.actorAccountId,1088 actorAccountId: input.actorAccountId,
1084 });1089 });
1085 yield* syncSourceMaterialization({1090 yield* input.sourceMaterialization.sync({
1086 rows: input.rows,
1087 source: persistedDraft,1091 source: persistedDraft,
1088 actorAccountId: input.actorAccountId,1092 actorAccountId: input.actorAccountId,
1089 resolveSecretMaterial: input.resolveSecretMaterial,
1090 });1093 });
1091 1094
1092 const discovered = yield* Effect.either(1095 const discovered = yield* Effect.either(
@@ -1100,15 +1103,14 @@ const connectMcpSourceInternal = (input: { · 7 unmodified lines
1100 onLeft: () => Effect.succeed(null),1103 onLeft: () => Effect.succeed(null),
1101 onRight: (result) =>1104 onRight: (result) =>
1102 Effect.gen(function* () {1105 Effect.gen(function* () {
1103 const connected = yield* updateSourceStatus(input.rows, persistedDraft, {1106 const connected = yield* updateSourceStatus(input.sourceStore, persistedDraft, {
1104 actorAccountId: input.actorAccountId,1107 actorAccountId: input.actorAccountId,
1105 status: "connected",1108 status: "connected",
1106 lastError: null,1109 lastError: null,
1107 auth: { kind: "none" },1110 auth: { kind: "none" },
1108 });1111 });
1109 const indexed = yield* Effect.either(1112 const indexed = yield* Effect.either(
1110 persistMcpRecipeMaterializationFromManifest({1113 input.sourceMaterialization.persistMcpRecipeMaterializationFromManifest({
1111 rows: input.rows,
1112 source: connected,1114 source: connected,
1113 manifestEntries: result.manifest.tools,1115 manifestEntries: result.manifest.tools,
1114 }),1116 }),
@@ -1116,7 +1118,7 @@ const connectMcpSourceInternal = (input: { · 1 unmodified line
1116 1118
1117 return yield* Either.match(indexed, {1119 return yield* Either.match(indexed, {
1118 onLeft: (error) =>1120 onLeft: (error) =>
1119 updateSourceStatus(input.rows, connected, {1121 updateSourceStatus(input.sourceStore, connected, {
1120 actorAccountId: input.actorAccountId,1122 actorAccountId: input.actorAccountId,
1121 status: "error",1123 status: "error",
1122 lastError: error.message,1124 lastError: error.message,
@@ -1156,7 +1158,7 @@ const connectMcpSourceInternal = (input: { · 33 unmodified lines
1156 state,1158 state,
1157 });1159 });
1158 1160
1159 const authRequiredSource = yield* updateSourceStatus(input.rows, persistedDraft, {1161 const authRequiredSource = yield* updateSourceStatus(input.sourceStore, persistedDraft, {
1160 actorAccountId: input.actorAccountId,1162 actorAccountId: input.actorAccountId,
1161 status: "auth_required",1163 status: "auth_required",
1162 lastError: null,1164 lastError: null,
@@ -1203,6 +1205,8 @@ const connectMcpSourceInternal = (input: { · 40 unmodified lines
1203 1205
1204const addExecutorHttpSource = (input: {1206const addExecutorHttpSource = (input: {
1205 rows: ControlPlaneStoreShape;1207 rows: ControlPlaneStoreShape;
1208 sourceStore: RuntimeSourceStore;
1209 sourceMaterialization: RuntimeSourceMaterializationShape;
1206 sourceInput: Extract<ExecutorAddSourceInput, { kind: "openapi" | "graphql" }>;1210 sourceInput: Extract<ExecutorAddSourceInput, { kind: "openapi" | "graphql" }>;
1207 storeSecretMaterial: StoreSecretMaterial;1211 storeSecretMaterial: StoreSecretMaterial;
1208 resolveSecretMaterial: ResolveSecretMaterial;1212 resolveSecretMaterial: ResolveSecretMaterial;
@@ -1214,8 +1218,7 @@ const addExecutorHttpSource = (input: { · 5 unmodified lines
1214 const normalizedSpecUrl = input.sourceInput.kind === "openapi"1218 const normalizedSpecUrl = input.sourceInput.kind === "openapi"
1215 ? normalizeEndpoint(input.sourceInput.specUrl)1219 ? normalizeEndpoint(input.sourceInput.specUrl)
1216 : null;1220 : null;
1217 const existingSources = yield* loadSourcesInWorkspace(1221 const existingSources = yield* input.sourceStore.loadSourcesInWorkspace(
1218 input.rows,
1219 input.sourceInput.workspaceId,1222 input.sourceInput.workspaceId,
1220 {1223 {
1221 actorAccountId: input.sourceInput.actorAccountId,1224 actorAccountId: input.sourceInput.actorAccountId,
@@ -1313,7 +1316,7 @@ const addExecutorHttpSource = (input: { · 91 unmodified lines
1313 now,1316 now,
1314 });1317 });
1315 1318
1316 const persistedDraft = yield* persistSource(input.rows, draftSource, {1319 const persistedDraft = yield* input.sourceStore.persistSource(draftSource, {
1317 actorAccountId: input.sourceInput.actorAccountId,1320 actorAccountId: input.sourceInput.actorAccountId,
1318 });1321 });
1319 1322
@@ -1327,6 +1330,7 @@ const addExecutorHttpSource = (input: { · 7 unmodified lines
1327 if (baseUrl) {1330 if (baseUrl) {
1328 const oauthRequired = yield* startOauth2PkceSourceCredentialSetup({1331 const oauthRequired = yield* startOauth2PkceSourceCredentialSetup({
1329 rows: input.rows,1332 rows: input.rows,
1333 sourceStore: input.sourceStore,
1330 source: persistedDraft,1334 source: persistedDraft,
1331 actorAccountId: input.sourceInput.actorAccountId,1335 actorAccountId: input.sourceInput.actorAccountId,
1332 executionId: input.sourceInput.executionId,1336 executionId: input.sourceInput.executionId,
@@ -1340,7 +1344,7 @@ const addExecutorHttpSource = (input: { · 7 unmodified lines
1340 }1344 }
1341 }1345 }
1342 1346
1343 const authRequiredSource = yield* updateSourceStatus(input.rows, persistedDraft, {1347 const authRequiredSource = yield* updateSourceStatus(input.sourceStore, persistedDraft, {
1344 actorAccountId: input.sourceInput.actorAccountId,1348 actorAccountId: input.sourceInput.actorAccountId,
1345 status: "auth_required",1349 status: "auth_required",
1346 lastError: null,1350 lastError: null,
@@ -1354,21 +1358,19 @@ const addExecutorHttpSource = (input: { · 7 unmodified lines
1354 }1358 }
1355 1359
1356 const synced = yield* Effect.either(1360 const synced = yield* Effect.either(
1357 syncSourceMaterialization({1361 input.sourceMaterialization.sync({
1358 rows: input.rows,
1359 source: {1362 source: {
1360 ...persistedDraft,1363 ...persistedDraft,
1361 status: "connected",1364 status: "connected",
1362 },1365 },
1363 actorAccountId: input.sourceInput.actorAccountId,1366 actorAccountId: input.sourceInput.actorAccountId,
1364 resolveSecretMaterial: input.resolveSecretMaterial,
1365 }),1367 }),
1366 );1368 );
1367 1369
1368 return yield* Either.match(synced, {1370 return yield* Either.match(synced, {
1369 onLeft: (error) =>1371 onLeft: (error) =>
1370 isSourceCredentialRequiredError(error)1372 isSourceCredentialRequiredError(error)
1371 ? updateSourceStatus(input.rows, persistedDraft, {1373 ? updateSourceStatus(input.sourceStore, persistedDraft, {
1372 actorAccountId: input.sourceInput.actorAccountId,1374 actorAccountId: input.sourceInput.actorAccountId,
1373 status: "auth_required",1375 status: "auth_required",
1374 lastError: null,1376 lastError: null,
@@ -1381,7 +1383,7 @@ const addExecutorHttpSource = (input: { · 6 unmodified lines
1381 } satisfies ExecutorSourceAddResult)1383 } satisfies ExecutorSourceAddResult)
1382 ),1384 ),
1383 )1385 )
1384 : updateSourceStatus(input.rows, persistedDraft, {1386 : updateSourceStatus(input.sourceStore, persistedDraft, {
1385 actorAccountId: input.sourceInput.actorAccountId,1387 actorAccountId: input.sourceInput.actorAccountId,
1386 status: "error",1388 status: "error",
1387 lastError: error.message,1389 lastError: error.message,
@@ -1389,7 +1391,7 @@ const addExecutorHttpSource = (input: { · 1 unmodified line
1389 Effect.zipRight(Effect.fail(error)),1391 Effect.zipRight(Effect.fail(error)),
1390 ),1392 ),
1391 onRight: () =>1393 onRight: () =>
1392 updateSourceStatus(input.rows, persistedDraft, {1394 updateSourceStatus(input.sourceStore, persistedDraft, {
1393 actorAccountId: input.sourceInput.actorAccountId,1395 actorAccountId: input.sourceInput.actorAccountId,
1394 status: "connected",1396 status: "connected",
1395 lastError: null,1397 lastError: null,
@@ -1406,6 +1408,8 @@ const addExecutorHttpSource = (input: { · 10 unmodified lines
1406 1408
1407const addExecutorGoogleDiscoverySource = (input: {1409const addExecutorGoogleDiscoverySource = (input: {
1408 rows: ControlPlaneStoreShape;1410 rows: ControlPlaneStoreShape;
1411 sourceStore: RuntimeSourceStore;
1412 sourceMaterialization: RuntimeSourceMaterializationShape;
1409 sourceInput: Extract<ExecutorAddSourceInput, { kind: "google_discovery" }>;1413 sourceInput: Extract<ExecutorAddSourceInput, { kind: "google_discovery" }>;
1410 storeSecretMaterial: StoreSecretMaterial;1414 storeSecretMaterial: StoreSecretMaterial;
1411 resolveSecretMaterial: ResolveSecretMaterial;1415 resolveSecretMaterial: ResolveSecretMaterial;
@@ -1419,8 +1423,7 @@ const addExecutorGoogleDiscoverySource = (input: { · 7 unmodified lines
1419 trimOrNull(input.sourceInput.discoveryUrl)1423 trimOrNull(input.sourceInput.discoveryUrl)
1420 ?? defaultGoogleDiscoveryUrl(normalizedService, normalizedVersion),1424 ?? defaultGoogleDiscoveryUrl(normalizedService, normalizedVersion),
1421 );1425 );
1422 const existingSources = yield* loadSourcesInWorkspace(1426 const existingSources = yield* input.sourceStore.loadSourcesInWorkspace(
1423 input.rows,
1424 input.sourceInput.workspaceId,1427 input.sourceInput.workspaceId,
1425 {1428 {
1426 actorAccountId: input.sourceInput.actorAccountId,1429 actorAccountId: input.sourceInput.actorAccountId,
@@ -1524,7 +1527,7 @@ const addExecutorGoogleDiscoverySource = (input: { · 97 unmodified lines
1524 now,1527 now,
1525 });1528 });
1526 1529
1527 const persistedDraft = yield* persistSource(input.rows, draftSource, {1530 const persistedDraft = yield* input.sourceStore.persistSource(draftSource, {
1528 actorAccountId: input.sourceInput.actorAccountId,1531 actorAccountId: input.sourceInput.actorAccountId,
1529 });1532 });
1530 1533
@@ -1547,6 +1550,7 @@ const addExecutorGoogleDiscoverySource = (input: { · 16 unmodified lines
1547 if (baseUrl) {1550 if (baseUrl) {
1548 const oauthRequired = yield* startOauth2PkceSourceCredentialSetup({1551 const oauthRequired = yield* startOauth2PkceSourceCredentialSetup({
1549 rows: input.rows,1552 rows: input.rows,
1553 sourceStore: input.sourceStore,
1550 source: persistedDraft,1554 source: persistedDraft,
1551 actorAccountId: input.sourceInput.actorAccountId,1555 actorAccountId: input.sourceInput.actorAccountId,
1552 executionId: input.sourceInput.executionId,1556 executionId: input.sourceInput.executionId,
@@ -1560,7 +1564,7 @@ const addExecutorGoogleDiscoverySource = (input: { · 7 unmodified lines
1560 }1564 }
1561 }1565 }
1562 1566
1563 const authRequiredSource = yield* updateSourceStatus(input.rows, persistedDraft, {1567 const authRequiredSource = yield* updateSourceStatus(input.sourceStore, persistedDraft, {
1564 actorAccountId: input.sourceInput.actorAccountId,1568 actorAccountId: input.sourceInput.actorAccountId,
1565 status: "auth_required",1569 status: "auth_required",
1566 lastError: null,1570 lastError: null,
@@ -1574,21 +1578,19 @@ const addExecutorGoogleDiscoverySource = (input: { · 7 unmodified lines
1574 }1578 }
1575 1579
1576 const synced = yield* Effect.either(1580 const synced = yield* Effect.either(
1577 syncSourceMaterialization({1581 input.sourceMaterialization.sync({
1578 rows: input.rows,
1579 source: {1582 source: {
1580 ...persistedDraft,1583 ...persistedDraft,
1581 status: "connected",1584 status: "connected",
1582 },1585 },
1583 actorAccountId: input.sourceInput.actorAccountId,1586 actorAccountId: input.sourceInput.actorAccountId,
1584 resolveSecretMaterial: input.resolveSecretMaterial,
1585 }),1587 }),
1586 );1588 );
1587 1589
1588 return yield* Either.match(synced, {1590 return yield* Either.match(synced, {
1589 onLeft: (error) =>1591 onLeft: (error) =>
1590 isSourceCredentialRequiredError(error)1592 isSourceCredentialRequiredError(error)
1591 ? updateSourceStatus(input.rows, persistedDraft, {1593 ? updateSourceStatus(input.sourceStore, persistedDraft, {
1592 actorAccountId: input.sourceInput.actorAccountId,1594 actorAccountId: input.sourceInput.actorAccountId,
1593 status: "auth_required",1595 status: "auth_required",
1594 lastError: null,1596 lastError: null,
@@ -1601,7 +1603,7 @@ const addExecutorGoogleDiscoverySource = (input: { · 6 unmodified lines
1601 } satisfies ExecutorSourceAddResult)1603 } satisfies ExecutorSourceAddResult)
1602 ),1604 ),
1603 )1605 )
1604 : updateSourceStatus(input.rows, persistedDraft, {1606 : updateSourceStatus(input.sourceStore, persistedDraft, {
1605 actorAccountId: input.sourceInput.actorAccountId,1607 actorAccountId: input.sourceInput.actorAccountId,
1606 status: "error",1608 status: "error",
1607 lastError: error.message,1609 lastError: error.message,
@@ -1609,7 +1611,7 @@ const addExecutorGoogleDiscoverySource = (input: { · 1 unmodified line
1609 Effect.zipRight(Effect.fail(error)),1611 Effect.zipRight(Effect.fail(error)),
1610 ),1612 ),
1611 onRight: () =>1613 onRight: () =>
1612 updateSourceStatus(input.rows, persistedDraft, {1614 updateSourceStatus(input.sourceStore, persistedDraft, {
1613 actorAccountId: input.sourceInput.actorAccountId,1615 actorAccountId: input.sourceInput.actorAccountId,
1614 status: "connected",1616 status: "connected",
1615 lastError: null,1617 lastError: null,
@@ -1668,19 +1670,13 @@ type RuntimeSourceAuthServiceShape = { · 52 unmodified lines
1668export const createRuntimeSourceAuthService = (input: {1670export const createRuntimeSourceAuthService = (input: {
1669 rows: ControlPlaneStoreShape;1671 rows: ControlPlaneStoreShape;
1670 liveExecutionManager: LiveExecutionManager;1672 liveExecutionManager: LiveExecutionManager;
1673 sourceStore: RuntimeSourceStore;
1674 sourceMaterialization: RuntimeSourceMaterializationShape;
1675 resolveSecretMaterial: ResolveSecretMaterial;
1676 storeSecretMaterial: StoreSecretMaterial;
1671 getLocalServerBaseUrl?: () => string | undefined;1677 getLocalServerBaseUrl?: () => string | undefined;
1672 localConfig?: LocalExecutorConfig | null;
1673 workspaceRoot?: string | null;
1674 localWorkspaceState?: RuntimeLocalWorkspaceState;1678 localWorkspaceState?: RuntimeLocalWorkspaceState;
1675}) => {1679}) => {
1676 const resolveSecretMaterial = createDefaultSecretMaterialResolver({
1677 rows: input.rows,
1678 localConfig: input.localConfig,
1679 workspaceRoot: input.workspaceRoot,
1680 });
1681 const storeSecretMaterial = createDefaultSecretMaterialStorer({
1682 rows: input.rows,
1683 });
1684 const mirrorLocalSourceResult = (1680 const mirrorLocalSourceResult = (
1685 result: ExecutorSourceAddResult,1681 result: ExecutorSourceAddResult,
1686 ): Effect.Effect<ExecutorSourceAddResult, Error, WorkspaceStorageServices> =>1682 ): Effect.Effect<ExecutorSourceAddResult, Error, WorkspaceStorageServices> =>
@@ -1690,27 +1686,20 @@ export const createRuntimeSourceAuthService = (input: { · 3 unmodified lines
1690 ): Effect.Effect<McpSourceConnectResult, Error, WorkspaceStorageServices> =>1686 ): Effect.Effect<McpSourceConnectResult, Error, WorkspaceStorageServices> =>
1691 Effect.succeed(result);1687 Effect.succeed(result);
1692 const provideLocalWorkspace = <A, E, R>(effect: Effect.Effect<A, E, R>) =>1688 const provideLocalWorkspace = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
1693 input.localWorkspaceState1689 provideOptionalRuntimeLocalWorkspace(effect, input.localWorkspaceState);
1694 ? effect.pipe(
1695 Effect.provideService(
1696 RuntimeLocalWorkspaceService,
1697 input.localWorkspaceState,
1698 ),
1699 )
1700 : effect;
1701 1690
1702 return {1691 return {
1703 getLocalServerBaseUrl: () => input.getLocalServerBaseUrl?.() ?? null,1692 getLocalServerBaseUrl: () => input.getLocalServerBaseUrl?.() ?? null,
1704 1693
1705 storeSecretMaterial: ({ purpose, value }) =>1694 storeSecretMaterial: ({ purpose, value }) =>
1706 storeSecretMaterial({1695 input.storeSecretMaterial({
1707 purpose,1696 purpose,
1708 value,1697 value,
1709 }),1698 }),
1710 1699
1711 getSourceById: ({ workspaceId, sourceId, actorAccountId }) =>1700 getSourceById: ({ workspaceId, sourceId, actorAccountId }) =>
1712 provideLocalWorkspace(1701 provideLocalWorkspace(
1713 loadSourceById(input.rows, {1702 input.sourceStore.loadSourceById({
1714 workspaceId,1703 workspaceId,
1715 sourceId,1704 sourceId,
1716 actorAccountId,1705 actorAccountId,
@@ -1722,26 +1711,32 @@ export const createRuntimeSourceAuthService = (input: { · 5 unmodified lines
1722 (sourceInput.kind === "google_discovery"1711 (sourceInput.kind === "google_discovery"
1723 ? addExecutorGoogleDiscoverySource({1712 ? addExecutorGoogleDiscoverySource({
1724 rows: input.rows,1713 rows: input.rows,
1714 sourceStore: input.sourceStore,
1715 sourceMaterialization: input.sourceMaterialization,
1725 sourceInput,1716 sourceInput,
1726 storeSecretMaterial,1717 storeSecretMaterial: input.storeSecretMaterial,
1727 resolveSecretMaterial,1718 resolveSecretMaterial: input.resolveSecretMaterial,
1728 getLocalServerBaseUrl: input.getLocalServerBaseUrl,1719 getLocalServerBaseUrl: input.getLocalServerBaseUrl,
1729 baseUrl: options?.baseUrl,1720 baseUrl: options?.baseUrl,
1730 })1721 })
1731 : hasSourceAdapterFamily(sourceInput.kind ?? "mcp", "http_api")1722 : hasSourceAdapterFamily(sourceInput.kind ?? "mcp", "http_api")
1732 ? addExecutorHttpSource({1723 ? addExecutorHttpSource({
1733 rows: input.rows,1724 rows: input.rows,
1725 sourceStore: input.sourceStore,
1726 sourceMaterialization: input.sourceMaterialization,
1734 sourceInput: sourceInput as Extract<1727 sourceInput: sourceInput as Extract<
1735 ExecutorAddSourceInput,1728 ExecutorAddSourceInput,
1736 { kind: "openapi" | "graphql" }1729 { kind: "openapi" | "graphql" }
1737 >,1730 >,
1738 storeSecretMaterial,1731 storeSecretMaterial: input.storeSecretMaterial,
1739 resolveSecretMaterial,1732 resolveSecretMaterial: input.resolveSecretMaterial,
1740 getLocalServerBaseUrl: input.getLocalServerBaseUrl,1733 getLocalServerBaseUrl: input.getLocalServerBaseUrl,
1741 baseUrl: options?.baseUrl,1734 baseUrl: options?.baseUrl,
1742 })1735 })
1743 : connectMcpSourceInternal({1736 : connectMcpSourceInternal({
1744 rows: input.rows,1737 rows: input.rows,
1738 sourceStore: input.sourceStore,
1739 sourceMaterialization: input.sourceMaterialization,
1745 getLocalServerBaseUrl: input.getLocalServerBaseUrl,1740 getLocalServerBaseUrl: input.getLocalServerBaseUrl,
1746 workspaceId: sourceInput.workspaceId,1741 workspaceId: sourceInput.workspaceId,
1747 actorAccountId: sourceInput.actorAccountId,1742 actorAccountId: sourceInput.actorAccountId,
@@ -1752,7 +1747,7 @@ export const createRuntimeSourceAuthService = (input: { · 4 unmodified lines
1752 namespace: sourceInput.namespace,1747 namespace: sourceInput.namespace,
1753 mcpDiscoveryElicitation: options?.mcpDiscoveryElicitation,1748 mcpDiscoveryElicitation: options?.mcpDiscoveryElicitation,
1754 baseUrl: options?.baseUrl,1749 baseUrl: options?.baseUrl,
1755 resolveSecretMaterial,1750 resolveSecretMaterial: input.resolveSecretMaterial,
1756 })).pipe(1751 })).pipe(
1757 Effect.flatMap(mirrorLocalSourceResult),1752 Effect.flatMap(mirrorLocalSourceResult),
1758 ),1753 ),
@@ -1762,6 +1757,8 @@ export const createRuntimeSourceAuthService = (input: { · 3 unmodified lines
1762 provideLocalWorkspace(1757 provideLocalWorkspace(
1763 connectMcpSourceInternal({1758 connectMcpSourceInternal({
1764 rows: input.rows,1759 rows: input.rows,
1760 sourceStore: input.sourceStore,
1761 sourceMaterialization: input.sourceMaterialization,
1765 getLocalServerBaseUrl: input.getLocalServerBaseUrl,1762 getLocalServerBaseUrl: input.getLocalServerBaseUrl,
1766 workspaceId: sourceInput.workspaceId,1763 workspaceId: sourceInput.workspaceId,
1767 actorAccountId: sourceInput.actorAccountId,1764 actorAccountId: sourceInput.actorAccountId,
@@ -1776,7 +1773,7 @@ export const createRuntimeSourceAuthService = (input: { · 8 unmodified lines
1776 queryParams: sourceInput.queryParams,1773 queryParams: sourceInput.queryParams,
1777 headers: sourceInput.headers,1774 headers: sourceInput.headers,
1778 baseUrl: sourceInput.baseUrl,1775 baseUrl: sourceInput.baseUrl,
1779 resolveSecretMaterial,1776 resolveSecretMaterial: input.resolveSecretMaterial,
1780 }).pipe(1777 }).pipe(
1781 Effect.flatMap(mirrorLocalMcpSourceResult),1778 Effect.flatMap(mirrorLocalMcpSourceResult),
1782 ),1779 ),
@@ -1912,13 +1909,13 @@ export const createRuntimeSourceAuthService = (input: { · 129 unmodified lines
1912 displayName: readSourceOAuthSessionDisplayName(session.state),1909 displayName: readSourceOAuthSessionDisplayName(session.state),
1913 endpoint: sessionData.endpoint,1910 endpoint: sessionData.endpoint,
1914 });1911 });
1915 const accessTokenRef = yield* storeSecretMaterial({1912 const accessTokenRef = yield* input.storeSecretMaterial({
1916 purpose: "oauth_access_token",1913 purpose: "oauth_access_token",
1917 value: exchanged.tokens.access_token,1914 value: exchanged.tokens.access_token,
1918 name: oauthSecretName,1915 name: oauthSecretName,
1919 });1916 });
1920 const refreshTokenRef = exchanged.tokens.refresh_token1917 const refreshTokenRef = exchanged.tokens.refresh_token
1921 ? yield* storeSecretMaterial({1918 ? yield* input.storeSecretMaterial({
1922 purpose: "oauth_refresh_token",1919 purpose: "oauth_refresh_token",
1923 value: exchanged.tokens.refresh_token,1920 value: exchanged.tokens.refresh_token,
1924 name: `${oauthSecretName} Refresh`,1921 name: `${oauthSecretName} Refresh`,
@@ -1991,7 +1988,7 @@ export const createRuntimeSourceAuthService = (input: { · 66 unmodified lines
1991 );1988 );
1992 }1989 }
1993 1990
1994 const source = yield* loadSourceById(input.rows, {1991 const source = yield* input.sourceStore.loadSourceById({
1995 workspaceId: session.workspaceId,1992 workspaceId: session.workspaceId,
1996 sourceId: session.sourceId,1993 sourceId: session.sourceId,
1997 actorAccountId: session.actorAccountId,1994 actorAccountId: session.actorAccountId,
@@ -2024,16 +2021,14 @@ export const createRuntimeSourceAuthService = (input: { · 26 unmodified lines
2024 errorText: reason,2021 errorText: reason,
2025 }),2022 }),
2026 );2023 );
2027 const failedSource = yield* updateSourceStatus(input.rows, source, {2024 const failedSource = yield* updateSourceStatus(input.sourceStore, source, {
2028 actorAccountId: session.actorAccountId,2025 actorAccountId: session.actorAccountId,
2029 status: "error",2026 status: "error",
2030 lastError: reason,2027 lastError: reason,
2031 });2028 });
2032 yield* syncSourceMaterialization({2029 yield* input.sourceMaterialization.sync({
2033 rows: input.rows,
2034 source: failedSource,2030 source: failedSource,
2035 actorAccountId: session.actorAccountId,2031 actorAccountId: session.actorAccountId,
2036 resolveSecretMaterial,
2037 });2032 });
2038 yield* completeLiveInteraction({2033 yield* completeLiveInteraction({
2039 rows: input.rows,2034 rows: input.rows,
@@ -2063,7 +2058,7 @@ export const createRuntimeSourceAuthService = (input: { · 23 unmodified lines
2063 const oauthSessionData = decodeOauth2PkceSourceAuthSessionData(session);2058 const oauthSessionData = decodeOauth2PkceSourceAuthSessionData(session);
2064 let clientSecret: string | null = null;2059 let clientSecret: string | null = null;
2065 if (oauthSessionData.clientSecret) {2060 if (oauthSessionData.clientSecret) {
2066 clientSecret = yield* resolveSecretMaterial({2061 clientSecret = yield* input.resolveSecretMaterial({
2067 ref: oauthSessionData.clientSecret,2062 ref: oauthSessionData.clientSecret,
2068 });2063 });
2069 }2064 }
@@ -2084,7 +2079,7 @@ export const createRuntimeSourceAuthService = (input: { · 14 unmodified lines
2084 );2079 );
2085 }2080 }
2086 2081
2087 const refreshTokenRef = yield* storeSecretMaterial({2082 const refreshTokenRef = yield* input.storeSecretMaterial({
2088 purpose: "oauth_refresh_token",2083 purpose: "oauth_refresh_token",
2089 value: refreshToken,2084 value: refreshToken,
2090 name: `${source.name} Refresh`,2085 name: `${source.name} Refresh`,
@@ -2092,7 +2087,7 @@ export const createRuntimeSourceAuthService = (input: { · 1 unmodified line
2092 const grantedScopes = trimOrNull(exchanged.scope)2087 const grantedScopes = trimOrNull(exchanged.scope)
2093 ? exchanged.scope!.split(/\s+/).filter((scope) => scope.length > 0)2088 ? exchanged.scope!.split(/\s+/).filter((scope) => scope.length > 0)
2094 : [...oauthSessionData.scopes];2089 : [...oauthSessionData.scopes];
2095 connectedSource = yield* updateSourceStatus(input.rows, source, {2090 connectedSource = yield* updateSourceStatus(input.sourceStore, source, {
2096 actorAccountId: session.actorAccountId,2091 actorAccountId: session.actorAccountId,
2097 status: "connected",2092 status: "connected",
2098 lastError: null,2093 lastError: null,
@@ -2157,20 +2152,20 @@ export const createRuntimeSourceAuthService = (input: { · 58 unmodified lines
2157 displayName: source.name,2152 displayName: source.name,
2158 endpoint: source.endpoint,2153 endpoint: source.endpoint,
2159 });2154 });
2160 const accessTokenRef = yield* storeSecretMaterial({2155 const accessTokenRef = yield* input.storeSecretMaterial({
2161 purpose: "oauth_access_token",2156 purpose: "oauth_access_token",
2162 value: exchanged.tokens.access_token,2157 value: exchanged.tokens.access_token,
2163 name: oauthSecretName,2158 name: oauthSecretName,
2164 });2159 });
2165 const refreshTokenRef = exchanged.tokens.refresh_token2160 const refreshTokenRef = exchanged.tokens.refresh_token
2166 ? yield* storeSecretMaterial({2161 ? yield* input.storeSecretMaterial({
2167 purpose: "oauth_refresh_token",2162 purpose: "oauth_refresh_token",
2168 value: exchanged.tokens.refresh_token,2163 value: exchanged.tokens.refresh_token,
2169 name: `${oauthSecretName} Refresh`,2164 name: `${oauthSecretName} Refresh`,
2170 })2165 })
2171 : null;2166 : null;
2172 2167
2173 connectedSource = yield* updateSourceStatus(input.rows, source, {2168 connectedSource = yield* updateSourceStatus(input.sourceStore, source, {
2174 actorAccountId: session.actorAccountId,2169 actorAccountId: session.actorAccountId,
2175 status: "connected",2170 status: "connected",
2176 lastError: null,2171 lastError: null,
@@ -2204,16 +2199,14 @@ export const createRuntimeSourceAuthService = (input: { · 27 unmodified lines
2204 );2199 );
2205 }2200 }
2206 const indexed = yield* Effect.either(2201 const indexed = yield* Effect.either(
2207 syncSourceMaterialization({2202 input.sourceMaterialization.sync({
2208 rows: input.rows,
2209 source: connectedSource,2203 source: connectedSource,
2210 actorAccountId: session.actorAccountId,2204 actorAccountId: session.actorAccountId,
2211 resolveSecretMaterial,
2212 }),2205 }),
2213 );2206 );
2214 yield* Either.match(indexed, {2207 yield* Either.match(indexed, {
2215 onLeft: (error) =>2208 onLeft: (error) =>
2216 updateSourceStatus(input.rows, connectedSource, {2209 updateSourceStatus(input.sourceStore, connectedSource, {
2217 actorAccountId: session.actorAccountId,2210 actorAccountId: session.actorAccountId,
2218 status: "error",2211 status: "error",
2219 lastError: error.message,2212 lastError: error.message,
@@ -2245,23 +2238,27 @@ export class RuntimeSourceAuthServiceTag extends Context.Tag( · 25 unmodified lines
2245 2238
2246export const RuntimeSourceAuthServiceLive = (input: {2239export const RuntimeSourceAuthServiceLive = (input: {
2247 getLocalServerBaseUrl?: () => string | undefined;2240 getLocalServerBaseUrl?: () => string | undefined;
2248 localConfig?: LocalExecutorConfig | null;
2249 workspaceRoot?: string | null;
2250 localWorkspaceState?: RuntimeLocalWorkspaceState;
2251} = {}) =>2241} = {}) =>
2252 Layer.effect(2242 Layer.effect(
2253 RuntimeSourceAuthServiceTag,2243 RuntimeSourceAuthServiceTag,
2254 Effect.gen(function* () {2244 Effect.gen(function* () {
2255 const rows = yield* ControlPlaneStore;2245 const rows = yield* ControlPlaneStore;
2256 const liveExecutionManager = yield* LiveExecutionManagerService;2246 const liveExecutionManager = yield* LiveExecutionManagerService;
2247 const sourceStore = yield* RuntimeSourceStoreService;
2248 const sourceMaterialization = yield* RuntimeSourceMaterializationService;
2249 const resolveSecretMaterial = yield* SecretMaterialResolverService;
2250 const storeSecretMaterial = yield* SecretMaterialStorerService;
2251 const runtimeLocalWorkspace = yield* getRuntimeLocalWorkspaceOption();
2257 2252
2258 return createRuntimeSourceAuthService({2253 return createRuntimeSourceAuthService({
2259 rows,2254 rows,
2260 liveExecutionManager,2255 liveExecutionManager,
2256 sourceStore,
2257 sourceMaterialization,
2258 resolveSecretMaterial,
2259 storeSecretMaterial,
2261 getLocalServerBaseUrl: input.getLocalServerBaseUrl,2260 getLocalServerBaseUrl: input.getLocalServerBaseUrl,
2262 localConfig: input.localConfig,2261 localWorkspaceState: runtimeLocalWorkspace ?? undefined,
2263 workspaceRoot: input.workspaceRoot,
2264 localWorkspaceState: input.localWorkspaceState,
2265 });2262 });
2266 }),2263 }),
2267 );2264 );
packages/control-plane/src/runtime/source-inspection.ts+5−9
@@ -26,14 +26,12 @@ import { · 25 unmodified lines
26 getSourceAdapterForOperation,26 getSourceAdapterForOperation,
27} from "./source-adapters";27} from "./source-adapters";
28import {28import {
29 loadSourceWithRecipe,29 RuntimeSourceRecipeStoreService,
30 recipeToolMetadata,30 recipeToolMetadata,
31 recipeToolPath,31 recipeToolPath,
32} from "./source-recipes-runtime";32} from "./source-recipes-runtime";
33import type { SourceAdapterPersistedOperationMetadata } from "./source-adapters/types";33import type { SourceAdapterPersistedOperationMetadata } from "./source-adapters/types";
34import { namespaceFromSourceName } from "./source-names";34import { namespaceFromSourceName } from "./source-names";
35import { loadSourceById } from "./source-store";
36import { ControlPlaneStore } from "./store";
37 35
38const sourceInspectOps = {36const sourceInspectOps = {
39 bundle: operationErrors("sources.inspect.bundle"),37 bundle: operationErrors("sources.inspect.bundle"),
@@ -109,10 +107,9 @@ const loadSourceRecipeRecord = (input: { · 69 unmodified lines
109 workspaceId: WorkspaceId;107 workspaceId: WorkspaceId;
110 sourceId: SourceId;108 sourceId: SourceId;
111}) =>109}) =>
112 Effect.flatMap(ControlPlaneStore, (store) =>110 Effect.flatMap(RuntimeSourceRecipeStoreService, (sourceRecipeStore) =>
113 Effect.gen(function* () {111 Effect.gen(function* () {
114 const recipe = yield* loadSourceWithRecipe({112 const recipe = yield* sourceRecipeStore.loadSourceWithRecipe({
115 rows: store,
116 workspaceId: input.workspaceId,113 workspaceId: input.workspaceId,
117 sourceId: input.sourceId,114 sourceId: input.sourceId,
118 }).pipe(115 }).pipe(
@@ -390,10 +387,9 @@ export const getSourceInspectionSchemaBundle = (input: { · 271 unmodified lines
390 sourceId: SourceId;387 sourceId: SourceId;
391 schemaBundleId: string;388 schemaBundleId: string;
392}) =>389}) =>
393 Effect.flatMap(ControlPlaneStore, (store) =>390 Effect.flatMap(RuntimeSourceRecipeStoreService, (sourceRecipeStore) =>
394 Effect.gen(function* () {391 Effect.gen(function* () {
395 const recipe = yield* loadSourceWithRecipe({392 const recipe = yield* sourceRecipeStore.loadSourceWithRecipe({
396 rows: store,
397 workspaceId: input.workspaceId,393 workspaceId: input.workspaceId,
398 sourceId: input.sourceId,394 sourceId: input.sourceId,
399 }).pipe(395 }).pipe(
packages/control-plane/src/runtime/source-materialization.ts+165−43
@@ -3,51 +3,99 @@ import type { · 2 unmodified lines
3 Source,3 Source,
4 SourceStatus,4 SourceStatus,
5} from "#schema";5} from "#schema";
6import * as Context from "effect/Context";
6import * as Effect from "effect/Effect";7import * as Effect from "effect/Effect";
8import * as Layer from "effect/Layer";
7 9
8import type { ControlPlaneStoreShape } from "./store";
9import {10import {
10 requireRuntimeLocalWorkspace,11 RuntimeLocalWorkspaceService,
12 type RuntimeLocalWorkspaceState,
11} from "./local-runtime-context";13} from "./local-runtime-context";
12import {14import {
13 type WorkspaceStorageServices,
14 SourceArtifactStore,15 SourceArtifactStore,
16 type SourceArtifactStoreShape,
15 WorkspaceStateStore,17 WorkspaceStateStore,
18 type WorkspaceStateStoreShape,
16} from "./local-storage";19} from "./local-storage";
17import {20import {
18 type LocalWorkspaceState,21 type LocalWorkspaceState,
19} from "./local-workspace-state";22} from "./local-workspace-state";
20import { resolveSourceAuthMaterial } from "./source-auth-material";23import {
24 RuntimeSourceAuthMaterialService,
25} from "./source-auth-material";
21import {26import {
22 getSourceAdapterForSource,27 getSourceAdapterForSource,
23} from "./source-adapters";28} from "./source-adapters";
24import {29import {
25 materializationFromMcpManifestEntries,30 materializationFromMcpManifestEntries,
26} from "./source-adapters/mcp";31} from "./source-adapters/mcp";
27import type {32import { SecretMaterialResolverService } from "./secret-material-providers";
28 ResolveSecretMaterial as ResolveSourceSecretMaterial,
29} from "./secret-material-providers";
30 33
31const shouldIndexSource = (source: Source): boolean =>34const shouldIndexSource = (source: Source): boolean =>
32 source.enabled35 source.enabled
33 && source.status === "connected"36 && source.status === "connected"
34 && getSourceAdapterForSource(source).family !== "internal";37 && getSourceAdapterForSource(source).family !== "internal";
35 38
36export const syncSourceMaterialization = (input: {39type RuntimeSourceMaterializationDeps = {
37 rows: ControlPlaneStoreShape;40 runtimeLocalWorkspace: RuntimeLocalWorkspaceState;
38 source: Source;41 workspaceStateStore: WorkspaceStateStoreShape;
39 actorAccountId?: AccountId | null;42 sourceArtifactStore: SourceArtifactStoreShape;
40 resolveSecretMaterial: ResolveSourceSecretMaterial;43 resolveSecretMaterial: Effect.Effect.Success<typeof SecretMaterialResolverService>;
41}): Effect.Effect<void, Error, WorkspaceStorageServices> =>44 sourceAuthMaterialService: Effect.Effect.Success<typeof RuntimeSourceAuthMaterialService>;
45};
46
47type SourceMaterializationServices =
48 | RuntimeLocalWorkspaceService
49 | WorkspaceStateStore
50 | SourceArtifactStore
51 | RuntimeSourceAuthMaterialService
52 | SecretMaterialResolverService;
53
54export type RuntimeSourceMaterializationShape = {
55 sync: (input: {
56 source: Source;
57 actorAccountId?: AccountId | null;
58 }) => Effect.Effect<void, Error, never>;
59 persistMcpRecipeMaterializationFromManifest: (input: {
60 source: Source;
61 manifestEntries: Parameters<typeof materializationFromMcpManifestEntries>[0]["manifestEntries"];
62 }) => Effect.Effect<void, Error, never>;
63};
64
65export class RuntimeSourceMaterializationService extends Context.Tag(
66 "#runtime/RuntimeSourceMaterializationService",
67)<RuntimeSourceMaterializationService, RuntimeSourceMaterializationShape>() {}
68
69const ensureRuntimeMaterializationWorkspace = (
70 deps: RuntimeSourceMaterializationDeps,
71 workspaceId: Source["workspaceId"],
72) => {
73 if (deps.runtimeLocalWorkspace.installation.workspaceId !== workspaceId) {
74 return Effect.fail(
75 new Error(
76 `Runtime local workspace mismatch: expected ${workspaceId}, got ${deps.runtimeLocalWorkspace.installation.workspaceId}`,
77 ),
78 );
79 }
80
81 return Effect.succeed(deps.runtimeLocalWorkspace.context);
82};
83
84const syncSourceMaterializationWithDeps = (
85 deps: RuntimeSourceMaterializationDeps,
86 input: {
87 source: Source;
88 actorAccountId?: AccountId | null;
89 },
90): Effect.Effect<void, Error, never> =>
42 Effect.gen(function* () {91 Effect.gen(function* () {
43 const runtimeLocalWorkspace = yield* requireRuntimeLocalWorkspace(input.source.workspaceId);92 const workspaceContext = yield* ensureRuntimeMaterializationWorkspace(
44 const workspaceStateStore = yield* WorkspaceStateStore;93 deps,
45 const sourceArtifactStore = yield* SourceArtifactStore;94 input.source.workspaceId,
95 );
46 96
47 if (!shouldIndexSource(input.source)) {97 if (!shouldIndexSource(input.source)) {
48 const state = yield* workspaceStateStore.load(98 const state = yield* deps.workspaceStateStore.load(workspaceContext);
49 runtimeLocalWorkspace.context,
50 );
51 const existingSourceState = state.sources[input.source.id];99 const existingSourceState = state.sources[input.source.id];
52 const nextState: LocalWorkspaceState = {100 const nextState: LocalWorkspaceState = {
53 ...state,101 ...state,
@@ -62,8 +110,8 @@ export const syncSourceMaterialization = (input: { · 8 unmodified lines
62 },110 },
63 },111 },
64 };112 };
65 yield* workspaceStateStore.write({113 yield* deps.workspaceStateStore.write({
66 context: runtimeLocalWorkspace.context,114 context: workspaceContext,
67 state: nextState,115 state: nextState,
68 });116 });
69 return;117 return;
@@ -72,28 +120,24 @@ export const syncSourceMaterialization = (input: { · 2 unmodified lines
72 const adapter = getSourceAdapterForSource(input.source);120 const adapter = getSourceAdapterForSource(input.source);
73 const materialization = yield* adapter.materializeSource({121 const materialization = yield* adapter.materializeSource({
74 source: input.source,122 source: input.source,
75 resolveSecretMaterial: input.resolveSecretMaterial,123 resolveSecretMaterial: deps.resolveSecretMaterial,
76 resolveAuthMaterialForSlot: (slot) =>124 resolveAuthMaterialForSlot: (slot) =>
77 resolveSourceAuthMaterial({125 deps.sourceAuthMaterialService.resolve({
78 rows: input.rows,
79 source: input.source,126 source: input.source,
80 slot,127 slot,
81 actorAccountId: input.actorAccountId,128 actorAccountId: input.actorAccountId,
82 resolveSecretMaterial: input.resolveSecretMaterial,
83 }),129 }),
84 });130 });
85 yield* sourceArtifactStore.write({131 yield* deps.sourceArtifactStore.write({
86 context: runtimeLocalWorkspace.context,132 context: workspaceContext,
87 sourceId: input.source.id,133 sourceId: input.source.id,
88 artifact: sourceArtifactStore.build({134 artifact: deps.sourceArtifactStore.build({
89 source: input.source,135 source: input.source,
90 materialization,136 materialization,
91 }),137 }),
92 });138 });
93 139
94 const state = yield* workspaceStateStore.load(140 const state = yield* deps.workspaceStateStore.load(workspaceContext);
95 runtimeLocalWorkspace.context,
96 );
97 const existingSourceState = state.sources[input.source.id];141 const existingSourceState = state.sources[input.source.id];
98 const nextState: LocalWorkspaceState = {142 const nextState: LocalWorkspaceState = {
99 ...state,143 ...state,
@@ -108,32 +152,110 @@ export const syncSourceMaterialization = (input: { · 8 unmodified lines
108 },152 },
109 },153 },
110 };154 };
111 yield* workspaceStateStore.write({155 yield* deps.workspaceStateStore.write({
112 context: runtimeLocalWorkspace.context,156 context: workspaceContext,
113 state: nextState,157 state: nextState,
114 });158 });
115 });159 });
116 160
117export const persistMcpRecipeMaterializationFromManifest = (input: {161const persistMcpRecipeMaterializationFromManifestWithDeps = (
118 rows: ControlPlaneStoreShape;162 deps: RuntimeSourceMaterializationDeps,
119 source: Source;163 input: {
120 manifestEntries: Parameters<typeof materializationFromMcpManifestEntries>[0]["manifestEntries"];164 source: Source;
121}): Effect.Effect<void, Error, WorkspaceStorageServices> =>165 manifestEntries: Parameters<typeof materializationFromMcpManifestEntries>[0]["manifestEntries"];
166 },
167): Effect.Effect<void, Error, never> =>
122 Effect.gen(function* () {168 Effect.gen(function* () {
123 const runtimeLocalWorkspace = yield* requireRuntimeLocalWorkspace(input.source.workspaceId);169 const workspaceContext = yield* ensureRuntimeMaterializationWorkspace(
124 const sourceArtifactStore = yield* SourceArtifactStore;170 deps,
171 input.source.workspaceId,
172 );
125 const materialization = materializationFromMcpManifestEntries({173 const materialization = materializationFromMcpManifestEntries({
126 recipeRevisionId: "src_recipe_rev_materialization" as never,174 recipeRevisionId: "src_recipe_rev_materialization" as never,
127 endpoint: input.source.endpoint,175 endpoint: input.source.endpoint,
128 manifestEntries: input.manifestEntries,176 manifestEntries: input.manifestEntries,
129 });177 });
130 178
131 yield* sourceArtifactStore.write({179 yield* deps.sourceArtifactStore.write({
132 context: runtimeLocalWorkspace.context,180 context: workspaceContext,
133 sourceId: input.source.id,181 sourceId: input.source.id,
134 artifact: sourceArtifactStore.build({182 artifact: deps.sourceArtifactStore.build({
135 source: input.source,183 source: input.source,
136 materialization,184 materialization,
137 }),185 }),
138 });186 });
139 });187 });
188
189export const syncSourceMaterialization = (input: {
190 source: Source;
191 actorAccountId?: AccountId | null;
192}): Effect.Effect<void, Error, SourceMaterializationServices> =>
193 Effect.gen(function* () {
194 const runtimeLocalWorkspace = yield* RuntimeLocalWorkspaceService;
195 const workspaceStateStore = yield* WorkspaceStateStore;
196 const sourceArtifactStore = yield* SourceArtifactStore;
197 const resolveSecretMaterial = yield* SecretMaterialResolverService;
198 const sourceAuthMaterialService = yield* RuntimeSourceAuthMaterialService;
199
200 return yield* syncSourceMaterializationWithDeps(
201 {
202 runtimeLocalWorkspace,
203 workspaceStateStore,
204 sourceArtifactStore,
205 resolveSecretMaterial,
206 sourceAuthMaterialService,
207 },
208 {
209 source: input.source,
210 actorAccountId: input.actorAccountId,
211 },
212 );
213 });
214
215export const persistMcpRecipeMaterializationFromManifest = (input: {
216 source: Source;
217 manifestEntries: Parameters<typeof materializationFromMcpManifestEntries>[0]["manifestEntries"];
218}): Effect.Effect<void, Error, SourceMaterializationServices> =>
219 Effect.gen(function* () {
220 const runtimeLocalWorkspace = yield* RuntimeLocalWorkspaceService;
221 const workspaceStateStore = yield* WorkspaceStateStore;
222 const sourceArtifactStore = yield* SourceArtifactStore;
223 const resolveSecretMaterial = yield* SecretMaterialResolverService;
224 const sourceAuthMaterialService = yield* RuntimeSourceAuthMaterialService;
225
226 return yield* persistMcpRecipeMaterializationFromManifestWithDeps(
227 {
228 runtimeLocalWorkspace,
229 workspaceStateStore,
230 sourceArtifactStore,
231 resolveSecretMaterial,
232 sourceAuthMaterialService,
233 },
234 input,
235 );
236 });
237
238export const RuntimeSourceMaterializationLive = Layer.effect(
239 RuntimeSourceMaterializationService,
240 Effect.gen(function* () {
241 const runtimeLocalWorkspace = yield* RuntimeLocalWorkspaceService;
242 const workspaceStateStore = yield* WorkspaceStateStore;
243 const sourceArtifactStore = yield* SourceArtifactStore;
244 const resolveSecretMaterial = yield* SecretMaterialResolverService;
245 const sourceAuthMaterialService = yield* RuntimeSourceAuthMaterialService;
246
247 const deps: RuntimeSourceMaterializationDeps = {
248 runtimeLocalWorkspace,
249 workspaceStateStore,
250 sourceArtifactStore,
251 resolveSecretMaterial,
252 sourceAuthMaterialService,
253 };
254
255 return RuntimeSourceMaterializationService.of({
256 sync: (input) => syncSourceMaterializationWithDeps(deps, input),
257 persistMcpRecipeMaterializationFromManifest: (input) =>
258 persistMcpRecipeMaterializationFromManifestWithDeps(deps, input),
259 });
260 }),
261);
packages/control-plane/src/runtime/source-recipes-runtime.ts+255−67
@@ -12,24 +12,29 @@ import type { · 11 unmodified lines
12 StoredSourceRecipeRevisionRecord,12 StoredSourceRecipeRevisionRecord,
13 WorkspaceId,13 WorkspaceId,
14} from "#schema";14} from "#schema";
15import * as Context from "effect/Context";
15import * as Effect from "effect/Effect";16import * as Effect from "effect/Effect";
17import * as Layer from "effect/Layer";
16import { LocalSourceArtifactMissingError } from "./local-errors";18import { LocalSourceArtifactMissingError } from "./local-errors";
17import {19import {
18 requireRuntimeLocalWorkspace,20 RuntimeLocalWorkspaceService,
21 type RuntimeLocalWorkspaceState,
19} from "./local-runtime-context";22} from "./local-runtime-context";
20import {23import {
21 type WorkspaceStorageServices,
22 SourceArtifactStore,24 SourceArtifactStore,
25 type SourceArtifactStoreShape,
23} from "./local-storage";26} from "./local-storage";
24import { namespaceFromSourceName } from "./source-names";27import { namespaceFromSourceName } from "./source-names";
25import {28import {
26 getSourceAdapterForOperation,29 getSourceAdapterForOperation,
27 getSourceAdapterForSource,30 getSourceAdapterForSource,
28} from "./source-adapters";31} from "./source-adapters";
29import type { ControlPlaneStoreShape } from "./store";
30import type { SourceAdapterPersistedOperationMetadata } from "./source-adapters/types";32import type { SourceAdapterPersistedOperationMetadata } from "./source-adapters/types";
31import { firstSchemaBundle } from "./source-adapters/shared";33import { firstSchemaBundle } from "./source-adapters/shared";
32import { loadSourceById, loadSourcesInWorkspace } from "./source-store";34import {
35 RuntimeSourceStoreService,
36 type RuntimeSourceStore,
37} from "./source-store";
33 38
34type RecipeManifest = unknown | null;39type RecipeManifest = unknown | null;
35 40
@@ -167,46 +172,119 @@ const primarySchemaBundleForRevision = (input: { · 131 unmodified lines
167 : null;172 : null;
168};173};
169 174
170export const loadWorkspaceSourceRecipes = (input: {175const sourceRecordFromRecipeArtifact = (input: {
171 rows: ControlPlaneStoreShape;176 source: Source;
177 artifact: {
178 recipeId: StoredSourceRecord["recipeId"];
179 revision: StoredSourceRecipeRevisionRecord;
180 };
181}): StoredSourceRecord => ({
182 id: input.source.id,
183 workspaceId: input.source.workspaceId,
184 recipeId: input.artifact.recipeId,
185 recipeRevisionId: input.artifact.revision.id,
186 name: input.source.name,
187 kind: input.source.kind,
188 endpoint: input.source.endpoint,
189 status: input.source.status,
190 enabled: input.source.enabled,
191 namespace: input.source.namespace,
192 importAuthPolicy: input.source.importAuthPolicy,
193 bindingConfigJson: getSourceAdapterForSource(input.source).serializeBindingConfig(input.source),
194 sourceHash: input.source.sourceHash,
195 lastError: input.source.lastError,
196 createdAt: input.source.createdAt,
197 updatedAt: input.source.updatedAt,
198});
199
200type RuntimeSourceRecipeStoreShape = {
201 loadWorkspaceSourceRecipes: (input: {
202 workspaceId: WorkspaceId;
203 actorAccountId?: AccountId | null;
204 }) => Effect.Effect<readonly LoadedSourceRecipe[], Error, never>;
205 loadSourceWithRecipe: (input: {
206 workspaceId: WorkspaceId;
207 sourceId: Source["id"];
208 actorAccountId?: AccountId | null;
209 }) => Effect.Effect<LoadedSourceRecipe, Error | LocalSourceArtifactMissingError, never>;
210 loadWorkspaceSourceRecipeToolIndex: (input: {
211 workspaceId: WorkspaceId;
212 actorAccountId?: AccountId | null;
213 includeSchemas: boolean;
214 }) => Effect.Effect<readonly LoadedSourceRecipeToolIndexEntry[], Error, never>;
215 loadWorkspaceSourceRecipeToolByPath: (input: {
216 workspaceId: WorkspaceId;
217 path: string;
218 actorAccountId?: AccountId | null;
219 includeSchemas: boolean;
220 }) => Effect.Effect<LoadedSourceRecipeToolIndexEntry | null, Error, never>;
221 loadWorkspaceSchemaBundle: (input: {
222 workspaceId: WorkspaceId;
223 id: string;
224 }) => Effect.Effect<
225 { id: string; kind: string; hash: string; refsJson: string } | null,
226 Error,
227 never
228 >;
229};
230
231export type RuntimeSourceRecipeStore = RuntimeSourceRecipeStoreShape;
232
233export class RuntimeSourceRecipeStoreService extends Context.Tag(
234 "#runtime/RuntimeSourceRecipeStoreService",
235)<RuntimeSourceRecipeStoreService, RuntimeSourceRecipeStoreShape>() {}
236
237type RuntimeSourceRecipeStoreDeps = {
238 runtimeLocalWorkspace: RuntimeLocalWorkspaceState;
239 sourceStore: RuntimeSourceStore;
240 sourceArtifactStore: SourceArtifactStoreShape;
241};
242
243type SourceRecipeRuntimeServices =
244 | RuntimeLocalWorkspaceService
245 | RuntimeSourceStoreService
246 | SourceArtifactStore;
247
248const ensureRuntimeRecipeWorkspace = (
249 deps: RuntimeSourceRecipeStoreDeps,
250 workspaceId: WorkspaceId,
251) => {
252 if (deps.runtimeLocalWorkspace.installation.workspaceId !== workspaceId) {
253 return Effect.fail(
254 new Error(
255 `Runtime local workspace mismatch: expected ${workspaceId}, got ${deps.runtimeLocalWorkspace.installation.workspaceId}`,
256 ),
257 );
258 }
259
260 return Effect.succeed(deps.runtimeLocalWorkspace.context);
261};
262
263const loadWorkspaceSourceRecipesWithDeps = (deps: RuntimeSourceRecipeStoreDeps, input: {
172 workspaceId: WorkspaceId;264 workspaceId: WorkspaceId;
173 actorAccountId?: AccountId | null;265 actorAccountId?: AccountId | null;
174}): Effect.Effect<readonly LoadedSourceRecipe[], Error, WorkspaceStorageServices> =>266}): Effect.Effect<readonly LoadedSourceRecipe[], Error, never> =>
175 Effect.gen(function* () {267 Effect.gen(function* () {
176 const runtimeLocalWorkspace = yield* requireRuntimeLocalWorkspace(input.workspaceId);268 const workspaceContext = yield* ensureRuntimeRecipeWorkspace(
177 const sourceArtifactStore = yield* SourceArtifactStore;269 deps,
178 const sources = yield* loadSourcesInWorkspace(input.rows, input.workspaceId, {270 input.workspaceId,
179 actorAccountId: input.actorAccountId,271 );
180 });272 const sources = yield* deps.sourceStore.loadSourcesInWorkspace(
273 input.workspaceId,
274 {
275 actorAccountId: input.actorAccountId,
276 },
277 );
181 const localRecipes = yield* Effect.forEach(sources, (source) =>278 const localRecipes = yield* Effect.forEach(sources, (source) =>
182 Effect.gen(function* () {279 Effect.gen(function* () {
183 const artifact = yield* sourceArtifactStore.read({280 const artifact = yield* deps.sourceArtifactStore.read({
184 context: runtimeLocalWorkspace.context,281 context: workspaceContext,
185 sourceId: source.id,282 sourceId: source.id,
186 });283 });
187 if (artifact === null) {284 if (artifact === null) {
188 return null;285 return null;
189 }286 }
190 287
191 const sourceRecord: StoredSourceRecord = {
192 id: source.id,
193 workspaceId: source.workspaceId,
194 recipeId: artifact.recipeId,
195 recipeRevisionId: artifact.revision.id,
196 name: source.name,
197 kind: source.kind,
198 endpoint: source.endpoint,
199 status: source.status,
200 enabled: source.enabled,
201 namespace: source.namespace,
202 importAuthPolicy: source.importAuthPolicy,
203 bindingConfigJson: getSourceAdapterForSource(source).serializeBindingConfig(source),
204 sourceHash: source.sourceHash,
205 lastError: source.lastError,
206 createdAt: source.createdAt,
207 updatedAt: source.updatedAt,
208 };
209
210 const manifest = yield* parseManifestForRecipe({288 const manifest = yield* parseManifestForRecipe({
211 source,289 source,
212 revision: artifact.revision,290 revision: artifact.revision,
@@ -214,7 +292,7 @@ export const loadWorkspaceSourceRecipes = (input: { · 1 unmodified line
214 292
215 return {293 return {
216 source,294 source,
217 sourceRecord,295 sourceRecord: sourceRecordFromRecipeArtifact({ source, artifact }),
218 revision: artifact.revision,296 revision: artifact.revision,
219 documents: artifact.documents,297 documents: artifact.documents,
220 schemaBundles: artifact.schemaBundles,298 schemaBundles: artifact.schemaBundles,
@@ -226,22 +304,23 @@ export const loadWorkspaceSourceRecipes = (input: { · 5 unmodified lines
226 return localRecipes.filter((recipe): recipe is LoadedSourceRecipe => recipe !== null);304 return localRecipes.filter((recipe): recipe is LoadedSourceRecipe => recipe !== null);
227 });305 });
228 306
229export const loadSourceWithRecipe = (input: {307const loadSourceWithRecipeWithDeps = (deps: RuntimeSourceRecipeStoreDeps, input: {
230 rows: ControlPlaneStoreShape;
231 workspaceId: WorkspaceId;308 workspaceId: WorkspaceId;
232 sourceId: Source["id"];309 sourceId: Source["id"];
233 actorAccountId?: AccountId | null;310 actorAccountId?: AccountId | null;
234}): Effect.Effect<LoadedSourceRecipe, Error | LocalSourceArtifactMissingError, WorkspaceStorageServices> =>311}): Effect.Effect<LoadedSourceRecipe, Error | LocalSourceArtifactMissingError, never> =>
235 Effect.gen(function* () {312 Effect.gen(function* () {
236 const runtimeLocalWorkspace = yield* requireRuntimeLocalWorkspace(input.workspaceId);313 const workspaceContext = yield* ensureRuntimeRecipeWorkspace(
237 const sourceArtifactStore = yield* SourceArtifactStore;314 deps,
238 const source = yield* loadSourceById(input.rows, {315 input.workspaceId,
316 );
317 const source = yield* deps.sourceStore.loadSourceById({
239 workspaceId: input.workspaceId,318 workspaceId: input.workspaceId,
240 sourceId: input.sourceId,319 sourceId: input.sourceId,
241 actorAccountId: input.actorAccountId,320 actorAccountId: input.actorAccountId,
242 });321 });
243 const artifact = yield* sourceArtifactStore.read({322 const artifact = yield* deps.sourceArtifactStore.read({
244 context: runtimeLocalWorkspace.context,323 context: workspaceContext,
245 sourceId: source.id,324 sourceId: source.id,
246 });325 });
247 if (artifact === null) {326 if (artifact === null) {
@@ -253,24 +332,6 @@ export const loadSourceWithRecipe = (input: { · 5 unmodified lines
253 );332 );
254 }333 }
255 334
256 const sourceRecord: StoredSourceRecord = {
257 id: source.id,
258 workspaceId: source.workspaceId,
259 recipeId: artifact.recipeId,
260 recipeRevisionId: artifact.revision.id,
261 name: source.name,
262 kind: source.kind,
263 endpoint: source.endpoint,
264 status: source.status,
265 enabled: source.enabled,
266 namespace: source.namespace,
267 importAuthPolicy: source.importAuthPolicy,
268 bindingConfigJson: getSourceAdapterForSource(source).serializeBindingConfig(source),
269 sourceHash: source.sourceHash,
270 lastError: source.lastError,
271 createdAt: source.createdAt,
272 updatedAt: source.updatedAt,
273 };
274 const manifest = yield* parseManifestForRecipe({335 const manifest = yield* parseManifestForRecipe({
275 source,336 source,
276 revision: artifact.revision,337 revision: artifact.revision,
@@ -278,7 +339,7 @@ export const loadSourceWithRecipe = (input: { · 1 unmodified line
278 339
279 return {340 return {
280 source,341 source,
281 sourceRecord,342 sourceRecord: sourceRecordFromRecipeArtifact({ source, artifact }),
282 revision: artifact.revision,343 revision: artifact.revision,
283 documents: artifact.documents,344 documents: artifact.documents,
284 schemaBundles: artifact.schemaBundles,345 schemaBundles: artifact.schemaBundles,
@@ -287,6 +348,49 @@ export const loadSourceWithRecipe = (input: { · 2 unmodified lines
287 } satisfies LoadedSourceRecipe;348 } satisfies LoadedSourceRecipe;
288 });349 });
289 350
351export const loadWorkspaceSourceRecipes = (input: {
352 workspaceId: WorkspaceId;
353 actorAccountId?: AccountId | null;
354}): Effect.Effect<readonly LoadedSourceRecipe[], Error, SourceRecipeRuntimeServices> =>
355 Effect.gen(function* () {
356 const runtimeLocalWorkspace = yield* RuntimeLocalWorkspaceService;
357 const sourceStore = yield* RuntimeSourceStoreService;
358 const sourceArtifactStore = yield* SourceArtifactStore;
359
360 return yield* loadWorkspaceSourceRecipesWithDeps(
361 {
362 runtimeLocalWorkspace,
363 sourceStore,
364 sourceArtifactStore,
365 },
366 input,
367 );
368 });
369
370export const loadSourceWithRecipe = (input: {
371 workspaceId: WorkspaceId;
372 sourceId: Source["id"];
373 actorAccountId?: AccountId | null;
374}): Effect.Effect<
375 LoadedSourceRecipe,
376 Error | LocalSourceArtifactMissingError,
377 SourceRecipeRuntimeServices
378> =>
379 Effect.gen(function* () {
380 const runtimeLocalWorkspace = yield* RuntimeLocalWorkspaceService;
381 const sourceStore = yield* RuntimeSourceStoreService;
382 const sourceArtifactStore = yield* SourceArtifactStore;
383
384 return yield* loadSourceWithRecipeWithDeps(
385 {
386 runtimeLocalWorkspace,
387 sourceStore,
388 sourceArtifactStore,
389 },
390 input,
391 );
392 });
393
290export const expandRecipeTools = (input: {394export const expandRecipeTools = (input: {
291 recipes: readonly LoadedSourceRecipe[];395 recipes: readonly LoadedSourceRecipe[];
292 includeSchemas: boolean;396 includeSchemas: boolean;
@@ -348,14 +452,16 @@ export const expandRecipeTools = (input: { · 55 unmodified lines
348 );452 );
349 453
350export const loadWorkspaceSourceRecipeToolIndex = (input: {454export const loadWorkspaceSourceRecipeToolIndex = (input: {
351 rows: ControlPlaneStoreShape;
352 workspaceId: WorkspaceId;455 workspaceId: WorkspaceId;
353 actorAccountId?: AccountId | null;456 actorAccountId?: AccountId | null;
354 includeSchemas: boolean;457 includeSchemas: boolean;
355}): Effect.Effect<readonly LoadedSourceRecipeToolIndexEntry[], Error, WorkspaceStorageServices> =>458}): Effect.Effect<
459 readonly LoadedSourceRecipeToolIndexEntry[],
460 Error,
461 SourceRecipeRuntimeServices
462> =>
356 Effect.gen(function* () {463 Effect.gen(function* () {
357 const recipes = yield* loadWorkspaceSourceRecipes({464 const recipes = yield* loadWorkspaceSourceRecipes({
358 rows: input.rows,
359 workspaceId: input.workspaceId,465 workspaceId: input.workspaceId,
360 actorAccountId: input.actorAccountId,466 actorAccountId: input.actorAccountId,
361 });467 });
@@ -377,15 +483,17 @@ export const loadWorkspaceSourceRecipeToolIndex = (input: { · 15 unmodified lines
377 });483 });
378 484
379export const loadWorkspaceSourceRecipeToolByPath = (input: {485export const loadWorkspaceSourceRecipeToolByPath = (input: {
380 rows: ControlPlaneStoreShape;
381 workspaceId: WorkspaceId;486 workspaceId: WorkspaceId;
382 path: string;487 path: string;
383 actorAccountId?: AccountId | null;488 actorAccountId?: AccountId | null;
384 includeSchemas: boolean;489 includeSchemas: boolean;
385}): Effect.Effect<LoadedSourceRecipeToolIndexEntry | null, Error, WorkspaceStorageServices> =>490}): Effect.Effect<
491 LoadedSourceRecipeToolIndexEntry | null,
492 Error,
493 SourceRecipeRuntimeServices
494> =>
386 Effect.gen(function* () {495 Effect.gen(function* () {
387 const recipes = yield* loadWorkspaceSourceRecipes({496 const recipes = yield* loadWorkspaceSourceRecipes({
388 rows: input.rows,
389 workspaceId: input.workspaceId,497 workspaceId: input.workspaceId,
390 actorAccountId: input.actorAccountId,498 actorAccountId: input.actorAccountId,
391 });499 });
@@ -408,3 +516,83 @@ export const loadWorkspaceSourceRecipeToolByPath = (input: { · 16 unmodified lines
408 }516 }
409 : null;517 : null;
410 });518 });
519
520export const RuntimeSourceRecipeStoreLive = Layer.effect(
521 RuntimeSourceRecipeStoreService,
522 Effect.gen(function* () {
523 const runtimeLocalWorkspace = yield* RuntimeLocalWorkspaceService;
524 const sourceStore = yield* RuntimeSourceStoreService;
525 const sourceArtifactStore = yield* SourceArtifactStore;
526
527 const deps: RuntimeSourceRecipeStoreDeps = {
528 runtimeLocalWorkspace,
529 sourceStore,
530 sourceArtifactStore,
531 };
532
533 return RuntimeSourceRecipeStoreService.of({
534 loadWorkspaceSourceRecipes: (input) =>
535 loadWorkspaceSourceRecipesWithDeps(deps, input),
536 loadSourceWithRecipe: (input) =>
537 loadSourceWithRecipeWithDeps(deps, input),
538 loadWorkspaceSourceRecipeToolIndex: (input) =>
539 Effect.gen(function* () {
540 const recipes = yield* loadWorkspaceSourceRecipesWithDeps(deps, input);
541 const tools = yield* expandRecipeTools({
542 recipes,
543 includeSchemas: input.includeSchemas,
544 });
545 return tools.map((tool) => ({
546 path: tool.path,
547 searchNamespace: tool.searchNamespace,
548 searchText: tool.searchText,
549 source: tool.source,
550 sourceRecord: tool.sourceRecord,
551 operation: tool.operation,
552 metadata: tool.metadata,
553 schemaBundleId: tool.schemaBundleId,
554 descriptor: tool.descriptor,
555 }));
556 }),
557 loadWorkspaceSourceRecipeToolByPath: (input) =>
558 Effect.gen(function* () {
559 const recipes = yield* loadWorkspaceSourceRecipesWithDeps(deps, input);
560 const tools = yield* expandRecipeTools({
561 recipes,
562 includeSchemas: input.includeSchemas,
563 });
564 const tool = tools.find((entry) => entry.path === input.path) ?? null;
565 return tool
566 ? {
567 path: tool.path,
568 searchNamespace: tool.searchNamespace,
569 searchText: tool.searchText,
570 source: tool.source,
571 sourceRecord: tool.sourceRecord,
572 operation: tool.operation,
573 metadata: tool.metadata,
574 schemaBundleId: tool.schemaBundleId,
575 descriptor: tool.descriptor,
576 }
577 : null;
578 }),
579 loadWorkspaceSchemaBundle: (input) =>
580 Effect.gen(function* () {
581 const recipes = yield* loadWorkspaceSourceRecipesWithDeps(deps, {
582 workspaceId: input.workspaceId,
583 });
584 const localBundle = recipes
585 .flatMap((recipe) => recipe.schemaBundles)
586 .find((schemaBundle) => schemaBundle.id === input.id);
587 return localBundle
588 ? {
589 id: localBundle.id,
590 kind: localBundle.bundleKind,
591 hash: localBundle.contentHash,
592 refsJson: localBundle.refsJson,
593 }
594 : null;
595 }),
596 });
597 }),
598);
packages/control-plane/src/runtime/source-store.ts+306−55
@@ -9,7 +9,9 @@ import type { · 8 unmodified lines
9 WorkspaceId,9 WorkspaceId,
10} from "#schema";10} from "#schema";
11import { SourceIdSchema } from "#schema";11import { SourceIdSchema } from "#schema";
12import * as Context from "effect/Context";
12import * as Effect from "effect/Effect";13import * as Effect from "effect/Effect";
14import * as Layer from "effect/Layer";
13 15
14import {16import {
15 stableSourceRecipeId,17 stableSourceRecipeId,
@@ -34,6 +36,8 @@ import { · 18 unmodified lines
34} from "./local-errors";36} from "./local-errors";
35import {37import {
36 requireRuntimeLocalWorkspace,38 requireRuntimeLocalWorkspace,
39 RuntimeLocalWorkspaceService,
40 type RuntimeLocalWorkspaceState,
37} from "./local-runtime-context";41} from "./local-runtime-context";
38import type {42import type {
39 SourceArtifactStoreShape,43 SourceArtifactStoreShape,
@@ -56,7 +60,7 @@ import { · 16 unmodified lines
56import { createDefaultSecretMaterialDeleter } from "./secret-material-providers";60import { createDefaultSecretMaterialDeleter } from "./secret-material-providers";
57import { authArtifactSecretMaterialRefs } from "./auth-artifacts";61import { authArtifactSecretMaterialRefs } from "./auth-artifacts";
58import { removeAuthLeaseAndSecrets } from "./auth-leases";62import { removeAuthLeaseAndSecrets } from "./auth-leases";
59import type { ControlPlaneStoreShape } from "./store";63import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";
60import { getSourceAdapter } from "./source-adapters";64import { getSourceAdapter } from "./source-adapters";
61import { slugify } from "./slug";65import { slugify } from "./slug";
62 66
@@ -145,6 +149,27 @@ const deriveLocalSourceId = ( · 82 unmodified lines
145 return SourceIdSchema.make(candidate);149 return SourceIdSchema.make(candidate);
146};150};
147 151
152type RuntimeSourceStoreDeps = {
153 rows: ControlPlaneStoreShape;
154 runtimeLocalWorkspace: RuntimeLocalWorkspaceState;
155 workspaceConfigStore: WorkspaceConfigStoreShape;
156 workspaceStateStore: WorkspaceStateStoreShape;
157 sourceArtifactStore: SourceArtifactStoreShape;
158};
159
160type ResolvedSourceStoreWorkspace = {
161 context: ResolvedLocalWorkspaceContext;
162 installation: {
163 workspaceId: WorkspaceId;
164 accountId: AccountId;
165 };
166 workspaceConfigStore: WorkspaceConfigStoreShape;
167 workspaceStateStore: WorkspaceStateStoreShape;
168 sourceArtifactStore: SourceArtifactStoreShape;
169 loadedConfig: LoadedLocalExecutorConfig;
170 workspaceState: LocalWorkspaceState;
171};
172
148const resolveLocalConfigSecretProviderAlias = (config: LoadedLocalExecutorConfig["config"]): string | null => {173const resolveLocalConfigSecretProviderAlias = (config: LoadedLocalExecutorConfig["config"]): string | null => {
149 const defaultAlias = trimOrNull(config?.secrets?.defaults?.env);174 const defaultAlias = trimOrNull(config?.secrets?.defaults?.env);
150 if (defaultAlias !== null && config?.secrets?.providers?.[defaultAlias]) {175 if (defaultAlias !== null && config?.secrets?.providers?.[defaultAlias]) {
@@ -243,45 +268,71 @@ const configAuthFromSource = (input: { · 92 unmodified lines
243 return input.existingConfigAuth;268 return input.existingConfigAuth;
244};269};
245 270
246const resolveRuntimeLocalWorkspace = (workspaceId: WorkspaceId): Effect.Effect<{271const resolveRuntimeLocalWorkspaceFromDeps = (
247 context: ResolvedLocalWorkspaceContext;272 deps: RuntimeSourceStoreDeps,
248 installation: {273 workspaceId: WorkspaceId,
249 workspaceId: WorkspaceId;274): Effect.Effect<ResolvedSourceStoreWorkspace,
250 accountId: AccountId;275 | RuntimeLocalWorkspaceUnavailableError
251 };276 | RuntimeLocalWorkspaceMismatchError
252 workspaceConfigStore: WorkspaceConfigStoreShape;277 | LocalFileSystemError
253 workspaceStateStore: WorkspaceStateStoreShape;278 | LocalExecutorConfigDecodeError
254 sourceArtifactStore: SourceArtifactStoreShape;279 | LocalWorkspaceStateDecodeError
255 loadedConfig: LoadedLocalExecutorConfig;280 | Error,
256 workspaceState: LocalWorkspaceState;281never> =>
257},282 Effect.gen(function* () {
283 if (deps.runtimeLocalWorkspace.installation.workspaceId !== workspaceId) {
284 return yield* Effect.fail(
285 new RuntimeLocalWorkspaceMismatchError({
286 message: `Runtime local workspace mismatch: expected ${workspaceId}, got ${deps.runtimeLocalWorkspace.installation.workspaceId}`,
287 requestedWorkspaceId: workspaceId,
288 activeWorkspaceId: deps.runtimeLocalWorkspace.installation.workspaceId,
289 }),
290 );
291 }
292
293 const loadedConfig = yield* deps.workspaceConfigStore.load(
294 deps.runtimeLocalWorkspace.context,
295 );
296 const workspaceState = yield* deps.workspaceStateStore.load(
297 deps.runtimeLocalWorkspace.context,
298 );
299
300 return {
301 context: deps.runtimeLocalWorkspace.context,
302 installation: deps.runtimeLocalWorkspace.installation,
303 workspaceConfigStore: deps.workspaceConfigStore,
304 workspaceStateStore: deps.workspaceStateStore,
305 sourceArtifactStore: deps.sourceArtifactStore,
306 loadedConfig,
307 workspaceState,
308 };
309 });
310
311const loadRuntimeSourceStoreDeps = (
312 rows: ControlPlaneStoreShape,
313 workspaceId: WorkspaceId,
314): Effect.Effect<
315 RuntimeSourceStoreDeps,
258 | RuntimeLocalWorkspaceUnavailableError316 | RuntimeLocalWorkspaceUnavailableError
259 | RuntimeLocalWorkspaceMismatchError317 | RuntimeLocalWorkspaceMismatchError
260 | LocalFileSystemError318 | LocalFileSystemError
261 | LocalExecutorConfigDecodeError319 | LocalExecutorConfigDecodeError
262 | LocalWorkspaceStateDecodeError320 | LocalWorkspaceStateDecodeError
263 | Error,321 | Error,
264WorkspaceStorageServices> =>322 WorkspaceStorageServices
323> =>
265 Effect.gen(function* () {324 Effect.gen(function* () {
266 const runtimeLocalWorkspace = yield* requireRuntimeLocalWorkspace(workspaceId);325 const runtimeLocalWorkspace = yield* requireRuntimeLocalWorkspace(workspaceId);
267 const workspaceConfigStore = yield* WorkspaceConfigStore;326 const workspaceConfigStore = yield* WorkspaceConfigStore;
268 const workspaceStateStore = yield* WorkspaceStateStore;327 const workspaceStateStore = yield* WorkspaceStateStore;
269 const sourceArtifactStore = yield* SourceArtifactStore;328 const sourceArtifactStore = yield* SourceArtifactStore;
270 const loadedConfig = yield* workspaceConfigStore.load(
271 runtimeLocalWorkspace.context,
272 );
273 const workspaceState = yield* workspaceStateStore.load(
274 runtimeLocalWorkspace.context,
275 );
276 329
277 return {330 return {
278 context: runtimeLocalWorkspace.context,331 rows,
279 installation: runtimeLocalWorkspace.installation,332 runtimeLocalWorkspace,
280 workspaceConfigStore,333 workspaceConfigStore,
281 workspaceStateStore,334 workspaceStateStore,
282 sourceArtifactStore,335 sourceArtifactStore,
283 loadedConfig,
284 workspaceState,
285 };336 };
286 });337 });
287 338
@@ -396,8 +447,8 @@ const buildLocalSourceRecord = (input: { · 108 unmodified lines
396 };447 };
397 });448 });
398 449
399export const loadSourcesInWorkspace = (450const loadSourcesInWorkspaceWithDeps = (
400 rows: ControlPlaneStoreShape,451 deps: RuntimeSourceStoreDeps,
401 workspaceId: WorkspaceId,452 workspaceId: WorkspaceId,
402 options: {453 options: {
403 actorAccountId?: AccountId | null;454 actorAccountId?: AccountId | null;
@@ -411,11 +462,11 @@ export const loadSourcesInWorkspace = ( · 7 unmodified lines
411 | LocalWorkspaceStateDecodeError462 | LocalWorkspaceStateDecodeError
412 | LocalConfiguredSourceNotFoundError463 | LocalConfiguredSourceNotFoundError
413 | Error,464 | Error,
414 WorkspaceStorageServices465 never
415> =>466> =>
416 Effect.gen(function* () {467 Effect.gen(function* () {
417 const localWorkspace = yield* resolveRuntimeLocalWorkspace(workspaceId);468 const localWorkspace = yield* resolveRuntimeLocalWorkspaceFromDeps(deps, workspaceId);
418 const authArtifacts = yield* rows.authArtifacts.listByWorkspaceId(workspaceId);469 const authArtifacts = yield* deps.rows.authArtifacts.listByWorkspaceId(workspaceId);
419 return yield* Effect.forEach(470 return yield* Effect.forEach(
420 Object.keys(localWorkspace.loadedConfig.config?.sources ?? {}),471 Object.keys(localWorkspace.loadedConfig.config?.sources ?? {}),
421 (sourceId) =>472 (sourceId) =>
@@ -435,7 +486,104 @@ export const loadSourcesInWorkspace = ( · 13 unmodified lines
435 );486 );
436 });487 });
437 488
438export const loadSourceById = (rows: ControlPlaneStoreShape, input: {489export const loadSourcesInWorkspace = (
490 rows: ControlPlaneStoreShape,
491 workspaceId: WorkspaceId,
492 options: {
493 actorAccountId?: AccountId | null;
494 } = {},
495): Effect.Effect<
496 readonly Source[],
497 | RuntimeLocalWorkspaceUnavailableError
498 | RuntimeLocalWorkspaceMismatchError
499 | LocalFileSystemError
500 | LocalExecutorConfigDecodeError
501 | LocalWorkspaceStateDecodeError
502 | LocalConfiguredSourceNotFoundError
503 | Error,
504 WorkspaceStorageServices
505> =>
506 Effect.flatMap(
507 loadRuntimeSourceStoreDeps(rows, workspaceId),
508 (deps) => loadSourcesInWorkspaceWithDeps(deps, workspaceId, options),
509 );
510
511const listLinkedSecretSourcesInWorkspaceWithDeps = (
512 deps: RuntimeSourceStoreDeps,
513 workspaceId: WorkspaceId,
514 options: {
515 actorAccountId?: AccountId | null;
516 } = {},
517): Effect.Effect<
518 Map<string, Array<{ sourceId: string; sourceName: string }>>,
519 | RuntimeLocalWorkspaceUnavailableError
520 | RuntimeLocalWorkspaceMismatchError
521 | LocalFileSystemError
522 | LocalExecutorConfigDecodeError
523 | LocalWorkspaceStateDecodeError
524 | LocalConfiguredSourceNotFoundError
525 | Error,
526 never
527> =>
528 Effect.gen(function* () {
529 const [sources, authArtifacts, materialIds] = yield* Effect.all([
530 loadSourcesInWorkspaceWithDeps(deps, workspaceId, {
531 actorAccountId: options.actorAccountId,
532 }),
533 deps.rows.authArtifacts.listByWorkspaceId(workspaceId),
534 deps.rows.secretMaterials.listAll().pipe(
535 Effect.map((materials) => new Set(materials.map((material) => String(material.id)))),
536 ),
537 ]);
538
539 const sourceNames = new Map(
540 sources.map((source) => [source.id, source.name] as const),
541 );
542 const linkedSources = new Map<string, Array<{ sourceId: string; sourceName: string }>>();
543
544 for (const artifact of authArtifacts) {
545 for (const ref of authArtifactSecretMaterialRefs(artifact)) {
546 if (!materialIds.has(ref.handle)) {
547 continue;
548 }
549
550 const existing = linkedSources.get(ref.handle) ?? [];
551 if (!existing.some((link) => link.sourceId === artifact.sourceId)) {
552 existing.push({
553 sourceId: artifact.sourceId,
554 sourceName: sourceNames.get(artifact.sourceId) ?? artifact.sourceId,
555 });
556 linkedSources.set(ref.handle, existing);
557 }
558 }
559 }
560
561 return linkedSources;
562 });
563
564export const listLinkedSecretSourcesInWorkspace = (
565 rows: ControlPlaneStoreShape,
566 workspaceId: WorkspaceId,
567 options: {
568 actorAccountId?: AccountId | null;
569 } = {},
570): Effect.Effect<
571 Map<string, Array<{ sourceId: string; sourceName: string }>>,
572 | RuntimeLocalWorkspaceUnavailableError
573 | RuntimeLocalWorkspaceMismatchError
574 | LocalFileSystemError
575 | LocalExecutorConfigDecodeError
576 | LocalWorkspaceStateDecodeError
577 | LocalConfiguredSourceNotFoundError
578 | Error,
579 WorkspaceStorageServices
580> =>
581 Effect.flatMap(
582 loadRuntimeSourceStoreDeps(rows, workspaceId),
583 (deps) => listLinkedSecretSourcesInWorkspaceWithDeps(deps, workspaceId, options),
584 );
585
586const loadSourceByIdWithDeps = (deps: RuntimeSourceStoreDeps, input: {
439 workspaceId: WorkspaceId;587 workspaceId: WorkspaceId;
440 sourceId: Source["id"];588 sourceId: Source["id"];
441 actorAccountId?: AccountId | null;589 actorAccountId?: AccountId | null;
@@ -448,11 +596,11 @@ export const loadSourceById = (rows: ControlPlaneStoreShape, input: { · 6 unmodified lines
448 | LocalWorkspaceStateDecodeError596 | LocalWorkspaceStateDecodeError
449 | LocalConfiguredSourceNotFoundError597 | LocalConfiguredSourceNotFoundError
450 | Error,598 | Error,
451 WorkspaceStorageServices599 never
452> =>600> =>
453 Effect.gen(function* () {601 Effect.gen(function* () {
454 const localWorkspace = yield* resolveRuntimeLocalWorkspace(input.workspaceId);602 const localWorkspace = yield* resolveRuntimeLocalWorkspaceFromDeps(deps, input.workspaceId);
455 const authArtifacts = yield* rows.authArtifacts.listByWorkspaceId(input.workspaceId);603 const authArtifacts = yield* deps.rows.authArtifacts.listByWorkspaceId(input.workspaceId);
456 if (!localWorkspace.loadedConfig.config?.sources?.[input.sourceId]) {604 if (!localWorkspace.loadedConfig.config?.sources?.[input.sourceId]) {
457 return yield* Effect.fail(605 return yield* Effect.fail(
458 new LocalConfiguredSourceNotFoundError({606 new LocalConfiguredSourceNotFoundError({
@@ -476,6 +624,26 @@ export const loadSourceById = (rows: ControlPlaneStoreShape, input: { · 17 unmodified lines
476 return localSource.source;624 return localSource.source;
477 });625 });
478 626
627export const loadSourceById = (rows: ControlPlaneStoreShape, input: {
628 workspaceId: WorkspaceId;
629 sourceId: Source["id"];
630 actorAccountId?: AccountId | null;
631}): Effect.Effect<
632 Source,
633 | RuntimeLocalWorkspaceUnavailableError
634 | RuntimeLocalWorkspaceMismatchError
635 | LocalFileSystemError
636 | LocalExecutorConfigDecodeError
637 | LocalWorkspaceStateDecodeError
638 | LocalConfiguredSourceNotFoundError
639 | Error,
640 WorkspaceStorageServices
641> =>
642 Effect.flatMap(
643 loadRuntimeSourceStoreDeps(rows, input.workspaceId),
644 (deps) => loadSourceByIdWithDeps(deps, input),
645 );
646
479const configSourceFromLocalSource = (input: {647const configSourceFromLocalSource = (input: {
480 source: Source;648 source: Source;
481 existingConfigAuth: LocalConfigSecretInput | undefined;649 existingConfigAuth: LocalConfigSecretInput | undefined;
@@ -571,12 +739,12 @@ const removeAuthArtifactsForSource = (rows: ControlPlaneStoreShape, input: { · 89 unmodified lines
571 return existingAuthArtifacts.length;739 return existingAuthArtifacts.length;
572 });740 });
573 741
574export const removeSourceById = (rows: ControlPlaneStoreShape, input: {742const removeSourceByIdWithDeps = (deps: RuntimeSourceStoreDeps, input: {
575 workspaceId: WorkspaceId;743 workspaceId: WorkspaceId;
576 sourceId: Source["id"];744 sourceId: Source["id"];
577}): Effect.Effect<boolean, Error, WorkspaceStorageServices> =>745}): Effect.Effect<boolean, Error, never> =>
578 Effect.gen(function* () {746 Effect.gen(function* () {
579 const localWorkspace = yield* resolveRuntimeLocalWorkspace(input.workspaceId);747 const localWorkspace = yield* resolveRuntimeLocalWorkspaceFromDeps(deps, input.workspaceId);
580 if (!localWorkspace.loadedConfig.config?.sources?.[input.sourceId]) {748 if (!localWorkspace.loadedConfig.config?.sources?.[input.sourceId]) {
581 return false;749 return false;
582 }750 }
@@ -611,28 +779,37 @@ export const removeSourceById = (rows: ControlPlaneStoreShape, input: { · 28 unmodified lines
611 sourceId: input.sourceId,779 sourceId: input.sourceId,
612 });780 });
613 781
614 yield* rows.sourceAuthSessions.removeByWorkspaceAndSourceId(782 yield* deps.rows.sourceAuthSessions.removeByWorkspaceAndSourceId(
615 input.workspaceId,783 input.workspaceId,
616 input.sourceId,784 input.sourceId,
617 );785 );
618 yield* rows.sourceOauthClients.removeByWorkspaceAndSourceId({786 yield* deps.rows.sourceOauthClients.removeByWorkspaceAndSourceId({
619 workspaceId: input.workspaceId,787 workspaceId: input.workspaceId,
620 sourceId: input.sourceId,788 sourceId: input.sourceId,
621 });789 });
622 yield* removeAuthArtifactsForSource(rows, input);790 yield* removeAuthArtifactsForSource(deps.rows, input);
623 791
624 return true;792 return true;
625 });793 });
626 794
627export const persistSource = (795export const removeSourceById = (rows: ControlPlaneStoreShape, input: {
628 rows: ControlPlaneStoreShape,796 workspaceId: WorkspaceId;
797 sourceId: Source["id"];
798}): Effect.Effect<boolean, Error, WorkspaceStorageServices> =>
799 Effect.flatMap(
800 loadRuntimeSourceStoreDeps(rows, input.workspaceId),
801 (deps) => removeSourceByIdWithDeps(deps, input),
802 );
803
804const persistSourceWithDeps = (
805 deps: RuntimeSourceStoreDeps,
629 source: Source,806 source: Source,
630 options: {807 options: {
631 actorAccountId?: AccountId | null;808 actorAccountId?: AccountId | null;
632 } = {},809 } = {},
633): Effect.Effect<Source, Error, WorkspaceStorageServices> =>810): Effect.Effect<Source, Error, never> =>
634 Effect.gen(function* () {811 Effect.gen(function* () {
635 const localWorkspace = yield* resolveRuntimeLocalWorkspace(source.workspaceId);812 const localWorkspace = yield* resolveRuntimeLocalWorkspaceFromDeps(deps, source.workspaceId);
636 const nextSource = {813 const nextSource = {
637 ...source,814 ...source,
638 id:815 id:
@@ -644,7 +821,7 @@ export const persistSource = ( · 5 unmodified lines
644 new Set(Object.keys(localWorkspace.loadedConfig.config?.sources ?? {})),821 new Set(Object.keys(localWorkspace.loadedConfig.config?.sources ?? {})),
645 ),822 ),
646 } satisfies Source;823 } satisfies Source;
647 const existingAuthArtifacts = yield* rows.authArtifacts.listByWorkspaceAndSourceId({824 const existingAuthArtifacts = yield* deps.rows.authArtifacts.listByWorkspaceAndSourceId({
648 workspaceId: nextSource.workspaceId,825 workspaceId: nextSource.workspaceId,
649 sourceId: nextSource.id,826 sourceId: nextSource.id,
650 });827 });
@@ -687,58 +864,58 @@ export const persistSource = ( · 36 unmodified lines
687 864
688 if (runtimeAuthArtifact === null) {865 if (runtimeAuthArtifact === null) {
689 if (existingRuntimeAuthArtifact !== null) {866 if (existingRuntimeAuthArtifact !== null) {
690 yield* removeAuthLeaseAndSecrets(rows, {867 yield* removeAuthLeaseAndSecrets(deps.rows, {
691 authArtifactId: existingRuntimeAuthArtifact.id,868 authArtifactId: existingRuntimeAuthArtifact.id,
692 });869 });
693 }870 }
694 yield* rows.authArtifacts.removeByWorkspaceSourceAndActor({871 yield* deps.rows.authArtifacts.removeByWorkspaceSourceAndActor({
695 workspaceId: nextSource.workspaceId,872 workspaceId: nextSource.workspaceId,
696 sourceId: nextSource.id,873 sourceId: nextSource.id,
697 actorAccountId: options.actorAccountId ?? null,874 actorAccountId: options.actorAccountId ?? null,
698 slot: "runtime",875 slot: "runtime",
699 });876 });
700 } else {877 } else {
701 yield* rows.authArtifacts.upsert(runtimeAuthArtifact);878 yield* deps.rows.authArtifacts.upsert(runtimeAuthArtifact);
702 if (879 if (
703 existingRuntimeAuthArtifact !== null880 existingRuntimeAuthArtifact !== null
704 && existingRuntimeAuthArtifact.id !== runtimeAuthArtifact.id881 && existingRuntimeAuthArtifact.id !== runtimeAuthArtifact.id
705 ) {882 ) {
706 yield* removeAuthLeaseAndSecrets(rows, {883 yield* removeAuthLeaseAndSecrets(deps.rows, {
707 authArtifactId: existingRuntimeAuthArtifact.id,884 authArtifactId: existingRuntimeAuthArtifact.id,
708 });885 });
709 }886 }
710 }887 }
711 888
712 yield* cleanupAuthArtifactSecretRefs(rows, {889 yield* cleanupAuthArtifactSecretRefs(deps.rows, {
713 previous: existingRuntimeAuthArtifact ?? null,890 previous: existingRuntimeAuthArtifact ?? null,
714 next: runtimeAuthArtifact,891 next: runtimeAuthArtifact,
715 });892 });
716 893
717 if (importAuthArtifact === null) {894 if (importAuthArtifact === null) {
718 if (existingImportAuthArtifact !== null) {895 if (existingImportAuthArtifact !== null) {
719 yield* removeAuthLeaseAndSecrets(rows, {896 yield* removeAuthLeaseAndSecrets(deps.rows, {
720 authArtifactId: existingImportAuthArtifact.id,897 authArtifactId: existingImportAuthArtifact.id,
721 });898 });
722 }899 }
723 yield* rows.authArtifacts.removeByWorkspaceSourceAndActor({900 yield* deps.rows.authArtifacts.removeByWorkspaceSourceAndActor({
724 workspaceId: nextSource.workspaceId,901 workspaceId: nextSource.workspaceId,
725 sourceId: nextSource.id,902 sourceId: nextSource.id,
726 actorAccountId: options.actorAccountId ?? null,903 actorAccountId: options.actorAccountId ?? null,
727 slot: "import",904 slot: "import",
728 });905 });
729 } else {906 } else {
730 yield* rows.authArtifacts.upsert(importAuthArtifact);907 yield* deps.rows.authArtifacts.upsert(importAuthArtifact);
731 if (908 if (
732 existingImportAuthArtifact !== null909 existingImportAuthArtifact !== null
733 && existingImportAuthArtifact.id !== importAuthArtifact.id910 && existingImportAuthArtifact.id !== importAuthArtifact.id
734 ) {911 ) {
735 yield* removeAuthLeaseAndSecrets(rows, {912 yield* removeAuthLeaseAndSecrets(deps.rows, {
736 authArtifactId: existingImportAuthArtifact.id,913 authArtifactId: existingImportAuthArtifact.id,
737 });914 });
738 }915 }
739 }916 }
740 917
741 yield* cleanupAuthArtifactSecretRefs(rows, {918 yield* cleanupAuthArtifactSecretRefs(deps.rows, {
742 previous: existingImportAuthArtifact ?? null,919 previous: existingImportAuthArtifact ?? null,
743 next: importAuthArtifact,920 next: importAuthArtifact,
744 });921 });
@@ -762,9 +939,83 @@ export const persistSource = ( · 17 unmodified lines
762 state: workspaceState,939 state: workspaceState,
763 });940 });
764 941
765 return yield* loadSourceById(rows, {942 return yield* loadSourceByIdWithDeps(deps, {
766 workspaceId: nextSource.workspaceId,943 workspaceId: nextSource.workspaceId,
767 sourceId: nextSource.id,944 sourceId: nextSource.id,
768 actorAccountId: options.actorAccountId,945 actorAccountId: options.actorAccountId,
769 });946 });
770 });947 });
948
949export const persistSource = (
950 rows: ControlPlaneStoreShape,
951 source: Source,
952 options: {
953 actorAccountId?: AccountId | null;
954 } = {},
955): Effect.Effect<Source, Error, WorkspaceStorageServices> =>
956 Effect.flatMap(
957 loadRuntimeSourceStoreDeps(rows, source.workspaceId),
958 (deps) => persistSourceWithDeps(deps, source, options),
959 );
960
961type RuntimeSourceStoreShape = {
962 loadSourcesInWorkspace: (
963 workspaceId: WorkspaceId,
964 options?: { actorAccountId?: AccountId | null },
965 ) => ReturnType<typeof loadSourcesInWorkspaceWithDeps>;
966 listLinkedSecretSourcesInWorkspace: (
967 workspaceId: WorkspaceId,
968 options?: { actorAccountId?: AccountId | null },
969 ) => ReturnType<typeof listLinkedSecretSourcesInWorkspaceWithDeps>;
970 loadSourceById: (input: {
971 workspaceId: WorkspaceId;
972 sourceId: Source["id"];
973 actorAccountId?: AccountId | null;
974 }) => ReturnType<typeof loadSourceByIdWithDeps>;
975 removeSourceById: (input: {
976 workspaceId: WorkspaceId;
977 sourceId: Source["id"];
978 }) => ReturnType<typeof removeSourceByIdWithDeps>;
979 persistSource: (
980 source: Source,
981 options?: { actorAccountId?: AccountId | null },
982 ) => ReturnType<typeof persistSourceWithDeps>;
983};
984
985export type RuntimeSourceStore = RuntimeSourceStoreShape;
986
987export class RuntimeSourceStoreService extends Context.Tag(
988 "#runtime/RuntimeSourceStoreService",
989)<RuntimeSourceStoreService, RuntimeSourceStoreShape>() {}
990
991export const RuntimeSourceStoreLive = Layer.effect(
992 RuntimeSourceStoreService,
993 Effect.gen(function* () {
994 const rows = yield* ControlPlaneStore;
995 const runtimeLocalWorkspace = yield* RuntimeLocalWorkspaceService;
996 const workspaceConfigStore = yield* WorkspaceConfigStore;
997 const workspaceStateStore = yield* WorkspaceStateStore;
998 const sourceArtifactStore = yield* SourceArtifactStore;
999
1000 const deps: RuntimeSourceStoreDeps = {
1001 rows,
1002 runtimeLocalWorkspace,
1003 workspaceConfigStore,
1004 workspaceStateStore,
1005 sourceArtifactStore,
1006 };
1007
1008 return RuntimeSourceStoreService.of({
1009 loadSourcesInWorkspace: (workspaceId, options = {}) =>
1010 loadSourcesInWorkspaceWithDeps(deps, workspaceId, options),
1011 listLinkedSecretSourcesInWorkspace: (workspaceId, options = {}) =>
1012 listLinkedSecretSourcesInWorkspaceWithDeps(deps, workspaceId, options),
1013 loadSourceById: (input) =>
1014 loadSourceByIdWithDeps(deps, input),
1015 removeSourceById: (input) =>
1016 removeSourceByIdWithDeps(deps, input),
1017 persistSource: (source, options = {}) =>
1018 persistSourceWithDeps(deps, source, options),
1019 });
1020 }),
1021);
packages/control-plane/src/runtime/sources-operations.ts+22−20
@@ -24,14 +24,10 @@ import { · 23 unmodified lines
24import {24import {
25 operationErrors,25 operationErrors,
26} from "./operation-errors";26} from "./operation-errors";
27import { createDefaultSecretMaterialResolver } from "./secret-material-providers";
28import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";27import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";
29import { syncSourceMaterialization } from "./source-materialization";28import { RuntimeSourceMaterializationService } from "./source-materialization";
30import {29import {
31 loadSourceById,30 RuntimeSourceStoreService,
32 loadSourcesInWorkspace,
33 persistSource,
34 removeSourceById,
35} from "./source-store";31} from "./source-store";
36 32
37const sourceOps = {33const sourceOps = {
@@ -47,6 +43,7 @@ const shouldAutoProbeSource = (source: Source): boolean => · 9 unmodified lines
47 43
48const syncArtifactsForSource = (input: {44const syncArtifactsForSource = (input: {
49 store: ControlPlaneStoreShape;45 store: ControlPlaneStoreShape;
46 sourceStore: Effect.Effect.Success<typeof RuntimeSourceStoreService>;
50 source: Source;47 source: Source;
51 actorAccountId: AccountId;48 actorAccountId: AccountId;
52 operation:49 operation:
@@ -54,9 +51,7 @@ const syncArtifactsForSource = (input: { · 1 unmodified line
54 | typeof sourceOps.update;51 | typeof sourceOps.update;
55}) =>52}) =>
56 Effect.gen(function* () {53 Effect.gen(function* () {
57 const resolveSecretMaterial = createDefaultSecretMaterialResolver({54 const materializationService = yield* RuntimeSourceMaterializationService;
58 rows: input.store,
59 });
60 55
61 // For HTTP-backed source kinds that can validate themselves from a remote56 // For HTTP-backed source kinds that can validate themselves from a remote
62 // document, automatically attempt to probe and connect. This mirrors the57 // document, automatically attempt to probe and connect. This mirrors the
@@ -68,11 +63,9 @@ const syncArtifactsForSource = (input: { · 5 unmodified lines
68 : input.source;63 : input.source;
69 64
70 const synced = yield* Effect.either(65 const synced = yield* Effect.either(
71 syncSourceMaterialization({66 materializationService.sync({
72 rows: input.store,
73 source: sourceForSync,67 source: sourceForSync,
74 actorAccountId: input.actorAccountId,68 actorAccountId: input.actorAccountId,
75 resolveSecretMaterial,
76 }),69 }),
77 );70 );
78 71
@@ -94,7 +87,7 @@ const syncArtifactsForSource = (input: { · 15 unmodified lines
94 );87 );
95 yield* mapPersistenceError(88 yield* mapPersistenceError(
96 input.operation.child("source_connected"),89 input.operation.child("source_connected"),
97 persistSource(input.store, connectedSource, {90 input.sourceStore.persistSource(connectedSource, {
98 actorAccountId: input.actorAccountId,91 actorAccountId: input.actorAccountId,
99 }),92 }),
100 );93 );
@@ -123,7 +116,7 @@ const syncArtifactsForSource = (input: { · 22 unmodified lines
123 116
124 yield* mapPersistenceError(117 yield* mapPersistenceError(
125 input.operation.child("source_error"),118 input.operation.child("source_error"),
126 persistSource(input.store, erroredSource, {119 input.sourceStore.persistSource(erroredSource, {
127 actorAccountId: input.actorAccountId,120 actorAccountId: input.actorAccountId,
128 }),121 }),
129 );122 );
@@ -142,7 +135,9 @@ export const listSources = (input: { · 12 unmodified lines
142}) =>135}) =>
143 Effect.flatMap(ControlPlaneStore, (store) =>136 Effect.flatMap(ControlPlaneStore, (store) =>
144 Effect.gen(function* () {137 Effect.gen(function* () {
145 return yield* loadSourcesInWorkspace(store, input.workspaceId, {138 const sourceStore = yield* RuntimeSourceStoreService;
139
140 return yield* sourceStore.loadSourcesInWorkspace(input.workspaceId, {
146 actorAccountId: input.accountId,141 actorAccountId: input.accountId,
147 }).pipe(142 }).pipe(
148 Effect.mapError((error) =>143 Effect.mapError((error) =>
@@ -161,6 +156,7 @@ export const createSource = (input: { · 12 unmodified lines
161}) =>156}) =>
162 Effect.flatMap(ControlPlaneStore, (store) =>157 Effect.flatMap(ControlPlaneStore, (store) =>
163 Effect.gen(function* () {158 Effect.gen(function* () {
159 const sourceStore = yield* RuntimeSourceStoreService;
164 const now = Date.now();160 const now = Date.now();
165 161
166 const source = yield* createSourceFromPayload({162 const source = yield* createSourceFromPayload({
@@ -179,13 +175,14 @@ export const createSource = (input: { · 12 unmodified lines
179 175
180 const persistedSource = yield* mapPersistenceError(176 const persistedSource = yield* mapPersistenceError(
181 sourceOps.create.child("persist"),177 sourceOps.create.child("persist"),
182 persistSource(store, source, {178 sourceStore.persistSource(source, {
183 actorAccountId: input.accountId,179 actorAccountId: input.accountId,
184 }),180 }),
185 );181 );
186 182
187 const synchronizedSource = yield* syncArtifactsForSource({183 const synchronizedSource = yield* syncArtifactsForSource({
188 store,184 store,
185 sourceStore,
189 source: persistedSource,186 source: persistedSource,
190 actorAccountId: input.accountId,187 actorAccountId: input.accountId,
191 operation: sourceOps.create,188 operation: sourceOps.create,
@@ -201,7 +198,9 @@ export const getSource = (input: { · 9 unmodified lines
201}) =>198}) =>
202 Effect.flatMap(ControlPlaneStore, (store) =>199 Effect.flatMap(ControlPlaneStore, (store) =>
203 Effect.gen(function* () {200 Effect.gen(function* () {
204 return yield* loadSourceById(store, {201 const sourceStore = yield* RuntimeSourceStoreService;
202
203 return yield* sourceStore.loadSourceById({
205 workspaceId: input.workspaceId,204 workspaceId: input.workspaceId,
206 sourceId: input.sourceId,205 sourceId: input.sourceId,
207 actorAccountId: input.accountId,206 actorAccountId: input.accountId,
@@ -228,7 +227,8 @@ export const updateSource = (input: { · 20 unmodified lines
228}) =>227}) =>
229 Effect.flatMap(ControlPlaneStore, (store) =>228 Effect.flatMap(ControlPlaneStore, (store) =>
230 Effect.gen(function* () {229 Effect.gen(function* () {
231 const existingSource = yield* loadSourceById(store, {230 const sourceStore = yield* RuntimeSourceStoreService;
231 const existingSource = yield* sourceStore.loadSourceById({
232 workspaceId: input.workspaceId,232 workspaceId: input.workspaceId,
233 sourceId: input.sourceId,233 sourceId: input.sourceId,
234 actorAccountId: input.accountId,234 actorAccountId: input.accountId,
@@ -261,13 +261,14 @@ export const updateSource = (input: { · 26 unmodified lines
261 261
262 const persistedSource = yield* mapPersistenceError(262 const persistedSource = yield* mapPersistenceError(
263 sourceOps.update.child("persist"),263 sourceOps.update.child("persist"),
264 persistSource(store, updatedSource, {264 sourceStore.persistSource(updatedSource, {
265 actorAccountId: input.accountId,265 actorAccountId: input.accountId,
266 }),266 }),
267 );267 );
268 268
269 const synchronizedSource = yield* syncArtifactsForSource({269 const synchronizedSource = yield* syncArtifactsForSource({
270 store,270 store,
271 sourceStore,
271 source: persistedSource,272 source: persistedSource,
272 actorAccountId: input.accountId,273 actorAccountId: input.accountId,
273 operation: sourceOps.update,274 operation: sourceOps.update,
@@ -282,9 +283,10 @@ export const removeSource = (input: { · 8 unmodified lines
282}) =>283}) =>
283 Effect.flatMap(ControlPlaneStore, (store) =>284 Effect.flatMap(ControlPlaneStore, (store) =>
284 Effect.gen(function* () {285 Effect.gen(function* () {
286 const sourceStore = yield* RuntimeSourceStoreService;
285 const removed = yield* mapPersistenceError(287 const removed = yield* mapPersistenceError(
286 sourceOps.remove.child("remove"),288 sourceOps.remove.child("remove"),
287 removeSourceById(store, {289 sourceStore.removeSourceById({
288 workspaceId: input.workspaceId,290 workspaceId: input.workspaceId,
289 sourceId: input.sourceId,291 sourceId: input.sourceId,
290 }),292 }),
packages/control-plane/src/runtime/store.ts+0−5
@@ -124,11 +124,6 @@ export type ControlPlaneStoreShape = { · 123 unmodified lines
124 never124 never
125 >;125 >;
126 removeById: (id: SecretMaterial["id"]) => Effect.Effect<boolean, Error, never>;126 removeById: (id: SecretMaterial["id"]) => Effect.Effect<boolean, Error, never>;
127 listLinkedSources: () => Effect.Effect<
128 Map<string, Array<{ sourceId: string; sourceName: string }>>,
129 Error,
130 never
131 >;
132 };127 };
133 executions: {128 executions: {
134 getById: (129 getById: (
packages/control-plane/src/runtime/workspace-execution-environment.ts+63−85
@@ -29,19 +29,14 @@ import { · 28 unmodified lines
29 type RuntimeSourceAuthService,29 type RuntimeSourceAuthService,
30} from "./source-auth-service";30} from "./source-auth-service";
31import {31import {
32 loadSourceWithRecipe,32 RuntimeSourceRecipeStoreService,
33 loadWorkspaceSourceRecipeToolByPath,
34 loadWorkspaceSourceRecipeToolIndex,
35 loadWorkspaceSourceRecipes,
36 type LoadedSourceRecipeToolIndexEntry,33 type LoadedSourceRecipeToolIndexEntry,
37 recipeToolCatalogEntry,34 recipeToolCatalogEntry,
38} from "./source-recipes-runtime";35} from "./source-recipes-runtime";
39import { resolveSourceAuthMaterial } from "./source-auth-material";36import { RuntimeSourceAuthMaterialService } from "./source-auth-material";
40import { namespaceFromSourceName } from "./source-names";37import { namespaceFromSourceName } from "./source-names";
41import { getSourceAdapterForOperation } from "./source-adapters";38import { getSourceAdapterForOperation } from "./source-adapters";
42import {39import {
43 createDefaultSecretMaterialResolver,
44 type ResolveSecretMaterial,
45 type SecretMaterialResolveContext,40 type SecretMaterialResolveContext,
46} from "./secret-material-providers";41} from "./secret-material-providers";
47import {42import {
@@ -51,12 +46,17 @@ import { · 3 unmodified lines
51import { loadRuntimeLocalWorkspacePolicies } from "./policies-operations";46import { loadRuntimeLocalWorkspacePolicies } from "./policies-operations";
52import {47import {
53 getRuntimeLocalWorkspaceOption,48 getRuntimeLocalWorkspaceOption,
54 RuntimeLocalWorkspaceService,49 provideOptionalRuntimeLocalWorkspace,
55 type RuntimeLocalWorkspaceState,50 type RuntimeLocalWorkspaceState,
56} from "./local-runtime-context";51} from "./local-runtime-context";
57import { loadLocalToolRuntime, type LocalToolRuntime } from "./local-tools";52import {
53 LocalToolRuntimeLoaderService,
54 type LocalToolRuntime,
55 type LocalToolRuntimeLoaderShape,
56} from "./local-tools";
58import {57import {
59 SourceArtifactStore,58 SourceArtifactStore,
59 makeWorkspaceStorageLayer,
60 type SourceArtifactStoreShape,60 type SourceArtifactStoreShape,
61 WorkspaceConfigStore,61 WorkspaceConfigStore,
62 type WorkspaceConfigStoreShape,62 type WorkspaceConfigStoreShape,
@@ -64,7 +64,6 @@ import { · 1 unmodified line
64 type WorkspaceStateStoreShape,64 type WorkspaceStateStoreShape,
65 type WorkspaceStorageServices,65 type WorkspaceStorageServices,
66} from "./local-storage";66} from "./local-storage";
67import { ControlPlaneStore, type ControlPlaneStoreShape } from "./store";
68 67
69const asToolPath = (value: string): ToolPath => value as ToolPath;68const asToolPath = (value: string): ToolPath => value as ToolPath;
70 69
@@ -147,14 +146,13 @@ const queryTokenWeight = (token: string): number => · 76 unmodified lines
147 LOW_SIGNAL_QUERY_TOKENS.has(token) ? 0.25 : 1;146 LOW_SIGNAL_QUERY_TOKENS.has(token) ? 0.25 : 1;
148 147
149const loadWorkspaceRecipeTools = (input: {148const loadWorkspaceRecipeTools = (input: {
150 rows: ControlPlaneStoreShape;
151 workspaceId: Source["workspaceId"];149 workspaceId: Source["workspaceId"];
152 accountId: AccountId;150 accountId: AccountId;
151 sourceRecipeStore: Effect.Effect.Success<typeof RuntimeSourceRecipeStoreService>;
153 includeSchemas: boolean;152 includeSchemas: boolean;
154}): Effect.Effect<readonly LoadedSourceRecipeToolIndexEntry[], Error, WorkspaceStorageServices> =>153}): Effect.Effect<readonly LoadedSourceRecipeToolIndexEntry[], Error, WorkspaceStorageServices> =>
155 Effect.map(154 Effect.map(
156 loadWorkspaceSourceRecipeToolIndex({155 input.sourceRecipeStore.loadWorkspaceSourceRecipeToolIndex({
157 rows: input.rows,
158 workspaceId: input.workspaceId,156 workspaceId: input.workspaceId,
159 actorAccountId: input.accountId,157 actorAccountId: input.accountId,
160 includeSchemas: input.includeSchemas,158 includeSchemas: input.includeSchemas,
@@ -166,14 +164,13 @@ const loadWorkspaceRecipeTools = (input: { · 5 unmodified lines
166 );164 );
167 165
168const loadWorkspaceRecipeToolByPath = (input: {166const loadWorkspaceRecipeToolByPath = (input: {
169 rows: ControlPlaneStoreShape;
170 workspaceId: Source["workspaceId"];167 workspaceId: Source["workspaceId"];
171 accountId: AccountId;168 accountId: AccountId;
169 sourceRecipeStore: Effect.Effect.Success<typeof RuntimeSourceRecipeStoreService>;
172 path: string;170 path: string;
173 includeSchemas: boolean;171 includeSchemas: boolean;
174}): Effect.Effect<LoadedSourceRecipeToolIndexEntry | null, Error, WorkspaceStorageServices> =>172}): Effect.Effect<LoadedSourceRecipeToolIndexEntry | null, Error, WorkspaceStorageServices> =>
175 loadWorkspaceSourceRecipeToolByPath({173 input.sourceRecipeStore.loadWorkspaceSourceRecipeToolByPath({
176 rows: input.rows,
177 workspaceId: input.workspaceId,174 workspaceId: input.workspaceId,
178 path: input.path,175 path: input.path,
179 actorAccountId: input.accountId,176 actorAccountId: input.accountId,
@@ -187,27 +184,25 @@ const loadWorkspaceRecipeToolByPath = (input: { · 7 unmodified lines
187 );184 );
188 185
189const loadWorkspaceSchemaBundle = (input: {186const loadWorkspaceSchemaBundle = (input: {
190 rows: ControlPlaneStoreShape;
191 workspaceId: Source["workspaceId"];187 workspaceId: Source["workspaceId"];
188 sourceRecipeStore: Effect.Effect.Success<typeof RuntimeSourceRecipeStoreService>;
192 id: SourceRecipeSchemaBundleId;189 id: SourceRecipeSchemaBundleId;
193}): Effect.Effect<ToolSchemaBundle | null, Error, WorkspaceStorageServices> =>190}): Effect.Effect<ToolSchemaBundle | null, Error, WorkspaceStorageServices> =>
194 Effect.gen(function* () {191 Effect.map(
195 const recipes = yield* loadWorkspaceSourceRecipes({192 input.sourceRecipeStore.loadWorkspaceSchemaBundle({
196 rows: input.rows,
197 workspaceId: input.workspaceId,193 workspaceId: input.workspaceId,
198 });194 id: input.id,
199 const localBundle = recipes195 }),
200 .flatMap((recipe) => recipe.schemaBundles)196 (bundle) =>
201 .find((schemaBundle) => schemaBundle.id === input.id);197 bundle
202 return localBundle198 ? {
203 ? {199 id: bundle.id,
204 id: localBundle.id,200 kind: bundle.kind as ToolSchemaBundle["kind"],
205 kind: localBundle.bundleKind,201 hash: bundle.hash,
206 hash: localBundle.contentHash,202 refsJson: bundle.refsJson,
207 refsJson: localBundle.refsJson,203 }
208 }204 : null,
209 : null;205 );
210 });
211 206
212const scoreRecipeTool = (207const scoreRecipeTool = (
213 queryTokens: readonly string[],208 queryTokens: readonly string[],
@@ -363,7 +358,6 @@ const toInvocationDescriptorFromRecipeTool = (input: { · 149 unmodified lines
363});358});
364 359
365const authorizePersistedToolInvocation = (input: {360const authorizePersistedToolInvocation = (input: {
366 rows: ControlPlaneStoreShape;
367 workspaceId: Source["workspaceId"];361 workspaceId: Source["workspaceId"];
368 accountId: AccountId;362 accountId: AccountId;
369 descriptor: InvocationDescriptor;363 descriptor: InvocationDescriptor;
@@ -456,38 +450,31 @@ const provideRuntimeLocalWorkspace = <A, E, R>( · 86 unmodified lines
456 effect: Effect.Effect<A, E, R>,450 effect: Effect.Effect<A, E, R>,
457 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null,451 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null,
458): Effect.Effect<A, E, R> =>452): Effect.Effect<A, E, R> =>
459 runtimeLocalWorkspace === null453 provideOptionalRuntimeLocalWorkspace(effect, runtimeLocalWorkspace);
460 ? effect
461 : effect.pipe(
462 Effect.provideService(
463 RuntimeLocalWorkspaceService,
464 runtimeLocalWorkspace,
465 ),
466 );
467 454
468const createWorkspaceRecipeCatalog = (input: {455const createWorkspaceRecipeCatalog = (input: {
469 workspaceId: Source["workspaceId"];456 workspaceId: Source["workspaceId"];
470 accountId: AccountId;457 accountId: AccountId;
471 rows: ControlPlaneStoreShape;458 sourceRecipeStore: Effect.Effect.Success<typeof RuntimeSourceRecipeStoreService>;
472 workspaceConfigStore: WorkspaceConfigStoreShape;459 workspaceConfigStore: WorkspaceConfigStoreShape;
473 workspaceStateStore: WorkspaceStateStoreShape;460 workspaceStateStore: WorkspaceStateStoreShape;
474 sourceArtifactStore: SourceArtifactStoreShape;461 sourceArtifactStore: SourceArtifactStoreShape;
475 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null;462 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null;
476}): ToolCatalog => {463}): ToolCatalog => {
477 const workspaceStorageLayer = Layer.mergeAll(464 const workspaceStorageLayer = makeWorkspaceStorageLayer({
478 Layer.succeed(WorkspaceConfigStore, input.workspaceConfigStore),465 workspaceConfigStore: input.workspaceConfigStore,
479 Layer.succeed(WorkspaceStateStore, input.workspaceStateStore),466 workspaceStateStore: input.workspaceStateStore,
480 Layer.succeed(SourceArtifactStore, input.sourceArtifactStore),467 sourceArtifactStore: input.sourceArtifactStore,
481 );468 });
482 const provideWorkspaceStorage = <A, E, R>(effect: Effect.Effect<A, E, R>) =>469 const provideWorkspaceStorage = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
483 effect.pipe(Effect.provide(workspaceStorageLayer));470 effect.pipe(Effect.provide(workspaceStorageLayer));
484 471
485 const createSharedCatalog = (includeSchemas: boolean): Effect.Effect<ToolCatalog, Error, never> =>472 const createSharedCatalog = (includeSchemas: boolean): Effect.Effect<ToolCatalog, Error, never> =>
486 provideWorkspaceStorage(Effect.gen(function* () {473 provideWorkspaceStorage(Effect.gen(function* () {
487 const recipeTools = yield* loadWorkspaceRecipeTools({474 const recipeTools = yield* loadWorkspaceRecipeTools({
488 rows: input.rows,
489 workspaceId: input.workspaceId,475 workspaceId: input.workspaceId,
490 accountId: input.accountId,476 accountId: input.accountId,
477 sourceRecipeStore: input.sourceRecipeStore,
491 includeSchemas,478 includeSchemas,
492 });479 });
493 480
@@ -500,8 +487,8 @@ const createWorkspaceRecipeCatalog = (input: { · 6 unmodified lines
500 ),487 ),
501 getSchemaBundle: ({ id }) =>488 getSchemaBundle: ({ id }) =>
502 provideWorkspaceStorage(loadWorkspaceSchemaBundle({489 provideWorkspaceStorage(loadWorkspaceSchemaBundle({
503 rows: input.rows,
504 workspaceId: input.workspaceId,490 workspaceId: input.workspaceId,
491 sourceRecipeStore: input.sourceRecipeStore,
505 id: SourceRecipeSchemaBundleIdSchema.make(id),492 id: SourceRecipeSchemaBundleIdSchema.make(id),
506 })),493 })),
507 });494 });
@@ -562,11 +549,11 @@ const createWorkspaceRecipeCatalog = (input: { · 54 unmodified lines
562const createWorkspaceToolInvoker = (input: {549const createWorkspaceToolInvoker = (input: {
563 workspaceId: Source["workspaceId"];550 workspaceId: Source["workspaceId"];
564 accountId: AccountId;551 accountId: AccountId;
565 rows: ControlPlaneStoreShape;552 sourceRecipeStore: Effect.Effect.Success<typeof RuntimeSourceRecipeStoreService>;
566 workspaceConfigStore: WorkspaceConfigStoreShape;553 workspaceConfigStore: WorkspaceConfigStoreShape;
567 workspaceStateStore: WorkspaceStateStoreShape;554 workspaceStateStore: WorkspaceStateStoreShape;
568 sourceArtifactStore: SourceArtifactStoreShape;555 sourceArtifactStore: SourceArtifactStoreShape;
569 resolveSecretMaterial: ResolveSecretMaterial;556 sourceAuthMaterialService: Effect.Effect.Success<typeof RuntimeSourceAuthMaterialService>;
570 sourceAuthService: RuntimeSourceAuthService;557 sourceAuthService: RuntimeSourceAuthService;
571 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null;558 runtimeLocalWorkspace: RuntimeLocalWorkspaceState | null;
572 localToolRuntime: LocalToolRuntime;559 localToolRuntime: LocalToolRuntime;
@@ -577,11 +564,11 @@ const createWorkspaceToolInvoker = (input: { · 4 unmodified lines
577 catalog: ToolCatalog;564 catalog: ToolCatalog;
578 toolInvoker: ToolInvoker;565 toolInvoker: ToolInvoker;
579} => {566} => {
580 const workspaceStorageLayer = Layer.mergeAll(567 const workspaceStorageLayer = makeWorkspaceStorageLayer({
581 Layer.succeed(WorkspaceConfigStore, input.workspaceConfigStore),568 workspaceConfigStore: input.workspaceConfigStore,
582 Layer.succeed(WorkspaceStateStore, input.workspaceStateStore),569 workspaceStateStore: input.workspaceStateStore,
583 Layer.succeed(SourceArtifactStore, input.sourceArtifactStore),570 sourceArtifactStore: input.sourceArtifactStore,
584 );571 });
585 const provideWorkspaceStorage = <A, E, R>(effect: Effect.Effect<A, E, R>) =>572 const provideWorkspaceStorage = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
586 effect.pipe(Effect.provide(workspaceStorageLayer));573 effect.pipe(Effect.provide(workspaceStorageLayer));
587 574
@@ -594,7 +581,7 @@ const createWorkspaceToolInvoker = (input: { · 6 unmodified lines
594 const recipeCatalog = createWorkspaceRecipeCatalog({581 const recipeCatalog = createWorkspaceRecipeCatalog({
595 workspaceId: input.workspaceId,582 workspaceId: input.workspaceId,
596 accountId: input.accountId,583 accountId: input.accountId,
597 rows: input.rows,584 sourceRecipeStore: input.sourceRecipeStore,
598 workspaceConfigStore: input.workspaceConfigStore,585 workspaceConfigStore: input.workspaceConfigStore,
599 workspaceStateStore: input.workspaceStateStore,586 workspaceStateStore: input.workspaceStateStore,
600 sourceArtifactStore: input.sourceArtifactStore,587 sourceArtifactStore: input.sourceArtifactStore,
@@ -635,9 +622,9 @@ const createWorkspaceToolInvoker = (input: { · 34 unmodified lines
635 provideRuntimeLocalWorkspace(622 provideRuntimeLocalWorkspace(
636 provideWorkspaceStorage(Effect.gen(function* () {623 provideWorkspaceStorage(Effect.gen(function* () {
637 const recipeTool = yield* loadWorkspaceRecipeToolByPath({624 const recipeTool = yield* loadWorkspaceRecipeToolByPath({
638 rows: input.rows,
639 workspaceId: input.workspaceId,625 workspaceId: input.workspaceId,
640 accountId: input.accountId,626 accountId: input.accountId,
627 sourceRecipeStore: input.sourceRecipeStore,
641 path: invocation.path,628 path: invocation.path,
642 includeSchemas: false,629 includeSchemas: false,
643 });630 });
@@ -648,7 +635,6 @@ const createWorkspaceToolInvoker = (input: { · 4 unmodified lines
648 }635 }
649 636
650 yield* authorizePersistedToolInvocation({637 yield* authorizePersistedToolInvocation({
651 rows: input.rows,
652 workspaceId: input.workspaceId,638 workspaceId: input.workspaceId,
653 accountId: input.accountId,639 accountId: input.accountId,
654 descriptor: toInvocationDescriptorFromRecipeTool({640 descriptor: toInvocationDescriptorFromRecipeTool({
@@ -660,24 +646,21 @@ const createWorkspaceToolInvoker = (input: { · 5 unmodified lines
660 onElicitation: input.onElicitation,646 onElicitation: input.onElicitation,
661 });647 });
662 648
663 const auth = yield* resolveSourceAuthMaterial({649 const auth = yield* input.sourceAuthMaterialService.resolve({
664 rows: input.rows,
665 source: recipeTool.source,650 source: recipeTool.source,
666 actorAccountId: input.accountId,651 actorAccountId: input.accountId,
667 resolveSecretMaterial: input.resolveSecretMaterial,
668 context: toSecretResolutionContext(invocation.context),652 context: toSecretResolutionContext(invocation.context),
669 });653 });
670 const schemaBundle = recipeTool.schemaBundleId654 const schemaBundle = recipeTool.schemaBundleId
671 ? yield* loadWorkspaceSchemaBundle({655 ? yield* loadWorkspaceSchemaBundle({
672 rows: input.rows,
673 workspaceId: input.workspaceId,656 workspaceId: input.workspaceId,
657 sourceRecipeStore: input.sourceRecipeStore,
674 id: SourceRecipeSchemaBundleIdSchema.make(658 id: SourceRecipeSchemaBundleIdSchema.make(
675 recipeTool.schemaBundleId,659 recipeTool.schemaBundleId,
676 ),660 ),
677 })661 })
678 : null;662 : null;
679 const recipe = yield* loadSourceWithRecipe({663 const recipe = yield* input.sourceRecipeStore.loadSourceWithRecipe({
680 rows: input.rows,
681 workspaceId: input.workspaceId,664 workspaceId: input.workspaceId,
682 sourceId: recipeTool.source.id,665 sourceId: recipeTool.source.id,
683 actorAccountId: input.accountId,666 actorAccountId: input.accountId,
@@ -686,7 +669,6 @@ const createWorkspaceToolInvoker = (input: { · 2 unmodified lines
686 return yield* getSourceAdapterForOperation(669 return yield* getSourceAdapterForOperation(
687 recipeTool.operation,670 recipeTool.operation,
688 ).invokePersistedTool({671 ).invokePersistedTool({
689 rows: input.rows,
690 workspaceId: input.workspaceId,672 workspaceId: input.workspaceId,
691 accountId: input.accountId,673 accountId: input.accountId,
692 source: recipeTool.source,674 source: recipeTool.source,
@@ -718,20 +700,15 @@ const createWorkspaceToolInvoker = (input: { · 25 unmodified lines
718};700};
719 701
720export const createWorkspaceExecutionEnvironmentResolver = (input: {702export const createWorkspaceExecutionEnvironmentResolver = (input: {
721 rows: ControlPlaneStoreShape;703 sourceAuthMaterialService: Effect.Effect.Success<typeof RuntimeSourceAuthMaterialService>;
722 resolveSecretMaterial?: ResolveSecretMaterial;
723 sourceAuthService: RuntimeSourceAuthService;704 sourceAuthService: RuntimeSourceAuthService;
705 sourceRecipeStore: Effect.Effect.Success<typeof RuntimeSourceRecipeStoreService>;
706 localToolRuntimeLoader: LocalToolRuntimeLoaderShape;
724 workspaceConfigStore: WorkspaceConfigStoreShape;707 workspaceConfigStore: WorkspaceConfigStoreShape;
725 workspaceStateStore: WorkspaceStateStoreShape;708 workspaceStateStore: WorkspaceStateStoreShape;
726 sourceArtifactStore: SourceArtifactStoreShape;709 sourceArtifactStore: SourceArtifactStoreShape;
727}): ResolveExecutionEnvironment => {710}): ResolveExecutionEnvironment =>
728 const resolveSecretMaterial =711 ({ workspaceId, accountId, onElicitation }) =>
729 input.resolveSecretMaterial ??
730 createDefaultSecretMaterialResolver({
731 rows: input.rows,
732 });
733
734 return ({ workspaceId, accountId, onElicitation }) =>
735 Effect.gen(function* () {712 Effect.gen(function* () {
736 const runtimeLocalWorkspace = yield* getRuntimeLocalWorkspaceOption();713 const runtimeLocalWorkspace = yield* getRuntimeLocalWorkspaceOption();
737 const localToolRuntime =714 const localToolRuntime =
@@ -742,15 +719,15 @@ export const createWorkspaceExecutionEnvironmentResolver = (input: { · 4 unmodified lines
742 toolInvoker: makeToolInvokerFromTools({ tools: {} }),719 toolInvoker: makeToolInvokerFromTools({ tools: {} }),
743 toolPaths: new Set<string>(),720 toolPaths: new Set<string>(),
744 }721 }
745 : yield* loadLocalToolRuntime(runtimeLocalWorkspace.context);722 : yield* input.localToolRuntimeLoader.load(runtimeLocalWorkspace.context);
746 const { catalog, toolInvoker } = createWorkspaceToolInvoker({723 const { catalog, toolInvoker } = createWorkspaceToolInvoker({
747 workspaceId,724 workspaceId,
748 accountId,725 accountId,
749 rows: input.rows,726 sourceRecipeStore: input.sourceRecipeStore,
750 workspaceConfigStore: input.workspaceConfigStore,727 workspaceConfigStore: input.workspaceConfigStore,
751 workspaceStateStore: input.workspaceStateStore,728 workspaceStateStore: input.workspaceStateStore,
752 sourceArtifactStore: input.sourceArtifactStore,729 sourceArtifactStore: input.sourceArtifactStore,
753 resolveSecretMaterial,730 sourceAuthMaterialService: input.sourceAuthMaterialService,
754 sourceAuthService: input.sourceAuthService,731 sourceAuthService: input.sourceAuthService,
755 runtimeLocalWorkspace,732 runtimeLocalWorkspace,
756 localToolRuntime,733 localToolRuntime,
@@ -765,7 +742,6 @@ export const createWorkspaceExecutionEnvironmentResolver = (input: { · 8 unmodified lines
765 catalog,742 catalog,
766 } satisfies ExecutionEnvironment;743 } satisfies ExecutionEnvironment;
767 });744 });
768};
769 745
770export class RuntimeExecutionResolverService extends Context.Tag(746export class RuntimeExecutionResolverService extends Context.Tag(
771 "#runtime/RuntimeExecutionResolverService",747 "#runtime/RuntimeExecutionResolverService",
@@ -777,7 +753,6 @@ export class RuntimeExecutionResolverService extends Context.Tag( · 5 unmodified lines
777export const RuntimeExecutionResolverLive = (753export const RuntimeExecutionResolverLive = (
778 input: {754 input: {
779 executionResolver?: ResolveExecutionEnvironment;755 executionResolver?: ResolveExecutionEnvironment;
780 resolveSecretMaterial?: ResolveSecretMaterial;
781 } = {},756 } = {},
782) =>757) =>
783 input.executionResolver758 input.executionResolver
@@ -785,16 +760,19 @@ export const RuntimeExecutionResolverLive = ( · 1 unmodified line
785 : Layer.effect(760 : Layer.effect(
786 RuntimeExecutionResolverService,761 RuntimeExecutionResolverService,
787 Effect.gen(function* () {762 Effect.gen(function* () {
788 const rows = yield* ControlPlaneStore;763 const sourceAuthMaterialService = yield* RuntimeSourceAuthMaterialService;
789 const sourceAuthService = yield* RuntimeSourceAuthServiceTag;764 const sourceAuthService = yield* RuntimeSourceAuthServiceTag;
765 const sourceRecipeStore = yield* RuntimeSourceRecipeStoreService;
766 const localToolRuntimeLoader = yield* LocalToolRuntimeLoaderService;
790 const workspaceConfigStore = yield* WorkspaceConfigStore;767 const workspaceConfigStore = yield* WorkspaceConfigStore;
791 const workspaceStateStore = yield* WorkspaceStateStore;768 const workspaceStateStore = yield* WorkspaceStateStore;
792 const sourceArtifactStore = yield* SourceArtifactStore;769 const sourceArtifactStore = yield* SourceArtifactStore;
793 770
794 return createWorkspaceExecutionEnvironmentResolver({771 return createWorkspaceExecutionEnvironmentResolver({
795 rows,
796 sourceAuthService,772 sourceAuthService,
797 resolveSecretMaterial: input.resolveSecretMaterial,773 sourceAuthMaterialService,
774 sourceRecipeStore,
775 localToolRuntimeLoader,
798 workspaceConfigStore,776 workspaceConfigStore,
799 workspaceStateStore,777 workspaceStateStore,
800 sourceArtifactStore,778 sourceArtifactStore,