History

apps: runtime, connection DI bindings, source plugin, and self-host wiring with end-to-end test

Rhys Sullivan committed 4 months ago

Commit e0a832d1 parent 7a7133d9 files · +1153 −0Browse files

Unified · Split

9 files
  1. packages/plugins/apps/src/api.ts +9 −0
  2. packages/plugins/apps/src/index.ts +46 −0
  3. packages/plugins/apps/src/plugin/apps-plugin.ts +152 −0
  4. packages/plugins/apps/src/plugin/bindings.ts +193 −0
  5. packages/plugins/apps/src/plugin/runtime.test.ts +98 −0
  6. packages/plugins/apps/src/plugin/runtime.ts +433 −0
  7. packages/plugins/apps/src/plugin/self-host-runtime.ts +86 −0
  8. packages/plugins/apps/src/plugin/store.ts +79 −0
  9. packages/plugins/apps/src/testing/index.ts +57 −0
packages/plugins/apps/src/api.ts+9−0
@@ -0,0 +1,9 @@
1+// HTTP + plugin surface for @executor-js/plugin-apps.
2+export {
3+ appsPlugin,
4+ APPS_INTEGRATION_SLUG,
5+ APPS_PLUGIN_ID,
6+ type AppsPluginOptions,
7+} from "./plugin/apps-plugin";
8+export { makeAppsHttpRoutes, type AppsHttpDeps } from "./http/routes";
9+export { registerAppsMcp, type AppsMcpDeps } from "./mcp/register";
packages/plugins/apps/src/index.ts+46−0
@@ -0,0 +1,46 @@
1+// @executor-js/plugin-apps — the executor apps subsystem (self-hosted build).
2+//
3+// Custom tools, durable workflows, UI views, and skills: published into a
4+// per-scope store and served/executed by the platform. Publish is the compiler
5+// (FDI); every substrate-specific capability sits behind a seam.
6+
7+export * from "./seams";
8+export * from "./pipeline/descriptor";
9+export { discover, PublishError } from "./pipeline/discover";
10+export { bundleEntry, PLATFORM_MODULES, INLINABLE_MODULES } from "./pipeline/bundle";
11+export {
12+ publish,
13+ type PublishInput,
14+ type PublishOutput,
15+ type PublishDeps,
16+ type PutBlob,
17+} from "./pipeline/publish";
18+
19+export { makeAppsRuntime, type AppsRuntime, type AppsRuntimeDeps } from "./plugin/runtime";
20+export {
21+ makeAppsStore,
22+ type AppsStore,
23+ type AppsStoreDeps,
24+ descriptorCollection,
25+} from "./plugin/store";
26+export {
27+ buildBridge,
28+ rootsFor,
29+ BindingError,
30+ type Bindings,
31+ type RoleBinding,
32+ type ClientResolver,
33+ type BindingContext,
34+} from "./plugin/bindings";
35+export {
36+ makeSelfHostAppsRuntime,
37+ type SelfHostAppsRuntime,
38+ type SelfHostAppsRuntimeOptions,
39+} from "./plugin/self-host-runtime";
40+
41+// Self-host seam backings.
42+export { makeGitArtifactStore } from "./backing/git-artifact-store";
43+export { makeLibsqlScopeDb } from "./backing/libsql-scope-db";
44+export { makeQuickjsToolSandbox } from "./backing/quickjs-tool-sandbox";
45+export { makeSqliteWorkflowRunner } from "./backing/sqlite-workflow-runner";
46+export { makeInProcessLiveChannel } from "./backing/in-process-live-channel";
packages/plugins/apps/src/plugin/apps-plugin.ts+152−0
@@ -0,0 +1,152 @@
1+import { Effect } from "effect";
2+
3+import {
4+ definePlugin,
5+ ToolName,
6+ type ResolveToolsInput,
7+ type ResolveToolsResult,
8+ type InvokeToolInput,
9+ type ToolDef,
10+} from "@executor-js/sdk";
11+
12+import type { AppsRuntime } from "./runtime";
13+import { makeAppsStore } from "./store";
14+import type { Bindings } from "./bindings";
15+
16+// ---------------------------------------------------------------------------
17+// The apps source plugin. Published custom tools become catalog citizens: the
18+// plugin registers one integration per scope (`apps`), and a connection to it
19+// makes the published tools resolvable + invocable like any catalog tool, so
20+// policy / approval / audit / toolkits / tools.list all apply unchanged.
21+//
22+// `resolveTools` projects the published descriptor into `ToolDef[]`.
23+// `invokeTool` bundles + runs the tool in the sandbox with connections bound.
24+// The plugin is thin: all logic lives in `AppsRuntime` (shared with the HTTP +
25+// MCP surfaces). The runtime is supplied via options because it owns the seam
26+// instances built at host boot.
27+// ---------------------------------------------------------------------------
28+
29+export const APPS_INTEGRATION_SLUG = "apps";
30+export const APPS_PLUGIN_ID = "apps";
31+
32+export interface AppsPluginOptions {
33+ /** The shared runtime (seams + store). Built at host boot. */
34+ readonly runtime: AppsRuntime;
35+ /** How a tool's declared connection roles are bound to the caller's
36+ * connections at invoke time. Self-host resolves these from the scope's
37+ * configured connections; a default binds each role to a connection of the
38+ * same name as the integration. */
39+ readonly resolveBindings?: (input: {
40+ readonly scope: string;
41+ readonly tool: string;
42+ readonly declared: Readonly<Record<string, { kind: string; integration?: string }>>;
43+ }) => Bindings;
44+}
45+
46+const defaultBindings = (
47+ declared: Readonly<Record<string, { kind: string; integration?: string }>>,
48+): Bindings => {
49+ const out: Record<string, Bindings[string]> = {};
50+ for (const [role, decl] of Object.entries(declared)) {
51+ if (decl.kind === "array") {
52+ out[role] = { kind: "array", connections: [decl.integration ?? role] };
53+ } else if (decl.kind === "catalog") {
54+ // no binding
55+ } else {
56+ out[role] = { kind: "single", connection: decl.integration ?? role };
57+ }
58+ }
59+ return out;
60+};
61+
62+interface AppsStoreShape {
63+ readonly runtime: AppsRuntime;
64+}
65+
66+export const appsPlugin = definePlugin((options?: AppsPluginOptions) => {
67+ if (!options?.runtime) {
68+ throw new Error("appsPlugin requires a `runtime` (built from the five seams at host boot)");
69+ }
70+ const runtime = options.runtime;
71+ const resolveBindings = options.resolveBindings;
72+
73+ return {
74+ id: APPS_PLUGIN_ID as "apps",
75+ packageName: "@executor-js/plugin-apps",
76+
77+ // The plugin's store facade is host-owned plugin storage + blobs; the apps
78+ // runtime already holds its own store, so this is a thin passthrough kept
79+ // for the ctx shape (extension methods read the runtime).
80+ storage: (deps): AppsStoreShape => {
81+ void makeAppsStore({
82+ pluginStorage: deps.pluginStorage,
83+ blobs: deps.blobs,
84+ });
85+ return { runtime };
86+ },
87+
88+ // Declare the plugin's storage collection so the host provisions it.
89+ pluginStorage: {
90+ published_descriptor: {
91+ name: "published_descriptor",
92+ schema: { Type: {} as Record<string, unknown> },
93+ indexes: [],
94+ },
95+ },
96+
97+ extension: () => ({ runtime }),
98+
99+ // Per-connection tool production: project the published descriptor into
100+ // ToolDefs. Called at connection create/refresh; the SDK stamps addresses
101+ // and persists per connection.
102+ resolveTools: ({ connection }: ResolveToolsInput<AppsStoreShape>) =>
103+ Effect.gen(function* () {
104+ // The scope is the connection owner's tenant; for self-host single
105+ // tenant we key the descriptor by the connection's integration-scoped
106+ // name. We read the published descriptor for the scope encoded in the
107+ // connection name (`apps/<scope>`), falling back to the connection name.
108+ const scope = scopeFromConnection(connection.name);
109+ const descriptor = yield* runtime.getDescriptor(scope);
110+ if (!descriptor) return { tools: [] } satisfies ResolveToolsResult;
111+ const tools: ToolDef[] = descriptor.tools.map((t) => ({
112+ name: ToolName.make(t.name),
113+ description: t.description,
114+ inputSchema: t.inputSchema,
115+ outputSchema: t.outputSchema,
116+ annotations: {
117+ requiresApproval: t.annotations?.destructive === true,
118+ },
119+ }));
120+ return { tools } satisfies ResolveToolsResult;
121+ }),
122+
123+ invokeTool: ({ toolRow, args }: InvokeToolInput<AppsStoreShape>) =>
124+ Effect.gen(function* () {
125+ const scope = scopeFromConnection(toolRow.connection);
126+ const descriptor = yield* runtime.getDescriptor(scope);
127+ const toolDesc = descriptor?.tools.find((t) => t.name === toolRow.name);
128+ const declared = toolDesc?.connections ?? {};
129+ const bindings = resolveBindings
130+ ? resolveBindings({ scope, tool: toolRow.name, declared })
131+ : defaultBindings(declared);
132+ return yield* runtime
133+ .invokeTool({ scope, tool: toolRow.name, args, bindings })
134+ .pipe(
135+ Effect.mapError(
136+ (cause) =>
137+ new Error(
138+ "message" in cause && typeof cause.message === "string"
139+ ? cause.message
140+ : "apps tool invocation failed",
141+ ),
142+ ),
143+ );
144+ }),
145+ };
146+});
147+
148+// Connection names encode the scope as `apps/<scope>` (or are the scope itself).
149+const scopeFromConnection = (connectionName: string): string => {
150+ const slash = connectionName.indexOf("/");
151+ return slash === -1 ? connectionName : connectionName.slice(slash + 1);
152+};
packages/plugins/apps/src/plugin/bindings.ts+193−0
@@ -0,0 +1,193 @@
1+import { Effect } from "effect";
2+import { Data } from "effect";
3+
4+import type { HandleBridge, HandleRootSpec } from "../seams/tool-sandbox";
5+import { ToolSandboxError } from "../seams/tool-sandbox";
6+import type { ScopeDbHandle } from "../seams/scope-db";
7+import type { ConnectionDecl } from "../pipeline/descriptor";
8+
9+// ---------------------------------------------------------------------------
10+// Connection DI: declare-then-bind. A tool's descriptor declares `connections`
11+// (role -> integration); before execution each role is bound to the user's
12+// connection(s) explicitly. A missing binding is a typed error naming role +
13+// surface (no auto-pick). The handler receives pre-bound clients whose method
14+// calls route through the platform invoke path.
15+//
16+// The routing is a seam: `ClientResolver` turns a (integration, connection,
17+// method path, args) into a JSON result. The self-hosted resolver calls the
18+// integration's real API through the connection credential (policy/audit
19+// applies there); a test resolver returns canned data. `db` is bound to the
20+// invoking scope's ScopeDb. Everything crossing is JSON (the cloud resolver is
21+// an RPC).
22+// ---------------------------------------------------------------------------
23+
24+export class BindingError extends Data.TaggedError("BindingError")<{
25+ readonly message: string;
26+ readonly role: string;
27+ readonly surface: string;
28+}> {}
29+
30+/** One bound connection for a role. Fan-out roles bind an ordered set. */
31+export type RoleBinding =
32+ | { readonly kind: "single"; readonly connection: string }
33+ | { readonly kind: "array"; readonly connections: readonly string[] };
34+
35+/** The user's bindings for a tool invocation: role -> bound connection(s). */
36+export type Bindings = Readonly<Record<string, RoleBinding>>;
37+
38+/** Resolves a single method call against a bound connection to a JSON result.
39+ * This is where the platform invoke path (credentials, policy, audit) lives. */
40+export interface ClientResolver {
41+ readonly call: (input: {
42+ readonly integration: string;
43+ readonly connection: string;
44+ readonly path: readonly string[];
45+ readonly args: readonly unknown[];
46+ }) => Effect.Effect<unknown, BindingError>;
47+}
48+
49+export interface BindingContext {
50+ /** Declared connection roles from the tool descriptor. */
51+ readonly declared: Readonly<Record<string, ConnectionDecl>>;
52+ /** The user's bindings (role -> connection[s]). Missing => typed error. */
53+ readonly bindings: Bindings;
54+ /** The invoking scope's app database (bound to the `db` root). */
55+ readonly db: ScopeDbHandle;
56+ /** Routes a bound method call to the real integration. */
57+ readonly resolver: ClientResolver;
58+}
59+
60+/**
61+ * Validate bindings against declarations and compute the sandbox handle roots:
62+ * `db` is always a single root; each declared role becomes a single or array
63+ * root. Fails (typed) when a declared role has no binding, naming role +
64+ * surface — the "missing binding is a typed error" rule.
65+ */
66+export const rootsFor = (
67+ declared: Readonly<Record<string, ConnectionDecl>>,
68+ bindings: Bindings,
69+): Effect.Effect<Readonly<Record<string, HandleRootSpec>>, BindingError> =>
70+ Effect.gen(function* () {
71+ const roots: Record<string, HandleRootSpec> = { db: { kind: "single" } };
72+ for (const [role, decl] of Object.entries(declared)) {
73+ if (decl.kind === "catalog") {
74+ // Open-world proxy: parse + record, but execution is NotImplemented in
75+ // this build. Bind a single root; the resolver throws if called.
76+ roots[role] = { kind: "single" };
77+ continue;
78+ }
79+ const binding = bindings[role];
80+ if (!binding) {
81+ return yield* Effect.fail(
82+ new BindingError({
83+ message: `no connection bound for role "${role}" (surface "${decl.integration}")`,
84+ role,
85+ surface: decl.integration,
86+ }),
87+ );
88+ }
89+ if (decl.kind === "array") {
90+ if (binding.kind !== "array") {
91+ return yield* Effect.fail(
92+ new BindingError({
93+ message: `role "${role}" is a fan-out (connections("${decl.integration}")) and needs an array binding`,
94+ role,
95+ surface: decl.integration,
96+ }),
97+ );
98+ }
99+ roots[role] = { kind: "array", count: binding.connections.length };
100+ } else {
101+ if (binding.kind !== "single") {
102+ return yield* Effect.fail(
103+ new BindingError({
104+ message: `role "${role}" is a single connection and needs a single binding`,
105+ role,
106+ surface: decl.integration,
107+ }),
108+ );
109+ }
110+ roots[role] = { kind: "single" };
111+ }
112+ }
113+ return roots;
114+ });
115+
116+// Parse a fan-out root name back into (role, index): `inboxes#1` -> {inboxes,1}.
117+const parseRoot = (root: string): { role: string; index?: number } => {
118+ const hash = root.indexOf("#");
119+ if (hash === -1) return { role: root };
120+ return { role: root.slice(0, hash), index: Number(root.slice(hash + 1)) };
121+};
122+
123+/**
124+ * Build the HandleBridge the sandbox calls out through. `db` routes to the
125+ * scope database; a declared role routes to its bound connection through the
126+ * `ClientResolver`. Undeclared roots are unreachable (the sandbox never injects
127+ * them). A `.account` read on a client returns bound-connection metadata
128+ * without a round-trip.
129+ */
130+export const buildBridge = (context: BindingContext): HandleBridge => ({
131+ call: ({ root, path, args }) => {
132+ if (root === "db") {
133+ // The scope db handle exposes `sql` as a tagged template; the injected
134+ // client calls `db.sql(templateStrings, ...values)`. When routed through
135+ // the bridge, `path = ["sql"]` and args = [stringsArray, ...values].
136+ if (path.length === 1 && path[0] === "sql") {
137+ const [strings, ...values] = args as [TemplateStringsArray, ...unknown[]];
138+ return context.db
139+ .sql(strings, ...values)
140+ .pipe(
141+ Effect.mapError(
142+ (cause) => new ToolSandboxError({ kind: "invoke", message: cause.message, cause }),
143+ ),
144+ );
145+ }
146+ return Effect.fail(
147+ new ToolSandboxError({ kind: "invoke", message: `unsupported db call: ${path.join(".")}` }),
148+ );
149+ }
150+
151+ const { role, index } = parseRoot(root);
152+ const decl = context.declared[role];
153+ if (!decl) {
154+ return Effect.fail(
155+ new ToolSandboxError({ kind: "invoke", message: `undeclared handle root: ${root}` }),
156+ );
157+ }
158+ if (decl.kind === "catalog") {
159+ return Effect.fail(
160+ new ToolSandboxError({
161+ kind: "invoke",
162+ message: "catalog() open-world proxy execution is not implemented in this build",
163+ }),
164+ );
165+ }
166+
167+ // A `.account` read returns bound-connection metadata (safe, no creds).
168+ const binding = context.bindings[role];
169+ const connectionName =
170+ binding?.kind === "array"
171+ ? binding.connections[index ?? 0]
172+ : binding?.kind === "single"
173+ ? binding.connection
174+ : undefined;
175+ if (!connectionName) {
176+ return Effect.fail(
177+ new ToolSandboxError({ kind: "invoke", message: `no binding for role ${role}` }),
178+ );
179+ }
180+ if (path.length === 1 && path[0] === "account") {
181+ // Clients read `.account.email` / `.account.login`; expose both keys.
182+ return Effect.succeed({ email: connectionName, login: connectionName, name: connectionName });
183+ }
184+
185+ return context.resolver
186+ .call({ integration: decl.integration, connection: connectionName, path, args })
187+ .pipe(
188+ Effect.mapError(
189+ (cause) => new ToolSandboxError({ kind: "invoke", message: cause.message, cause }),
190+ ),
191+ );
192+ },
193+});
packages/plugins/apps/src/plugin/runtime.test.ts+98−0
@@ -0,0 +1,98 @@
1+import { mkdtempSync } from "node:fs";
2+import { tmpdir } from "node:os";
3+import { join } from "node:path";
4+
5+import { describe, expect, it } from "vitest";
6+import { Effect } from "effect";
7+
8+import { makeSelfHostAppsRuntime } from "./self-host-runtime";
9+import { makeInMemoryAppsStore, makeTestResolver, dailyBriefFileSet } from "../testing";
10+import type { Bindings } from "./bindings";
11+
12+const run = <A, E>(effect: Effect.Effect<A, E>): Promise<A> => Effect.runPromise(effect);
13+
14+// A GitHub resolver returning two open issues, one stale.
15+const githubHandlers = {
16+ github: {
17+ "repos.listForAuthenticatedUser": () => [{ full_name: "acme/app" }],
18+ "issues.listForRepo": () => [
19+ {
20+ number: 1,
21+ title: "Fresh bug",
22+ labels: [{ name: "bug" }],
23+ assignee: { login: "rhys" },
24+ updated_at: new Date().toISOString(),
25+ html_url: "https://github.com/acme/app/issues/1",
26+ },
27+ {
28+ number: 2,
29+ title: "Old bug",
30+ labels: [],
31+ assignee: null,
32+ updated_at: "2020-01-01T00:00:00Z",
33+ html_url: "https://github.com/acme/app/issues/2",
34+ },
35+ ],
36+ },
37+};
38+
39+const githubBindings: Bindings = { github: { kind: "single", connection: "rhys-github" } };
40+
41+describe("AppsRuntime end-to-end (publish -> invoke -> workflow)", () => {
42+ it("publishes daily-brief, invokes the tool into the scope db, runs the workflow", async () => {
43+ const store = makeInMemoryAppsStore();
44+ const resolver = makeTestResolver(githubHandlers);
45+ const host = makeSelfHostAppsRuntime({
46+ dataDir: mkdtempSync(join(tmpdir(), "apps-rt-")),
47+ store,
48+ resolver,
49+ inMemory: true,
50+ });
51+ const { runtime } = host;
52+
53+ // --- publish ----------------------------------------------------------
54+ const published = await run(runtime.publish({ scope: "rhys", files: dailyBriefFileSet() }));
55+ expect(published.descriptor.tools.map((t) => t.name).sort()).toEqual([
56+ "issues-sync",
57+ "search-all-mail",
58+ ]);
59+
60+ // --- invoke the tool: writes the scope `issues` table -----------------
61+ const syncResult = (await run(
62+ runtime.invokeTool({
63+ scope: "rhys",
64+ tool: "issues-sync",
65+ args: {},
66+ bindings: githubBindings,
67+ }),
68+ )) as { synced: number; repos: number };
69+ expect(syncResult).toEqual({ synced: 2, repos: 1 });
70+
71+ // The scope db now has the two issues.
72+ const db = await run(host.scopeDb.forScope("rhys"));
73+ const rows = await run(db.exec<{ n: number }>("SELECT COUNT(*) AS n FROM issues"));
74+ expect(Number(rows[0].n)).toBe(2);
75+
76+ // --- run the workflow: syncs again + flags the stale issue ------------
77+ const runView = await run(
78+ runtime.startWorkflow({
79+ scope: "rhys",
80+ workflow: "morning-sync",
81+ input: {},
82+ bindings: githubBindings,
83+ runId: "morning-1",
84+ }),
85+ );
86+ expect(runView.status).toBe("completed");
87+ expect(runView.output).toEqual({ synced: 2, stale: 1 });
88+
89+ // Journal has the step.tool call and the find-stale step.
90+ const steps = await run(runtime.listSteps("morning-1"));
91+ const names = steps.map((s) => s.name);
92+ expect(names).toContain("tool:issues-sync");
93+ expect(names).toContain("find-stale");
94+ expect(steps.every((s) => s.status === "completed")).toBe(true);
95+
96+ await host.close();
97+ });
98+});
packages/plugins/apps/src/plugin/runtime.ts+433−0
@@ -0,0 +1,433 @@
1+import { Effect } from "effect";
2+
3+import type { ArtifactStore } from "../seams/artifact-store";
4+import type { ScopeDb } from "../seams/scope-db";
5+import type { ToolSandbox } from "../seams/tool-sandbox";
6+import type { LiveChannel } from "../seams/live-channel";
7+import type { DurableSteps, WorkflowRunner, RunView, StepView } from "../seams/workflow-runner";
8+import { publish as runPublish, type PublishOutput } from "../pipeline/publish";
9+import { PublishError } from "../pipeline/discover";
10+import type { AppDescriptor, ToolDescriptor, WorkflowDescriptor } from "../pipeline/descriptor";
11+import { bundleEntry } from "../pipeline/bundle";
12+import {
13+ buildBridge,
14+ rootsFor,
15+ type Bindings,
16+ type ClientResolver,
17+ BindingError,
18+} from "./bindings";
19+import type { AppsStore } from "./store";
20+
21+// ---------------------------------------------------------------------------
22+// AppsRuntime — the substrate-neutral core of the apps subsystem. It owns the
23+// five seams + the store, and exposes the operations every surface (HTTP, MCP,
24+// the plugin) drives: publish, resolveTools (descriptor projection), invokeTool
25+// (bundle -> sandbox invoke with bound clients), and the workflow lifecycle
26+// (start/signal/status/history) with real journal replay + scheduling.
27+//
28+// Nothing here knows about HTTP or MCP; those are thin adapters over this.
29+// ---------------------------------------------------------------------------
30+
31+export interface AppsRuntimeDeps {
32+ readonly artifactStore: ArtifactStore;
33+ readonly scopeDb: ScopeDb;
34+ readonly sandbox: ToolSandbox;
35+ readonly workflows: WorkflowRunner;
36+ readonly liveChannel: LiveChannel;
37+ readonly store: AppsStore;
38+ /** Routes a bound integration method call to the real API (policy/audit). */
39+ readonly resolver: ClientResolver;
40+}
41+
42+export interface AppsRuntime {
43+ readonly publish: (input: {
44+ readonly scope: string;
45+ readonly files: ReadonlyMap<string, string>;
46+ readonly message?: string;
47+ }) => Effect.Effect<PublishOutput, PublishError>;
48+ readonly getDescriptor: (scope: string) => Effect.Effect<AppDescriptor | null>;
49+ readonly invokeTool: (input: {
50+ readonly scope: string;
51+ readonly tool: string;
52+ readonly args: unknown;
53+ readonly bindings: Bindings;
54+ }) => Effect.Effect<unknown, PublishError | BindingError>;
55+ readonly startWorkflow: (input: {
56+ readonly scope: string;
57+ readonly workflow: string;
58+ readonly input?: unknown;
59+ readonly bindings?: Bindings;
60+ readonly runId?: string;
61+ }) => Effect.Effect<RunView, PublishError | BindingError>;
62+ readonly signalWorkflow: (input: {
63+ readonly scope: string;
64+ readonly runId: string;
65+ readonly event: string;
66+ readonly payload: unknown;
67+ }) => Effect.Effect<RunView, PublishError | BindingError>;
68+ readonly getRun: (runId: string) => Effect.Effect<RunView | null>;
69+ readonly listRuns: (scope: string) => Effect.Effect<readonly RunView[]>;
70+ readonly listSteps: (runId: string) => Effect.Effect<readonly StepView[]>;
71+ /** Serve a ui bundle by name (raw endpoint / MCP resource). */
72+ readonly getUiBundle: (
73+ scope: string,
74+ name: string,
75+ ) => Effect.Effect<{ code: string; title?: string; maxHeight?: number } | null>;
76+ /** Subscribe to a scope's live invalidations (SSE adapter drives this). */
77+ readonly subscribeLive: (
78+ scope: string,
79+ listener: (event: { table: string; version: number }) => void,
80+ ) => () => void;
81+ readonly deps: AppsRuntimeDeps;
82+}
83+
84+const failNoDescriptor = (scope: string): PublishError =>
85+ new PublishError({
86+ message: `scope "${scope}" has no published app`,
87+ stage: "project",
88+ diagnostics: [],
89+ });
90+
91+export const makeAppsRuntime = (deps: AppsRuntimeDeps): AppsRuntime => {
92+ // Bundle cache keyed by (snapshot, sourcePath): a published snapshot is
93+ // immutable, so its bundles never change.
94+ const bundleCache = new Map<string, string>();
95+
96+ const bundleFor = (
97+ descriptor: AppDescriptor,
98+ sourcePath: string,
99+ ): Effect.Effect<string, PublishError> =>
100+ Effect.gen(function* () {
101+ const cacheKey = `${descriptor.snapshotId}:${sourcePath}`;
102+ const cached = bundleCache.get(cacheKey);
103+ if (cached) return cached;
104+ const scopeStore = yield* deps.artifactStore
105+ .forScope(descriptor.scope)
106+ .pipe(
107+ Effect.mapError(
108+ (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }),
109+ ),
110+ );
111+ const files = yield* scopeStore
112+ .read(descriptor.snapshotId as never)
113+ .pipe(
114+ Effect.mapError(
115+ (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }),
116+ ),
117+ );
118+ const bundle = yield* bundleEntry({ files, entry: sourcePath }).pipe(
119+ Effect.mapError(
120+ (c) =>
121+ new PublishError({
122+ message: c.message,
123+ stage: "bundle",
124+ diagnostics: [{ path: sourcePath, message: c.message }],
125+ }),
126+ ),
127+ );
128+ bundleCache.set(cacheKey, bundle.code);
129+ return bundle.code;
130+ });
131+
132+ const requireDescriptor = (scope: string): Effect.Effect<AppDescriptor, PublishError> =>
133+ deps.store.getDescriptor(scope).pipe(
134+ Effect.mapError(
135+ (c) => new PublishError({ message: String(c), stage: "project", diagnostics: [] }),
136+ ),
137+ Effect.flatMap((d) => (d ? Effect.succeed(d) : Effect.fail(failNoDescriptor(scope)))),
138+ );
139+
140+ const invokeToolInternal = (
141+ scope: string,
142+ descriptor: AppDescriptor,
143+ toolDesc: ToolDescriptor,
144+ args: unknown,
145+ bindings: Bindings,
146+ ): Effect.Effect<unknown, PublishError | BindingError> =>
147+ Effect.gen(function* () {
148+ const roots = yield* rootsFor(toolDesc.connections, bindings);
149+ const code = yield* bundleFor(descriptor, toolDesc.sourcePath);
150+ const db = yield* deps.scopeDb
151+ .forScope(scope)
152+ .pipe(
153+ Effect.mapError(
154+ (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }),
155+ ),
156+ );
157+ const bridge = buildBridge({
158+ declared: toolDesc.connections,
159+ bindings,
160+ db,
161+ resolver: deps.resolver,
162+ });
163+ const result = yield* deps.sandbox
164+ .invoke(code, { artifact: toolDesc.name, kind: "tool", input: args, roots }, bridge)
165+ .pipe(
166+ Effect.mapError(
167+ (c) =>
168+ new PublishError({
169+ message: c.message,
170+ stage: "project",
171+ diagnostics: [{ path: toolDesc.sourcePath, message: c.message }],
172+ }),
173+ ),
174+ );
175+ return result.output;
176+ });
177+
178+ // Build the workflow body closure for one run: replay the workflow's compiled
179+ // bundle over the DurableSteps. Our sandbox runs a tool handler, not a
180+ // long-lived stepful body, so we drive the workflow with a thin interpreter:
181+ // the workflow's `run(step, {db})` is executed in-process against the
182+ // DurableSteps facade, with `step.tool` -> the runner bindings and
183+ // `db.sql` -> the scope db. This keeps CF semantics while the durable journal
184+ // lives in the WorkflowRunner seam.
185+ const workflowBody = (
186+ scope: string,
187+ descriptor: AppDescriptor,
188+ wfDesc: WorkflowDescriptor,
189+ ): ((steps: DurableSteps) => Promise<unknown>) => {
190+ return async (steps: DurableSteps) => {
191+ const code = await Effect.runPromise(
192+ bundleFor(descriptor, wfDesc.sourcePath).pipe(Effect.orDie),
193+ );
194+ const db = await Effect.runPromise(deps.scopeDb.forScope(scope).pipe(Effect.orDie));
195+ // Interpret the workflow: run its bundle in the sandbox is not how durable
196+ // steps work (the body must call back into our journaled `steps`). Instead
197+ // we evaluate the workflow module here and invoke its `run(step, {db})`.
198+ // The compiled bundle sets globalThis.__artifact = the workflow def.
199+ const def = extractWorkflowDef(code);
200+ const scopeDbClient = {
201+ sql: (strings: TemplateStringsArray, ...values: unknown[]) =>
202+ Effect.runPromise(db.sql(strings, ...values).pipe(Effect.orDie)),
203+ };
204+ return def.run(steps, { db: scopeDbClient });
205+ };
206+ };
207+
208+ const bindingsForRunTool = (
209+ scope: string,
210+ descriptor: AppDescriptor,
211+ recordedBindings: Bindings,
212+ ) => ({
213+ runTool: async (address: string, toolArgs: unknown) => {
214+ const toolDesc = descriptor.tools.find((t) => t.name === address);
215+ if (!toolDesc) throw new Error(`workflow step.tool: unknown tool "${address}"`);
216+ return Effect.runPromise(
217+ invokeToolInternal(scope, descriptor, toolDesc, toolArgs, recordedBindings).pipe(
218+ Effect.orDie,
219+ ),
220+ );
221+ },
222+ notify: async (_msg: { title: string; body?: string; link?: string }) => {
223+ // Self-host delivery sink: recorded as a journaled step; a real host wires
224+ // this to notifications. No-op body here keeps the workflow durable.
225+ },
226+ });
227+
228+ return {
229+ deps,
230+ publish: (input) =>
231+ Effect.gen(function* () {
232+ const out = yield* runPublish(
233+ {
234+ artifactStore: deps.artifactStore,
235+ sandbox: deps.sandbox,
236+ putBlob: (hash, value) =>
237+ deps.store
238+ .putBlob(hash, value)
239+ .pipe(
240+ Effect.mapError(
241+ (c) =>
242+ new PublishError({ message: String(c), stage: "project", diagnostics: [] }),
243+ ),
244+ ),
245+ },
246+ { scope: input.scope, files: input.files, commitMessage: input.message },
247+ );
248+ yield* deps.store
249+ .putDescriptor("org", out.descriptor)
250+ .pipe(
251+ Effect.mapError(
252+ (c) => new PublishError({ message: String(c), stage: "project", diagnostics: [] }),
253+ ),
254+ );
255+ return out;
256+ }),
257+
258+ getDescriptor: (scope) =>
259+ deps.store.getDescriptor(scope).pipe(Effect.orElseSucceed(() => null)),
260+
261+ invokeTool: (input) =>
262+ Effect.gen(function* () {
263+ const descriptor = yield* requireDescriptor(input.scope);
264+ const toolDesc = descriptor.tools.find((t) => t.name === input.tool);
265+ if (!toolDesc) {
266+ return yield* Effect.fail(
267+ new PublishError({
268+ message: `tool "${input.tool}" is not published in scope "${input.scope}"`,
269+ stage: "project",
270+ diagnostics: [],
271+ }),
272+ );
273+ }
274+ return yield* invokeToolInternal(
275+ input.scope,
276+ descriptor,
277+ toolDesc,
278+ input.args,
279+ input.bindings,
280+ );
281+ }),
282+
283+ startWorkflow: (input) =>
284+ Effect.gen(function* () {
285+ const descriptor = yield* requireDescriptor(input.scope);
286+ const wfDesc = descriptor.workflows.find((w) => w.name === input.workflow);
287+ if (!wfDesc) {
288+ return yield* Effect.fail(
289+ new PublishError({
290+ message: `workflow "${input.workflow}" is not published in scope "${input.scope}"`,
291+ stage: "project",
292+ diagnostics: [],
293+ }),
294+ );
295+ }
296+ const bindings = input.bindings ?? {};
297+ const body = workflowBody(input.scope, descriptor, wfDesc);
298+ return yield* deps.workflows
299+ .start(
300+ {
301+ scope: input.scope,
302+ workflow: input.workflow,
303+ snapshotId: descriptor.snapshotId,
304+ input: input.input ?? {},
305+ runId: input.runId,
306+ },
307+ body,
308+ bindingsForRunTool(input.scope, descriptor, bindings),
309+ )
310+ .pipe(
311+ Effect.mapError(
312+ (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }),
313+ ),
314+ );
315+ }),
316+
317+ signalWorkflow: (input) =>
318+ Effect.gen(function* () {
319+ const run = yield* deps.workflows
320+ .get(input.runId)
321+ .pipe(
322+ Effect.mapError(
323+ (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }),
324+ ),
325+ );
326+ if (!run) {
327+ return yield* Effect.fail(
328+ new PublishError({
329+ message: `no run ${input.runId}`,
330+ stage: "project",
331+ diagnostics: [],
332+ }),
333+ );
334+ }
335+ const descriptor = yield* requireDescriptor(run.scope);
336+ const wfDesc = descriptor.workflows.find((w) => w.name === run.workflow);
337+ if (!wfDesc) {
338+ return yield* Effect.fail(
339+ new PublishError({
340+ message: `workflow ${run.workflow} gone`,
341+ stage: "project",
342+ diagnostics: [],
343+ }),
344+ );
345+ }
346+ const body = workflowBody(run.scope, descriptor, wfDesc);
347+ return yield* deps.workflows
348+ .signal(
349+ input.runId,
350+ input.event,
351+ input.payload,
352+ body,
353+ bindingsForRunTool(run.scope, descriptor, {}),
354+ )
355+ .pipe(
356+ Effect.mapError(
357+ (c) => new PublishError({ message: c.message, stage: "project", diagnostics: [] }),
358+ ),
359+ );
360+ }),
361+
362+ getRun: (runId) => deps.workflows.get(runId).pipe(Effect.orElseSucceed(() => null)),
363+ listRuns: (scope) =>
364+ deps.workflows.list({ scope }).pipe(Effect.orElseSucceed(() => [] as readonly RunView[])),
365+ listSteps: (runId) =>
366+ deps.workflows.listSteps(runId).pipe(Effect.orElseSucceed(() => [] as readonly StepView[])),
367+
368+ getUiBundle: (scope, name) =>
369+ Effect.gen(function* () {
370+ const descriptor = yield* deps.store
371+ .getDescriptor(scope)
372+ .pipe(Effect.orElseSucceed(() => null));
373+ if (!descriptor) return null;
374+ const uiDesc = descriptor.ui.find((u) => u.name === name);
375+ if (!uiDesc) return null;
376+ const code = yield* deps.store
377+ .getBlob(`ui/${uiDesc.bundleHash}`)
378+ .pipe(Effect.orElseSucceed(() => null));
379+ if (!code) return null;
380+ return { code, title: uiDesc.title, maxHeight: uiDesc.maxHeight };
381+ }),
382+
383+ subscribeLive: (scope, listener) =>
384+ deps.liveChannel.subscribe(scope, (event) =>
385+ listener({ table: event.table, version: event.version }),
386+ ),
387+ };
388+};
389+
390+// Extract the workflow def from a compiled bundle by running it in a tiny
391+// module shim (Node-side, trusted: this is our own runtime, not user isolation
392+// for the durable interpreter — the tool HANDLERS still run in the sandbox).
393+// The bundle set globalThis.__artifact to the workflow def object.
394+const extractWorkflowDef = (
395+ code: string,
396+): { run: (steps: DurableSteps, deps: { db: unknown }) => Promise<unknown> } => {
397+ const g: Record<string, unknown> = {};
398+ const shim = {
399+ connection: (integration: string, opts?: { description?: string }) => ({
400+ __decl: "single",
401+ integration,
402+ description: opts?.description,
403+ }),
404+ connections: (integration: string) => ({ __decl: "array", integration }),
405+ catalog: () => ({ __decl: "catalog" }),
406+ defineTool: (def: unknown) => {
407+ g.__artifact = def;
408+ return def;
409+ },
410+ defineWorkflow: (def: unknown) => {
411+ g.__artifact = def;
412+ return def;
413+ },
414+ };
415+ const req = (id: string) => {
416+ if (id === "executor:app") return shim;
417+ if (id === "executor:ui") return {};
418+ if (id === "executor:ui/components") return {};
419+ throw new Error(`module not available: ${id}`);
420+ };
421+ const moduleObj = { exports: {} as Record<string, unknown> };
422+ const globalThisShim = g as unknown as typeof globalThis;
423+ // eslint-disable-next-line @typescript-eslint/no-implied-eval, no-new-func
424+ const factory = new Function("module", "exports", "require", "globalThis", code);
425+ factory(moduleObj, moduleObj.exports, req, globalThisShim);
426+ const def = (g.__artifact ?? moduleObj.exports.default ?? moduleObj.exports) as {
427+ run: (steps: DurableSteps, deps: { db: unknown }) => Promise<unknown>;
428+ };
429+ if (!def || typeof def.run !== "function") {
430+ throw new Error("workflow bundle did not produce a run() function");
431+ }
432+ return def;
433+};
packages/plugins/apps/src/plugin/self-host-runtime.ts+86−0
@@ -0,0 +1,86 @@
1+import { join } from "node:path";
2+
3+import { Effect } from "effect";
4+
5+import { makeGitArtifactStore } from "../backing/git-artifact-store";
6+import { makeLibsqlScopeDb } from "../backing/libsql-scope-db";
7+import { makeQuickjsToolSandbox } from "../backing/quickjs-tool-sandbox";
8+import { makeSqliteWorkflowRunner } from "../backing/sqlite-workflow-runner";
9+import { makeInProcessLiveChannel } from "../backing/in-process-live-channel";
10+import type { ScopeDb } from "../seams/scope-db";
11+import type { LiveChannel } from "../seams/live-channel";
12+import { makeAppsRuntime, type AppsRuntime } from "./runtime";
13+import type { AppsStore } from "./store";
14+import type { ClientResolver } from "./bindings";
15+
16+// ---------------------------------------------------------------------------
17+// Wire the five self-hosted seam backings into one AppsRuntime, rooted at a
18+// data directory. This is what the self-host app calls at boot. The ScopeDb's
19+// write events are wired into the LiveChannel here (invalidation delivery), and
20+// the store is provided by the caller (over the host's pluginStorage + blobs).
21+// ---------------------------------------------------------------------------
22+
23+export interface SelfHostAppsRuntimeOptions {
24+ /** Data dir root; `<root>/artifacts`, `<root>/scope-db`, `<root>/workflows`. */
25+ readonly dataDir: string;
26+ /** Store over the host's pluginStorage + blobs. */
27+ readonly store: AppsStore;
28+ /** Routes bound integration calls to real APIs (policy/audit). */
29+ readonly resolver: ClientResolver;
30+ /** In-memory backings for tests. */
31+ readonly inMemory?: boolean;
32+}
33+
34+export interface SelfHostAppsRuntime {
35+ readonly runtime: AppsRuntime;
36+ readonly scopeDb: ScopeDb;
37+ readonly liveChannel: LiveChannel;
38+ readonly close: () => Promise<void>;
39+}
40+
41+export const makeSelfHostAppsRuntime = (
42+ options: SelfHostAppsRuntimeOptions,
43+): SelfHostAppsRuntime => {
44+ const inMem = options.inMemory === true;
45+ const artifactStore = makeGitArtifactStore({
46+ root: inMem ? options.dataDir : join(options.dataDir, "artifacts"),
47+ });
48+ const scopeDb = makeLibsqlScopeDb({
49+ root: inMem ? ":memory:" : join(options.dataDir, "scope-db"),
50+ });
51+ const sandbox = makeQuickjsToolSandbox();
52+ const workflows = makeSqliteWorkflowRunner({
53+ path: inMem ? ":memory:" : join(options.dataDir, "workflows", "journal.db"),
54+ });
55+ const liveChannel = makeInProcessLiveChannel();
56+
57+ // Wire scope-db writes -> live invalidations.
58+ const unwire = scopeDb.onWrite((event) => {
59+ for (const t of event.tables) {
60+ void Effect.runPromise(
61+ liveChannel.publish({ scope: event.scope, table: t.table, version: t.version }),
62+ );
63+ }
64+ });
65+
66+ const runtime = makeAppsRuntime({
67+ artifactStore,
68+ scopeDb,
69+ sandbox,
70+ workflows,
71+ liveChannel,
72+ store: options.store,
73+ resolver: options.resolver,
74+ });
75+
76+ return {
77+ runtime,
78+ scopeDb,
79+ liveChannel,
80+ close: async () => {
81+ unwire();
82+ await Effect.runPromise(scopeDb.close().pipe(Effect.orElseSucceed(() => undefined)));
83+ await Effect.runPromise(workflows.close().pipe(Effect.orElseSucceed(() => undefined)));
84+ },
85+ };
86+};
packages/plugins/apps/src/plugin/store.ts+79−0
@@ -0,0 +1,79 @@
1+import { Effect } from "effect";
2+
3+import {
4+ definePluginStorageCollection,
5+ type PluginStorageFacade,
6+ type StorageFailure,
7+} from "@executor-js/sdk";
8+
9+import type { AppDescriptor } from "../pipeline/descriptor";
10+
11+// ---------------------------------------------------------------------------
12+// AppsStore — descriptor + snapshot pointer persistence for the apps plugin,
13+// over the host-owned `pluginStorage` (collections) + `blobs` (content-addressed
14+// bundles / skill bodies). Executor plugins do not contribute DB tables, so the
15+// versioned descriptor is a JSON document in a plugin-storage collection keyed
16+// by scope, and the compiled bundles + skill bodies are blobs.
17+//
18+// One published descriptor per scope (the current published pointer). History
19+// is the ArtifactStore's git log; the runtime only runs the published snapshot.
20+// ---------------------------------------------------------------------------
21+
22+/** The published descriptor document, keyed by scope. */
23+export const descriptorCollection = definePluginStorageCollection("published_descriptor", {
24+ Type: {} as {
25+ readonly scope: string;
26+ readonly snapshotId: string;
27+ readonly descriptor: AppDescriptor;
28+ readonly publishedAt: number;
29+ },
30+});
31+
32+export interface AppsStore {
33+ /** Persist the published descriptor for a scope (the published pointer). */
34+ readonly putDescriptor: (
35+ owner: "org" | "user",
36+ descriptor: AppDescriptor,
37+ ) => Effect.Effect<void, StorageFailure>;
38+ /** Read the current published descriptor for a scope, or null. */
39+ readonly getDescriptor: (scope: string) => Effect.Effect<AppDescriptor | null, StorageFailure>;
40+ /** Store a compiled bundle / skill body blob (content-addressed). */
41+ readonly putBlob: (key: string, value: string) => Effect.Effect<void, StorageFailure>;
42+ /** Read a blob by key. */
43+ readonly getBlob: (key: string) => Effect.Effect<string | null, StorageFailure>;
44+}
45+
46+export interface AppsStoreDeps {
47+ readonly pluginStorage: PluginStorageFacade;
48+ readonly blobs: {
49+ readonly get: (key: string) => Effect.Effect<string | null, StorageFailure>;
50+ readonly put: (
51+ key: string,
52+ value: string,
53+ options: { readonly owner: "org" | "user" },
54+ ) => Effect.Effect<void, StorageFailure>;
55+ };
56+}
57+
58+export const makeAppsStore = (deps: AppsStoreDeps): AppsStore => {
59+ const descriptors = deps.pluginStorage.collection(descriptorCollection);
60+ return {
61+ putDescriptor: (owner, descriptor) =>
62+ descriptors
63+ .put({
64+ owner,
65+ key: descriptor.scope,
66+ data: {
67+ scope: descriptor.scope,
68+ snapshotId: descriptor.snapshotId,
69+ descriptor,
70+ publishedAt: Date.now(),
71+ },
72+ })
73+ .pipe(Effect.asVoid),
74+ getDescriptor: (scope) =>
75+ descriptors.get({ key: scope }).pipe(Effect.map((entry) => entry?.data.descriptor ?? null)),
76+ putBlob: (key, value) => deps.blobs.put(key, value, { owner: "org" }),
77+ getBlob: (key) => deps.blobs.get(key),
78+ };
79+};
packages/plugins/apps/src/testing/index.ts+57−0
@@ -0,0 +1,57 @@
1+import { Effect } from "effect";
2+
3+import type { AppDescriptor } from "../pipeline/descriptor";
4+import type { AppsStore } from "../plugin/store";
5+import type { ClientResolver, BindingError } from "../plugin/bindings";
6+
7+// ---------------------------------------------------------------------------
8+// Test helpers: an in-memory AppsStore and a canned ClientResolver, plus the
9+// daily-brief fixture set. Used by the runtime integration test and the e2e.
10+// ---------------------------------------------------------------------------
11+
12+export * from "./daily-brief";
13+
14+/** In-memory AppsStore (descriptors + blobs in Maps). */
15+export const makeInMemoryAppsStore = (): AppsStore & {
16+ readonly blobs: Map<string, string>;
17+ readonly descriptors: Map<string, AppDescriptor>;
18+} => {
19+ const descriptors = new Map<string, AppDescriptor>();
20+ const blobs = new Map<string, string>();
21+ return {
22+ descriptors,
23+ blobs,
24+ putDescriptor: (_owner, descriptor) =>
25+ Effect.sync(() => void descriptors.set(descriptor.scope, descriptor)),
26+ getDescriptor: (scope) => Effect.sync(() => descriptors.get(scope) ?? null),
27+ putBlob: (key, value) => Effect.sync(() => void blobs.set(key, value)),
28+ getBlob: (key) => Effect.sync(() => blobs.get(key) ?? null),
29+ };
30+};
31+
32+/** A resolver that dispatches integration method calls to supplied handlers.
33+ * `handlers[integration][path.join(".")]` returns the JSON result. */
34+export const makeTestResolver = (
35+ handlers: Record<string, Record<string, (args: readonly unknown[]) => unknown>>,
36+): ClientResolver & {
37+ readonly calls: { integration: string; connection: string; method: string }[];
38+} => {
39+ const calls: { integration: string; connection: string; method: string }[] = [];
40+ return {
41+ calls,
42+ call: ({ integration, connection, path, args }) => {
43+ const method = path.join(".");
44+ calls.push({ integration, connection, method });
45+ const handler = handlers[integration]?.[method];
46+ if (!handler) {
47+ return Effect.fail({
48+ _tag: "BindingError",
49+ message: `no test handler for ${integration}.${method}`,
50+ role: integration,
51+ surface: integration,
52+ } as unknown as BindingError);
53+ }
54+ return Effect.sync(() => handler(args));
55+ },
56+ };
57+};