History

executor v1.5: integrations and connections rework

Rhys Sullivan committed 5 months ago

Commit a30f26b1 parent bec7b7b497 files · +41191 −81846Browse files

Showing the changes to examples/all-plugins/src (1 of 497 files). Show all

Unified · Split

examples/all-plugins/src/main.ts+184−123
@@ -1,67 +1,90 @@
11 // ---------------------------------------------------------------------------
22 // examples/all-plugins
33 //
4−// Wires every ported plugin into a single Executor and walks through the
5−// common flows: secrets, static control tools, dynamic source registration,
6−// tool invocation, filtered listing, and shutdown.
4+// Wires every ported plugin into a single Executor and walks the common v2
5+// flows: credential providers, integration registration, connection creation
6+// (a connection IS the credential), per-connection tool production, execution,
7+// filtered listing, and shutdown.
78 //
8−// This is what an app/local or app/cloud bootstrap file looks like under
9−// the new SDK shape — minus the HTTP API layer, runtime lifecycle, and
10−// scope persistence that real apps add on top.
9+// This is what an app/local or app/cloud bootstrap file looks like under the
10+// v2 SDK shape — minus the HTTP API layer, runtime lifecycle, and owner/tenant
11+// persistence that real apps add on top.
1112 //
1213 // Runs against the SDK's ephemeral in-memory FumaDB backend so you can
13−// `bun run src/main.ts` and watch the whole surface exercise itself.
14−// Plugins that need external infra (keychain prompts, 1Password unlock,
15−// MCP transport, WorkOS Vault, Google OAuth) are wired so their secret
16−// providers and extensions exist, but the flows that hit their
17−// backends are gated behind env vars and skipped by default.
14+// `bun run src/main.ts` and watch the whole surface exercise itself. Plugins
15+// that need external infra (keychain prompts, 1Password unlock, MCP transport,
16+// WorkOS Vault, Google OAuth) are wired so their credential providers and
17+// extensions exist, but the flows that hit their backends are skipped by
18+// default.
1819 // ---------------------------------------------------------------------------
1920
20−import { Cause, Effect } from "effect";
21+import { Cause, Effect, Result } from "effect";
2122
22−import { SecretId, Scope, ScopeId, SetSecretInput, createExecutor } from "@executor-js/sdk";
23+import {
24+ AuthTemplateSlug,
25+ ConnectionName,
26+ createExecutor,
27+ IntegrationSlug,
28+ ProviderItemId,
29+ ProviderKey,
30+ Tenant,
31+ ToolAddress,
32+ type CredentialProvider,
33+} from "@executor-js/sdk";
2334
2435 import { fileSecretsPlugin } from "@executor-js/plugin-file-secrets";
2536 import { graphqlPlugin } from "@executor-js/plugin-graphql";
2637 import { keychainPlugin } from "@executor-js/plugin-keychain";
2738 import { mcpPlugin } from "@executor-js/plugin-mcp";
2839 import { onepasswordPlugin } from "@executor-js/plugin-onepassword";
29−import { openApiPlugin } from "@executor-js/plugin-openapi";
40+import { openApiPlugin, variable } from "@executor-js/plugin-openapi";
3041 import { workosVaultPlugin } from "@executor-js/plugin-workos-vault";
3142
3243 // ---------------------------------------------------------------------------
3344 // 1. Build the ExecutorConfig.
3445 //
35−// Three pieces only: scope, FumaDB, plugins.
36−// Compare to the old SDK, where you'd pass pre-built ToolRegistry,
37−// SourceRegistry, SecretStore, and PolicyEngine service instances.
46+// Three pieces: tenant, plugins, and credential providers. The executor
47+// auto-registers every `plugin.credentialProviders`; `config.providers` adds
48+// inline ones (registered first, so they win as the default writable store).
49+// Compare to v1, where you'd pass pre-built ToolRegistry, SourceRegistry,
50+// SecretStore, and PolicyEngine service instances plus a scope stack.
3851 // ---------------------------------------------------------------------------
3952
40−const scope = Scope.make({
41− id: ScopeId.make("example-scope"),
42− name: "/tmp/example-workspace",
43− createdAt: new Date(),
44−});
53+// A connection's value lives in a writable credential provider. This tiny
54+// in-memory store is enough for a script; the keychain / file-secrets /
55+// 1Password plugins below contribute durable ones. Providers are Effect-native,
56+// so `get`/`set` return `Effect`s.
57+const memory = new Map<string, string>();
58+const memoryProvider: CredentialProvider = {
59+ key: ProviderKey.make("memory"),
60+ writable: true,
61+ get: (id: ProviderItemId) => Effect.sync(() => memory.get(String(id)) ?? null),
62+ set: (id: ProviderItemId, value: string) =>
63+ Effect.sync(() => {
64+ memory.set(String(id), value);
65+ }),
66+};
4567
4668 const plugins = [
47− // Secret providers — three of them contributed by three plugins.
48− // The executor auto-registers each one at startup via the new
49− // `plugin.secretProviders` field.
69+ // Credential providers — three of them contributed by three plugins. The
70+ // executor auto-registers each one at startup via `plugin.credentialProviders`
71+ // (the v2 successor to v1's `secretProviders`). A connection routes its value
72+ // through one of these.
5073 keychainPlugin(),
5174 fileSecretsPlugin(),
5275 onepasswordPlugin(),
5376
54− // Source plugins — these declare their own schemas (tables) and
55− // register tools dynamically when the user adds a spec / connects
56− // to a server / imports a discovery document.
77+ // Integration plugins — these declare their own schemas (tables), register
78+ // integrations via their extension methods (`addSpec` / `addServer` /
79+ // `addIntegration`), and produce tools per connection.
5780 graphqlPlugin(),
5881 mcpPlugin({ dangerouslyAllowStdioMCP: false }),
5982 openApiPlugin(),
6083
61− // workos-vault is a cloud-hosted secret provider. It would contribute
62− // a "workos-vault" provider if credentials were available. We skip it
63− // here because it needs a real WorkOS API key; uncomment and supply
64− // credentials to wire it in.
84+ // workos-vault is a cloud-hosted credential provider. It would contribute a
85+ // "workos-vault" provider if credentials were available. We skip it here
86+ // because it needs a real WorkOS API key; uncomment and supply credentials to
87+ // wire it in.
6588 //
6689 // workosVaultPlugin({
6790 // credentials: {
@@ -76,8 +99,8 @@ const plugins = [ · 8 unmodified lines
7699 void workosVaultPlugin;
77100
78101 // ---------------------------------------------------------------------------
79−// 2. A tiny OpenAPI spec we'll use to demonstrate dynamic source
80−// registration. Five operations, all deterministic.
102+// 2. A tiny OpenAPI spec we'll use to demonstrate integration + connection
103+// registration. Four operations, all deterministic.
81104 // ---------------------------------------------------------------------------
82105
83106 const exampleOpenApiSpec = JSON.stringify({
@@ -157,9 +180,13 @@ const program = Effect.gen(function* () { · 73 unmodified lines
157180 console.log("=".repeat(72));
158181
159182 const executor = yield* createExecutor({
160− scopes: [scope],
183+ tenant: Tenant.make("example-tenant"),
161184 plugins,
185+ providers: [memoryProvider],
162186 onElicitation: "accept-all" as const,
187+ // `redirectUri` is intentionally omitted: this example never runs an
188+ // interactive OAuth flow. A host that serves OAuth must pass
189+ // `${webBaseUrl}/oauth/callback` here (there is no localhost default).
163190 });
164191
165192 // Every plugin's extension is accessible as `executor[pluginId]`.
@@ -174,85 +201,73 @@ const program = Effect.gen(function* () { · 8 unmodified lines
174201 console.log(" executor.openapi ", typeof executor.openapi);
175202
176203 // -------------------------------------------------------------------------
177− // Secrets — three providers were contributed by plugins. List them, then
178− // store a secret pinned to the `file` provider (file-secrets writes to a
179− // local auth.json under $XDG_DATA_HOME).
204+ // Credential providers — the inline `memory` store plus whichever plugin
205+ // providers were reachable (keychain/file/1Password register at startup).
206+ // A connection routes its value through one of these; there is no separate
207+ // secret store in v2 (a connection IS the saved credential).
180208 // -------------------------------------------------------------------------
181209
182210 console.log("\n" + "-".repeat(72));
183− console.log("Secrets");
211+ console.log("Credential providers");
184212 console.log("-".repeat(72));
185213
186− const providers = yield* executor.secrets.providers();
187− console.log("Registered providers:", providers);
188−
189− yield* executor.secrets.set(
190− SetSecretInput.make({
191− id: SecretId.make("example-api-token"),
192− scope: "example-scope" as SetSecretInput["scope"],
193− name: "Example API Token",
194− value: "sk-example-redacted",
195− provider: "file",
196− }),
197− );
198−
199− const token = yield* executor.secrets.get("example-api-token");
200− console.log("Stored + read 'example-api-token':", token);
201−
202− const secretRefs = yield* executor.secrets.list();
214+ const providerKeys = yield* executor.providers.list();
203215 console.log(
204− "Secret refs:",
205− secretRefs.map((r) => `${r.id}@${r.provider}`),
216+ "Registered providers:",
217+ providerKeys.map((k) => String(k)),
206218 );
207219
208220 // -------------------------------------------------------------------------
209− // Static control tools — every source plugin exposes its built-in
210− // control tools via `staticSources`. They live in memory, not in the
211− // DB, and show up in `tools.list()` alongside dynamic ones.
221+ // Integration: OpenAPI — register a tiny spec as an integration. The
222+ // `authenticationTemplate` declares WHERE a connection's value renders on
223+ // each request (here an `X-API-Key` header); `variable("token")` is the slot
224+ // the resolved credential fills.
212225 // -------------------------------------------------------------------------
213226
214227 console.log("\n" + "-".repeat(72));
215− console.log("Static sources / control tools");
216− console.log("-".repeat(72));
217−
218− const sourcesBefore = yield* executor.sources.list();
219− const staticSources = sourcesBefore.filter((s) => s.runtime);
220− console.log(
221− `Runtime sources (${staticSources.length}):`,
222− staticSources.map((s) => s.id),
223− );
224−
225− const toolsBefore = yield* executor.tools.list();
226− const staticTools = toolsBefore.filter((t) => t.sourceId.endsWith(".control"));
227− console.log(
228− `Static control tools (${staticTools.length}):`,
229− staticTools.map((t) => t.id),
230− );
231−
232− // -------------------------------------------------------------------------
233− // Dynamic source: OpenAPI — register a tiny spec. Four tools land in
234− // the `tool` table under a `example-api` source, plus one `$defs` entry
235− // (the `Item` schema) lands in the `definition` table for $ref
236− // resolution at read time.
237− // -------------------------------------------------------------------------
238−
239− console.log("\n" + "-".repeat(72));
240− console.log("Dynamic source: OpenAPI");
228+ console.log("Integration: OpenAPI");
241229 console.log("-".repeat(72));
242230
243231 const addSpecResult = yield* executor.openapi.addSpec({
244232 spec: { kind: "blob", value: exampleOpenApiSpec },
245− namespace: "example-api",
246− name: "Example API",
233+ slug: "example-api",
234+ description: "Example API",
247235 baseUrl: "https://example.com/api",
248− scope: "example-scope",
236+ authenticationTemplate: [
237+ {
238+ slug: AuthTemplateSlug.make("apiKey"),
239+ type: "apiKey",
240+ headers: { "X-API-Key": [variable("token")] },
241+ },
242+ ],
243+ });
244+ console.log("Registered OpenAPI integration:", {
245+ slug: String(addSpecResult.slug),
246+ toolCount: addSpecResult.toolCount,
249247 });
250− console.log("Registered OpenAPI source:", addSpecResult);
251248
252− const exampleTools = yield* executor.tools.list({ sourceId: "example-api" });
249+ // A connection is the credential. Creating one with an inline `value` writes
250+ // it to the default writable provider (`memory`) and produces the
251+ // integration's per-connection tools, addressed
252+ // `tools.example-api.org.default.<tool>`.
253+ const openApiConnection = yield* executor.connections.create({
254+ owner: "org",
255+ name: ConnectionName.make("default"),
256+ integration: IntegrationSlug.make("example-api"),
257+ template: AuthTemplateSlug.make("apiKey"),
258+ value: "sk-example-redacted",
259+ });
260+ console.log("Created connection:", {
261+ address: String(openApiConnection.address),
262+ provider: String(openApiConnection.provider),
263+ });
264+
265+ const exampleTools = yield* executor.tools.list({
266+ integration: IntegrationSlug.make("example-api"),
267+ });
253268 console.log(
254269 "Tools under 'example-api':",
255− exampleTools.map((t) => t.name),
270+ exampleTools.map((t) => String(t.address)),
256271 );
257272
258273 // Annotations are derived at read time via plugin.resolveAnnotations.
@@ -260,28 +275,32 @@ const program = Effect.gen(function* () { · 1 unmodified line
260275 console.log(
261276 "Annotations on example-api tools:",
262277 exampleTools.map((t) => ({
263− name: t.name,
278+ name: String(t.name),
264279 requiresApproval: t.annotations?.requiresApproval ?? false,
265280 })),
266281 );
267282
268− // `tools.schema` walks the read path: reads the tool row, attaches
269− // matching $defs from the core `definition` table.
270− const getItemSchema = yield* executor.tools.schema("example-api.items.get");
271− console.log(
272− "Schema for items.get has $defs?",
273− getItemSchema?.inputSchema &&
274− typeof getItemSchema.inputSchema === "object" &&
275− "$defs" in getItemSchema.inputSchema,
276− );
283+ // `tools.schema` walks the read path: reads the tool row, attaches matching
284+ // $defs (the `Item` schema) for $ref resolution.
285+ const getItemTool = exampleTools.find((t) => String(t.name).startsWith("items__get"));
286+ if (getItemTool) {
287+ const getItemSchema = yield* executor.tools.schema(getItemTool.address);
288+ console.log(
289+ "Schema for items.get has $defs?",
290+ getItemSchema?.inputSchema &&
291+ typeof getItemSchema.inputSchema === "object" &&
292+ "$defs" in getItemSchema.inputSchema,
293+ );
294+ }
277295
278296 // -------------------------------------------------------------------------
279− // Dynamic source: GraphQL — introspect via a canned JSON doc so we
280− // don't need a real server running.
297+ // Integration: GraphQL — introspect via a canned JSON doc so we don't need a
298+ // real server running, then connect (this endpoint needs no credential, so
299+ // the connection carries an empty value through a "none" template).
281300 // -------------------------------------------------------------------------
282301
283302 console.log("\n" + "-".repeat(72));
284− console.log("Dynamic source: GraphQL");
303+ console.log("Integration: GraphQL");
285304 console.log("-".repeat(72));
286305
287306 const introspectionJson = JSON.stringify({
@@ -351,28 +370,41 @@ const program = Effect.gen(function* () { · 63 unmodified lines
351370 },
352371 });
353372
354− const gqlResult = yield* executor.graphql.addSource({
373+ const gqlResult = yield* executor.graphql.addIntegration({
355374 endpoint: "https://example.com/graphql",
356375 name: "Example GraphQL",
357376 introspectionJson,
358− namespace: "example-graphql",
359− scope: "example-scope",
377+ slug: "example-graphql",
378+ });
379+ console.log("Registered GraphQL integration:", gqlResult);
380+
381+ yield* executor.connections.create({
382+ owner: "org",
383+ name: ConnectionName.make("default"),
384+ integration: IntegrationSlug.make("example-graphql"),
385+ template: AuthTemplateSlug.make("none"),
386+ value: "",
360387 });
361− console.log("Registered GraphQL source:", gqlResult);
362388
363− const graphqlTools = yield* executor.tools.list({ sourceId: "example-graphql" });
389+ const graphqlTools = yield* executor.tools.list({
390+ integration: IntegrationSlug.make("example-graphql"),
391+ });
364392 console.log(
365393 "Tools under 'example-graphql':",
366394 graphqlTools.map((t) => ({
367− name: t.name,
395+ address: String(t.address),
368396 requiresApproval: t.annotations?.requiresApproval ?? false,
369397 })),
370398 );
371399
372400 // -------------------------------------------------------------------------
373− // MCP, Google OAuth, 1Password — shown but not exercised (they need
374− // real external infrastructure). Their extension methods exist, and
375− // calling them would register real dynamic sources the same way.
401+ // Other plugin extensions — shown but not exercised (they need real external
402+ // infrastructure). Their extension methods exist and would register real
403+ // integrations + connections the same way.
404+ //
405+ // removed: v1 `executor.secrets.set/get/list` and credential bindings — a
406+ // connection now IS the credential (see the OpenAPI flow above), and its value
407+ // lives in a registered provider rather than a free-floating secret store.
376408 // -------------------------------------------------------------------------
377409
378410 console.log("\n" + "-".repeat(72));
@@ -381,13 +413,35 @@ const program = Effect.gen(function* () { · 2 unmodified lines
381413
382414 console.log(" executor.keychain.isSupported:", executor.keychain.isSupported);
383415 console.log(" executor.keychain.displayName:", executor.keychain.displayName);
384−
385416 console.log(" executor.fileSecrets.filePath: ", executor.fileSecrets.filePath);
386417
387− // executor.mcp.addSource({ connector: { kind: "remote", endpoint: "..." } });
388− // executor.openapi.addSpec({ spec: { kind: "googleDiscovery", url: "..." }, ... });
418+ // executor.mcp.addServer({ transport: "remote", name: "...", endpoint: "...", slug: "..." });
419+ // executor.openapi.addSpec({ spec: { kind: "googleDiscovery", url: "..." }, slug: "..." });
389420 // executor.onepassword.configure({ auth: { kind: "desktop-app", accountName: "..." }, vaultId: "..." });
390421
422+ // -------------------------------------------------------------------------
423+ // Execute a tool over its connection. The executor resolves the connection's
424+ // credential (from the `memory` provider) and hands it to the owning plugin's
425+ // invokeTool, which renders it through the auth template onto the request.
426+ // (The example.com host isn't real, so this surfaces a transport-level
427+ // failure — the point is the resolve + dispatch path, not a live response.)
428+ // -------------------------------------------------------------------------
429+
430+ console.log("\n" + "-".repeat(72));
431+ console.log("Execute over a connection");
432+ console.log("-".repeat(72));
433+
434+ const listItemsTool = exampleTools.find((t) => String(t.name).startsWith("items__list"));
435+ if (listItemsTool) {
436+ const outcome = yield* Effect.result(
437+ executor.execute(ToolAddress.make(String(listItemsTool.address)), {}),
438+ );
439+ console.log(
440+ `execute ${String(listItemsTool.address)}:`,
441+ Result.isSuccess(outcome) ? "ok" : `failed (${outcome.failure.constructor.name})`,
442+ );
443+ }
444+
391445 // -------------------------------------------------------------------------
392446 // Whole-catalog tools listing + filtering
393447 // -------------------------------------------------------------------------
@@ -399,20 +453,27 @@ const program = Effect.gen(function* () { · 5 unmodified lines
399453 const allTools = yield* executor.tools.list();
400454 console.log(`Total tools: ${allTools.length}`);
401455
402− const allSources = yield* executor.sources.list();
456+ const allIntegrations = yield* executor.integrations.list();
457+ console.log(
458+ `Total integrations: ${allIntegrations.length}`,
459+ allIntegrations.map((i) => String(i.slug)),
460+ );
461+
462+ const allConnections = yield* executor.connections.list();
403463 console.log(
404− `Total sources: ${allSources.length} (${allSources.filter((s) => s.runtime).length} runtime, ${allSources.filter((s) => !s.runtime).length} dynamic)`,
464+ `Total connections: ${allConnections.length}`,
465+ allConnections.map((c) => String(c.address)),
405466 );
406467
407468 const mutationTools = yield* executor.tools.list({ query: "create" });
408469 console.log(
409470 "Tools matching 'create':",
410− mutationTools.map((t) => t.id),
471+ mutationTools.map((t) => String(t.address)),
411472 );
412473
413474 // -------------------------------------------------------------------------
414− // Shutdown — close() is called on every plugin that declared a `close`
415− // hook (the cache-backed ones like MCP tear down their connection pool).
475+ // Shutdown — close() is called on every plugin that declared a `close` hook
476+ // (the cache-backed ones like MCP tear down their connection pool).
416477 // -------------------------------------------------------------------------
417478
418479 console.log("\n" + "-".repeat(72));