executor v1.5: integrations and connections rework

Rhys Sullivan committed 5 months ago

Commit a30f26b1 parent bec7b7b497 files · +41191 −81846Browse files

Showing the changes to examples/all-plugins/src (1 of 497 files). Show all

Unified · Split

examples/all-plugins/src/main.ts+184−123

 // ---------------------------------------------------------------------------
 // examples/all-plugins
 //
-// Wires every ported plugin into a single Executor and walks through the
+// Wires every ported plugin into a single Executor and walks the common v2
-// common flows: secrets, static control tools, dynamic source registration,
+// flows: credential providers, integration registration, connection creation
-// tool invocation, filtered listing, and shutdown.
+// (a connection IS the credential), per-connection tool production, execution,
+// filtered listing, and shutdown.
 //
-// This is what an app/local or app/cloud bootstrap file looks like under
+// This is what an app/local or app/cloud bootstrap file looks like under the
-// the new SDK shape — minus the HTTP API layer, runtime lifecycle, and
+// v2 SDK shape — minus the HTTP API layer, runtime lifecycle, and owner/tenant
-// scope persistence that real apps add on top.
+// persistence that real apps add on top.
 //
 // Runs against the SDK's ephemeral in-memory FumaDB backend so you can
-// `bun run src/main.ts` and watch the whole surface exercise itself.
+// `bun run src/main.ts` and watch the whole surface exercise itself. Plugins
-// Plugins that need external infra (keychain prompts, 1Password unlock,
+// that need external infra (keychain prompts, 1Password unlock, MCP transport,
-// MCP transport, WorkOS Vault, Google OAuth) are wired so their secret
+// WorkOS Vault, Google OAuth) are wired so their credential providers and
-// providers and extensions exist, but the flows that hit their
+// extensions exist, but the flows that hit their backends are skipped by
-// backends are gated behind env vars and skipped by default.
+// default.
 // ---------------------------------------------------------------------------
-import { Cause, Effect } from "effect";
+import { Cause, Effect, Result } from "effect";
-import { SecretId, Scope, ScopeId, SetSecretInput, createExecutor } from "@executor-js/sdk";
+import {
+  AuthTemplateSlug,
+  ConnectionName,
+  createExecutor,
+  IntegrationSlug,
+  ProviderItemId,
+  ProviderKey,
+  Tenant,
+  ToolAddress,
+  type CredentialProvider,
+} from "@executor-js/sdk";
 import { fileSecretsPlugin } from "@executor-js/plugin-file-secrets";
 import { graphqlPlugin } from "@executor-js/plugin-graphql";
 import { keychainPlugin } from "@executor-js/plugin-keychain";
 import { mcpPlugin } from "@executor-js/plugin-mcp";
 import { onepasswordPlugin } from "@executor-js/plugin-onepassword";
-import { openApiPlugin } from "@executor-js/plugin-openapi";
+import { openApiPlugin, variable } from "@executor-js/plugin-openapi";
 import { workosVaultPlugin } from "@executor-js/plugin-workos-vault";
 // ---------------------------------------------------------------------------
 // 1. Build the ExecutorConfig.
 //
-// Three pieces only: scope, FumaDB, plugins.
+// Three pieces: tenant, plugins, and credential providers. The executor
-// Compare to the old SDK, where you'd pass pre-built ToolRegistry,
+// auto-registers every `plugin.credentialProviders`; `config.providers` adds
-// SourceRegistry, SecretStore, and PolicyEngine service instances.
+// inline ones (registered first, so they win as the default writable store).
+// Compare to v1, where you'd pass pre-built ToolRegistry, SourceRegistry,
+// SecretStore, and PolicyEngine service instances plus a scope stack.
 // ---------------------------------------------------------------------------
-const scope = Scope.make({
+// A connection's value lives in a writable credential provider. This tiny
-  id: ScopeId.make("example-scope"),
+// in-memory store is enough for a script; the keychain / file-secrets /
-  name: "/tmp/example-workspace",
+// 1Password plugins below contribute durable ones. Providers are Effect-native,
-  createdAt: new Date(),
+// so `get`/`set` return `Effect`s.
-});
+const memory = new Map<string, string>();
+const memoryProvider: CredentialProvider = {
+  key: ProviderKey.make("memory"),
+  writable: true,
+  get: (id: ProviderItemId) => Effect.sync(() => memory.get(String(id)) ?? null),
+  set: (id: ProviderItemId, value: string) =>
+    Effect.sync(() => {
+      memory.set(String(id), value);
+    }),
+};
 const plugins = [
-  // Secret providers — three of them contributed by three plugins.
+  // Credential providers — three of them contributed by three plugins. The
-  // The executor auto-registers each one at startup via the new
+  // executor auto-registers each one at startup via `plugin.credentialProviders`
-  // `plugin.secretProviders` field.
+  // (the v2 successor to v1's `secretProviders`). A connection routes its value
+  // through one of these.
   keychainPlugin(),
   fileSecretsPlugin(),
   onepasswordPlugin(),
-  // Source plugins — these declare their own schemas (tables) and
+  // Integration plugins — these declare their own schemas (tables), register
-  // register tools dynamically when the user adds a spec / connects
+  // integrations via their extension methods (`addSpec` / `addServer` /
-  // to a server / imports a discovery document.
+  // `addIntegration`), and produce tools per connection.
   graphqlPlugin(),
   mcpPlugin({ dangerouslyAllowStdioMCP: false }),
   openApiPlugin(),
-  // workos-vault is a cloud-hosted secret provider. It would contribute
+  // workos-vault is a cloud-hosted credential provider. It would contribute a
-  // a "workos-vault" provider if credentials were available. We skip it
+  // "workos-vault" provider if credentials were available. We skip it here
-  // here because it needs a real WorkOS API key; uncomment and supply
+  // because it needs a real WorkOS API key; uncomment and supply credentials to
-  // credentials to wire it in.
+  // wire it in.
   //
   // workosVaultPlugin({
   //   credentials: {
 void workosVaultPlugin;
 // ---------------------------------------------------------------------------
-// 2. A tiny OpenAPI spec we'll use to demonstrate dynamic source
+// 2. A tiny OpenAPI spec we'll use to demonstrate integration + connection
-// registration. Five operations, all deterministic.
+// registration. Four operations, all deterministic.
 // ---------------------------------------------------------------------------
 const exampleOpenApiSpec = JSON.stringify({
   console.log("=".repeat(72));
   const executor = yield* createExecutor({
-    scopes: [scope],
+    tenant: Tenant.make("example-tenant"),
     plugins,
+    providers: [memoryProvider],
     onElicitation: "accept-all" as const,
+    // `redirectUri` is intentionally omitted: this example never runs an
+    // interactive OAuth flow. A host that serves OAuth must pass
+    // `${webBaseUrl}/oauth/callback` here (there is no localhost default).
   });
   // Every plugin's extension is accessible as `executor[pluginId]`.
   console.log("  executor.openapi         ", typeof executor.openapi);
   // -------------------------------------------------------------------------
-  // Secrets — three providers were contributed by plugins. List them, then
+  // Credential providers — the inline `memory` store plus whichever plugin
-  // store a secret pinned to the `file` provider (file-secrets writes to a
+  // providers were reachable (keychain/file/1Password register at startup).
-  // local auth.json under $XDG_DATA_HOME).
+  // A connection routes its value through one of these; there is no separate
+  // secret store in v2 (a connection IS the saved credential).
   // -------------------------------------------------------------------------
   console.log("\n" + "-".repeat(72));
-  console.log("Secrets");
+  console.log("Credential providers");
   console.log("-".repeat(72));
-  const providers = yield* executor.secrets.providers();
+  const providerKeys = yield* executor.providers.list();
-  console.log("Registered providers:", providers);
-  yield* executor.secrets.set(
-    SetSecretInput.make({
-      id: SecretId.make("example-api-token"),
-      scope: "example-scope" as SetSecretInput["scope"],
-      name: "Example API Token",
-      value: "sk-example-redacted",
-      provider: "file",
-    }),
-  );
-  const token = yield* executor.secrets.get("example-api-token");
-  console.log("Stored + read 'example-api-token':", token);
-  const secretRefs = yield* executor.secrets.list();
   console.log(
-    "Secret refs:",
+    "Registered providers:",
-    secretRefs.map((r) => `${r.id}@${r.provider}`),
+    providerKeys.map((k) => String(k)),
   );
   // -------------------------------------------------------------------------
-  // Static control tools — every source plugin exposes its built-in
+  // Integration: OpenAPI — register a tiny spec as an integration. The
-  // control tools via `staticSources`. They live in memory, not in the
+  // `authenticationTemplate` declares WHERE a connection's value renders on
-  // DB, and show up in `tools.list()` alongside dynamic ones.
+  // each request (here an `X-API-Key` header); `variable("token")` is the slot
+  // the resolved credential fills.
   // -------------------------------------------------------------------------
   console.log("\n" + "-".repeat(72));
-  console.log("Static sources / control tools");
+  console.log("Integration: OpenAPI");
-  console.log("-".repeat(72));
-  const sourcesBefore = yield* executor.sources.list();
-  const staticSources = sourcesBefore.filter((s) => s.runtime);
-  console.log(
-    `Runtime sources (${staticSources.length}):`,
-    staticSources.map((s) => s.id),
-  );
-  const toolsBefore = yield* executor.tools.list();
-  const staticTools = toolsBefore.filter((t) => t.sourceId.endsWith(".control"));
-  console.log(
-    `Static control tools (${staticTools.length}):`,
-    staticTools.map((t) => t.id),
-  );
-  // -------------------------------------------------------------------------
-  // Dynamic source: OpenAPI — register a tiny spec. Four tools land in
-  // the `tool` table under a `example-api` source, plus one `$defs` entry
-  // (the `Item` schema) lands in the `definition` table for $ref
-  // resolution at read time.
-  // -------------------------------------------------------------------------
-  console.log("\n" + "-".repeat(72));
-  console.log("Dynamic source: OpenAPI");
   console.log("-".repeat(72));
   const addSpecResult = yield* executor.openapi.addSpec({
     spec: { kind: "blob", value: exampleOpenApiSpec },
-    namespace: "example-api",
+    slug: "example-api",
-    name: "Example API",
+    description: "Example API",
     baseUrl: "https://example.com/api",
-    scope: "example-scope",
+    authenticationTemplate: [
+      {
+        slug: AuthTemplateSlug.make("apiKey"),
+        type: "apiKey",
+        headers: { "X-API-Key": [variable("token")] },
+      },
+    ],
+  });
+  console.log("Registered OpenAPI integration:", {
+    slug: String(addSpecResult.slug),
+    toolCount: addSpecResult.toolCount,
   });
-  console.log("Registered OpenAPI source:", addSpecResult);
-  const exampleTools = yield* executor.tools.list({ sourceId: "example-api" });
+  // A connection is the credential. Creating one with an inline `value` writes
+  // it to the default writable provider (`memory`) and produces the
+  // integration's per-connection tools, addressed
+  // `tools.example-api.org.default.<tool>`.
+  const openApiConnection = yield* executor.connections.create({
+    owner: "org",
+    name: ConnectionName.make("default"),
+    integration: IntegrationSlug.make("example-api"),
+    template: AuthTemplateSlug.make("apiKey"),
+    value: "sk-example-redacted",
+  });
+  console.log("Created connection:", {
+    address: String(openApiConnection.address),
+    provider: String(openApiConnection.provider),
+  });
+  const exampleTools = yield* executor.tools.list({
+    integration: IntegrationSlug.make("example-api"),
+  });
   console.log(
     "Tools under 'example-api':",
-    exampleTools.map((t) => t.name),
+    exampleTools.map((t) => String(t.address)),
   );
   // Annotations are derived at read time via plugin.resolveAnnotations.
   console.log(
     "Annotations on example-api tools:",
     exampleTools.map((t) => ({
-      name: t.name,
+      name: String(t.name),
       requiresApproval: t.annotations?.requiresApproval ?? false,
     })),
   );
-  // `tools.schema` walks the read path: reads the tool row, attaches
+  // `tools.schema` walks the read path: reads the tool row, attaches matching
-  // matching $defs from the core `definition` table.
+  // $defs (the `Item` schema) for $ref resolution.
-  const getItemSchema = yield* executor.tools.schema("example-api.items.get");
+  const getItemTool = exampleTools.find((t) => String(t.name).startsWith("items__get"));
-  console.log(
+  if (getItemTool) {
-    "Schema for items.get has $defs?",
+    const getItemSchema = yield* executor.tools.schema(getItemTool.address);
-    getItemSchema?.inputSchema &&
+    console.log(
-      typeof getItemSchema.inputSchema === "object" &&
+      "Schema for items.get has $defs?",
-      "$defs" in getItemSchema.inputSchema,
+      getItemSchema?.inputSchema &&
-  );
+        typeof getItemSchema.inputSchema === "object" &&
+        "$defs" in getItemSchema.inputSchema,
+    );
+  }
   // -------------------------------------------------------------------------
-  // Dynamic source: GraphQL — introspect via a canned JSON doc so we
+  // Integration: GraphQL — introspect via a canned JSON doc so we don't need a
-  // don't need a real server running.
+  // real server running, then connect (this endpoint needs no credential, so
+  // the connection carries an empty value through a "none" template).
   // -------------------------------------------------------------------------
   console.log("\n" + "-".repeat(72));
-  console.log("Dynamic source: GraphQL");
+  console.log("Integration: GraphQL");
   console.log("-".repeat(72));
   const introspectionJson = JSON.stringify({
     },
   });
-  const gqlResult = yield* executor.graphql.addSource({
+  const gqlResult = yield* executor.graphql.addIntegration({
     endpoint: "https://example.com/graphql",
     name: "Example GraphQL",
     introspectionJson,
-    namespace: "example-graphql",
+    slug: "example-graphql",
-    scope: "example-scope",
+  });
+  console.log("Registered GraphQL integration:", gqlResult);
+  yield* executor.connections.create({
+    owner: "org",
+    name: ConnectionName.make("default"),
+    integration: IntegrationSlug.make("example-graphql"),
+    template: AuthTemplateSlug.make("none"),
+    value: "",
   });
-  console.log("Registered GraphQL source:", gqlResult);
-  const graphqlTools = yield* executor.tools.list({ sourceId: "example-graphql" });
+  const graphqlTools = yield* executor.tools.list({
+    integration: IntegrationSlug.make("example-graphql"),
+  });
   console.log(
     "Tools under 'example-graphql':",
     graphqlTools.map((t) => ({
-      name: t.name,
+      address: String(t.address),
       requiresApproval: t.annotations?.requiresApproval ?? false,
     })),
   );
   // -------------------------------------------------------------------------
-  // MCP, Google OAuth, 1Password — shown but not exercised (they need
+  // Other plugin extensions — shown but not exercised (they need real external
-  // real external infrastructure). Their extension methods exist, and
+  // infrastructure). Their extension methods exist and would register real
-  // calling them would register real dynamic sources the same way.
+  // integrations + connections the same way.
+  //
+  // removed: v1 `executor.secrets.set/get/list` and credential bindings — a
+  // connection now IS the credential (see the OpenAPI flow above), and its value
+  // lives in a registered provider rather than a free-floating secret store.
   // -------------------------------------------------------------------------
   console.log("\n" + "-".repeat(72));
   console.log("  executor.keychain.isSupported:", executor.keychain.isSupported);
   console.log("  executor.keychain.displayName:", executor.keychain.displayName);
   console.log("  executor.fileSecrets.filePath:   ", executor.fileSecrets.filePath);
-  // executor.mcp.addSource({ connector: { kind: "remote", endpoint: "..." } });
+  // executor.mcp.addServer({ transport: "remote", name: "...", endpoint: "...", slug: "..." });
-  // executor.openapi.addSpec({ spec: { kind: "googleDiscovery", url: "..." }, ... });
+  // executor.openapi.addSpec({ spec: { kind: "googleDiscovery", url: "..." }, slug: "..." });
   // executor.onepassword.configure({ auth: { kind: "desktop-app", accountName: "..." }, vaultId: "..." });
+  // -------------------------------------------------------------------------
+  // Execute a tool over its connection. The executor resolves the connection's
+  // credential (from the `memory` provider) and hands it to the owning plugin's
+  // invokeTool, which renders it through the auth template onto the request.
+  // (The example.com host isn't real, so this surfaces a transport-level
+  // failure — the point is the resolve + dispatch path, not a live response.)
+  // -------------------------------------------------------------------------
+  console.log("\n" + "-".repeat(72));
+  console.log("Execute over a connection");
+  console.log("-".repeat(72));
+  const listItemsTool = exampleTools.find((t) => String(t.name).startsWith("items__list"));
+  if (listItemsTool) {
+    const outcome = yield* Effect.result(
+      executor.execute(ToolAddress.make(String(listItemsTool.address)), {}),
+    );
+    console.log(
+      `execute ${String(listItemsTool.address)}:`,
+      Result.isSuccess(outcome) ? "ok" : `failed (${outcome.failure.constructor.name})`,
+    );
+  }
   // -------------------------------------------------------------------------
   // Whole-catalog tools listing + filtering
   // -------------------------------------------------------------------------
   const allTools = yield* executor.tools.list();
   console.log(`Total tools: ${allTools.length}`);
-  const allSources = yield* executor.sources.list();
+  const allIntegrations = yield* executor.integrations.list();
+  console.log(
+    `Total integrations: ${allIntegrations.length}`,
+    allIntegrations.map((i) => String(i.slug)),
+  );
+  const allConnections = yield* executor.connections.list();
   console.log(
-    `Total sources: ${allSources.length} (${allSources.filter((s) => s.runtime).length} runtime, ${allSources.filter((s) => !s.runtime).length} dynamic)`,
+    `Total connections: ${allConnections.length}`,
+    allConnections.map((c) => String(c.address)),
   );
   const mutationTools = yield* executor.tools.list({ query: "create" });
   console.log(
     "Tools matching 'create':",
-    mutationTools.map((t) => t.id),
+    mutationTools.map((t) => String(t.address)),
   );
   // -------------------------------------------------------------------------
-  // Shutdown — close() is called on every plugin that declared a `close`
+  // Shutdown — close() is called on every plugin that declared a `close` hook
-  // hook (the cache-backed ones like MCP tear down their connection pool).
+  // (the cache-backed ones like MCP tear down their connection pool).
   // -------------------------------------------------------------------------
   console.log("\n" + "-".repeat(72));