* feat(sdk): add built-in core-tools plugin with agent secret-create flow Introduces `coreToolsPlugin` in @executor-js/sdk exposing three static tools agents can call directly: - `scopes.list` — enumerate visible scopes by name; agents pick a scope by asking the user, not by guessing or defaulting. - `secrets.list` — visible secrets as `{id, name, provider}`; values never cross the agent boundary. - `secrets.create` — pre-allocates a secret id and returns a URL to the existing /secrets web form, pre-filled with name + id. The user enters the value in the browser; the agent confirms by re-listing. The plugin auto-registers when `coreTools: { webBaseUrl }` is set on ExecutorConfig, so hosts opt in with one line. apps/local wires it to the daemon's own port — same host as the API, no separate UI server. The /secrets web side reads the prefill params via TanStack Router `validateSearch`, opens the add modal pre-filled, and locks the id via a new `initialIdOverride` prop on SecretForm. URL prefilling works fully through the dev:cli vite proxy. * fix(sdk): use typed core tools errors * feat: expand agent executor configuration tools * feat: expose source presets to agents * fix: resolve agent source install scopes * fix: clarify secret creation for agents * fix: render open object schema previews * fix: expose typed source configure tools * fix: stabilize agent openapi preview flow * fix: let secret handoff choose scope * fix: explain secret handoff completion * fix: explain oauth handoff completion * fix: clarify oauth credential scope * fix: satisfy core tools ci * fix: clarify preset and resume guidance
Showing the changes to scripts (1 of 54 files). Show all
Unified · Split
| @@ -0,0 +1,340 @@ | ||
| 1 | +#!/usr/bin/env bun | |
| 2 | +/** | |
| 3 | + * End-to-end smoke for agent-driven Executor configuration. | |
| 4 | + * | |
| 5 | + * This intentionally drives the public dev CLI for agent actions, while using | |
| 6 | + * the local HTTP browser/API boundary for sensitive values and OAuth callback | |
| 7 | + * completion. Fake test credentials are used, but the shape matches the real | |
| 8 | + * flow: agents get handoff URLs, users/browsers enter secrets and complete | |
| 9 | + * OAuth outside the context window, then agents verify and bind the resulting | |
| 10 | + * ids. | |
| 11 | + */ | |
| 12 | +import { mkdtemp, mkdir, rm } from "node:fs/promises"; | |
| 13 | +import { tmpdir } from "node:os"; | |
| 14 | +import { dirname, join, resolve } from "node:path"; | |
| 15 | +import { fileURLToPath } from "node:url"; | |
| 16 | + | |
| 17 | +import { serveOAuthTestServer, type OAuthTestServerShape } from "../packages/core/sdk/src/testing"; | |
| 18 | +import { | |
| 19 | + type GraphqlTestServerShape, | |
| 20 | + makeGreetingGraphqlSchema, | |
| 21 | + serveGraphqlTestServer, | |
| 22 | +} from "../packages/plugins/graphql/src/testing/index"; | |
| 23 | +import { Effect } from "effect"; | |
| 24 | + | |
| 25 | +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); | |
| 26 | +const cliEntrypoint = join(repoRoot, "apps/cli/src/main.ts"); | |
| 27 | + | |
| 28 | +type CliContext = { | |
| 29 | + readonly dataDir: string; | |
| 30 | + readonly scopeDir: string; | |
| 31 | + readonly baseUrl: string; | |
| 32 | +}; | |
| 33 | + | |
| 34 | +type CliResult = { | |
| 35 | + readonly stdout: string; | |
| 36 | + readonly stderr: string; | |
| 37 | + readonly text: string; | |
| 38 | + readonly exitCode: number; | |
| 39 | +}; | |
| 40 | + | |
| 41 | +const assert = (condition: unknown, message: string): asserts condition => { | |
| 42 | + if (!condition) throw new Error(message); | |
| 43 | +}; | |
| 44 | + | |
| 45 | +const runCli = async ( | |
| 46 | + ctx: CliContext, | |
| 47 | + args: readonly string[], | |
| 48 | + options: { readonly allowFailure?: boolean } = {}, | |
| 49 | +): Promise<CliResult> => { | |
| 50 | + const proc = Bun.spawn([process.execPath, "run", cliEntrypoint, ...args], { | |
| 51 | + cwd: repoRoot, | |
| 52 | + env: { | |
| 53 | + ...process.env, | |
| 54 | + EXECUTOR_DEV: "1", | |
| 55 | + EXECUTOR_DATA_DIR: ctx.dataDir, | |
| 56 | + EXECUTOR_SCOPE_DIR: ctx.scopeDir, | |
| 57 | + }, | |
| 58 | + stdout: "pipe", | |
| 59 | + stderr: "pipe", | |
| 60 | + }); | |
| 61 | + | |
| 62 | + const [stdout, stderr, exitCode] = await Promise.all([ | |
| 63 | + new Response(proc.stdout).text(), | |
| 64 | + new Response(proc.stderr).text(), | |
| 65 | + proc.exited, | |
| 66 | + ]); | |
| 67 | + const result = { stdout, stderr, text: `${stdout}${stderr}`, exitCode }; | |
| 68 | + | |
| 69 | + if (exitCode !== 0 && !options.allowFailure) { | |
| 70 | + throw new Error(`CLI failed (${exitCode}) for ${args.join(" ")}\n${result.text}`); | |
| 71 | + } | |
| 72 | + | |
| 73 | + return result; | |
| 74 | +}; | |
| 75 | + | |
| 76 | +const parseJsonOutput = <T>(result: CliResult): T => { | |
| 77 | + const start = result.stdout.indexOf("{"); | |
| 78 | + const end = result.stdout.lastIndexOf("}"); | |
| 79 | + assert(start >= 0 && end > start, `CLI output did not contain JSON:\n${result.text}`); | |
| 80 | + return JSON.parse(result.stdout.slice(start, end + 1)) as T; | |
| 81 | +}; | |
| 82 | + | |
| 83 | +const toolData = <T>(value: unknown): T => { | |
| 84 | + const result = value as { readonly ok?: boolean; readonly data?: T; readonly error?: unknown }; | |
| 85 | + assert(result.ok === true, `Tool returned failure: ${JSON.stringify(value)}`); | |
| 86 | + return result.data as T; | |
| 87 | +}; | |
| 88 | + | |
| 89 | +const extractExecutionId = (text: string): string => { | |
| 90 | + const match = /executionId:\s*(exec_[A-Za-z0-9_]+)/.exec(text); | |
| 91 | + assert(match?.[1], `CLI output did not contain an execution id:\n${text}`); | |
| 92 | + return match[1]; | |
| 93 | +}; | |
| 94 | + | |
| 95 | +const callTool = (ctx: CliContext, path: readonly string[], args: unknown = {}) => | |
| 96 | + runCli(ctx, [ | |
| 97 | + "call", | |
| 98 | + ...path, | |
| 99 | + JSON.stringify(args), | |
| 100 | + "--base-url", | |
| 101 | + ctx.baseUrl, | |
| 102 | + "--scope", | |
| 103 | + ctx.scopeDir, | |
| 104 | + ]); | |
| 105 | + | |
| 106 | +const approvePausedCall = async (ctx: CliContext, paused: CliResult): Promise<CliResult> => { | |
| 107 | + const executionId = extractExecutionId(paused.text); | |
| 108 | + return await runCli(ctx, [ | |
| 109 | + "resume", | |
| 110 | + "--execution-id", | |
| 111 | + executionId, | |
| 112 | + "--base-url", | |
| 113 | + ctx.baseUrl, | |
| 114 | + "--scope", | |
| 115 | + ctx.scopeDir, | |
| 116 | + ]); | |
| 117 | +}; | |
| 118 | + | |
| 119 | +const postJson = async (url: string, payload: unknown): Promise<unknown> => { | |
| 120 | + const response = await fetch(url, { | |
| 121 | + method: "POST", | |
| 122 | + headers: { "content-type": "application/json" }, | |
| 123 | + body: JSON.stringify(payload), | |
| 124 | + }); | |
| 125 | + const bodyText = await response.text(); | |
| 126 | + if (!response.ok) { | |
| 127 | + throw new Error(`POST ${url} failed with HTTP ${response.status}: ${bodyText}`); | |
| 128 | + } | |
| 129 | + return bodyText.length > 0 ? JSON.parse(bodyText) : null; | |
| 130 | +}; | |
| 131 | + | |
| 132 | +const firstScope = (value: unknown): { readonly id: string; readonly name: string } => { | |
| 133 | + const data = toolData<{ | |
| 134 | + readonly scopes: readonly { readonly id: string; readonly name: string }[]; | |
| 135 | + }>(value); | |
| 136 | + const scopes = data.scopes; | |
| 137 | + const scope = scopes?.[0]; | |
| 138 | + assert(scope, `scopes.list returned no scopes: ${JSON.stringify(value)}`); | |
| 139 | + return scope; | |
| 140 | +}; | |
| 141 | + | |
| 142 | +const createSecretPlaceholder = async ( | |
| 143 | + ctx: CliContext, | |
| 144 | + scope: string, | |
| 145 | + expectedHandoffScope: string, | |
| 146 | + name: string, | |
| 147 | +) => { | |
| 148 | + const result = toolData<{ readonly id: string; readonly url: string }>( | |
| 149 | + parseJsonOutput( | |
| 150 | + await callTool(ctx, ["executor", "coreTools", "secrets", "create"], { name, scope }), | |
| 151 | + ), | |
| 152 | + ); | |
| 153 | + const handoff = new URL(result.url); | |
| 154 | + assert(handoff.pathname === "/secrets", `Expected /secrets handoff URL, got ${result.url}`); | |
| 155 | + assert( | |
| 156 | + handoff.searchParams.get("scope") === expectedHandoffScope, | |
| 157 | + `Secret handoff omitted scope: ${result.url}`, | |
| 158 | + ); | |
| 159 | + return result; | |
| 160 | +}; | |
| 161 | + | |
| 162 | +const setSecretViaBrowserBoundary = async ( | |
| 163 | + ctx: CliContext, | |
| 164 | + browserBaseUrl: string, | |
| 165 | + scopeId: string, | |
| 166 | + input: { readonly id: string; readonly name: string; readonly value: string }, | |
| 167 | +) => { | |
| 168 | + await postJson(`${browserBaseUrl}/api/scopes/${encodeURIComponent(scopeId)}/secrets`, input); | |
| 169 | + const status = toolData<{ readonly status: "resolved" | "missing" }>( | |
| 170 | + parseJsonOutput( | |
| 171 | + await callTool(ctx, ["executor", "coreTools", "secrets", "status"], { id: input.id }), | |
| 172 | + ), | |
| 173 | + ); | |
| 174 | + assert(status.status === "resolved", `Secret ${input.id} was not resolved`); | |
| 175 | +}; | |
| 176 | + | |
| 177 | +const runSmoke = async (oauth: OAuthTestServerShape, graph: GraphqlTestServerShape) => { | |
| 178 | + const root = await mkdtemp(join(tmpdir(), "executor-agent-config-smoke-")); | |
| 179 | + const ctx: CliContext = { | |
| 180 | + dataDir: join(root, "data"), | |
| 181 | + scopeDir: join(root, "scope"), | |
| 182 | + baseUrl: `http://127.0.0.1:${64180 + Math.floor(Math.random() * 1000)}`, | |
| 183 | + }; | |
| 184 | + await mkdir(ctx.dataDir, { recursive: true }); | |
| 185 | + await mkdir(ctx.scopeDir, { recursive: true }); | |
| 186 | + | |
| 187 | + try { | |
| 188 | + console.log("[agent-config-smoke] start dev CLI daemon"); | |
| 189 | + const port = new URL(ctx.baseUrl).port; | |
| 190 | + await runCli(ctx, ["daemon", "run", "--port", port, "--scope", ctx.scopeDir]); | |
| 191 | + | |
| 192 | + console.log("[agent-config-smoke] discover scope"); | |
| 193 | + const scope = firstScope( | |
| 194 | + parseJsonOutput(await callTool(ctx, ["executor", "coreTools", "scopes", "list"])), | |
| 195 | + ); | |
| 196 | + | |
| 197 | + console.log("[agent-config-smoke] create browser secret handoffs"); | |
| 198 | + const clientId = await createSecretPlaceholder(ctx, ctx.scopeDir, scope.id, "OAuth client id"); | |
| 199 | + const clientSecret = await createSecretPlaceholder( | |
| 200 | + ctx, | |
| 201 | + ctx.scopeDir, | |
| 202 | + scope.id, | |
| 203 | + "OAuth client secret", | |
| 204 | + ); | |
| 205 | + const browserBaseUrl = new URL(clientId.url).origin; | |
| 206 | + await setSecretViaBrowserBoundary(ctx, browserBaseUrl, scope.id, { | |
| 207 | + id: clientId.id, | |
| 208 | + name: "OAuth client id", | |
| 209 | + value: "test-client", | |
| 210 | + }); | |
| 211 | + await setSecretViaBrowserBoundary(ctx, browserBaseUrl, scope.id, { | |
| 212 | + id: clientSecret.id, | |
| 213 | + name: "OAuth client secret", | |
| 214 | + value: "test-secret", | |
| 215 | + }); | |
| 216 | + | |
| 217 | + console.log("[agent-config-smoke] start OAuth handoff through core tool"); | |
| 218 | + const oauthStart = toolData<{ | |
| 219 | + readonly sessionId: string; | |
| 220 | + readonly authorizationUrl: string | null; | |
| 221 | + }>( | |
| 222 | + parseJsonOutput( | |
| 223 | + await callTool(ctx, ["executor", "coreTools", "oauth", "start"], { | |
| 224 | + scope: ctx.scopeDir, | |
| 225 | + endpoint: oauth.resourceUrl, | |
| 226 | + connectionId: "agent-smoke-oauth", | |
| 227 | + pluginId: "graphql", | |
| 228 | + identityLabel: "Agent Smoke OAuth", | |
| 229 | + strategy: { | |
| 230 | + kind: "authorization-code", | |
| 231 | + authorizationEndpoint: oauth.authorizationEndpoint, | |
| 232 | + tokenEndpoint: oauth.tokenEndpoint, | |
| 233 | + clientIdSecretId: clientId.id, | |
| 234 | + clientSecretSecretId: clientSecret.id, | |
| 235 | + scopes: ["read"], | |
| 236 | + }, | |
| 237 | + }), | |
| 238 | + ), | |
| 239 | + ); | |
| 240 | + assert(oauthStart.authorizationUrl, "OAuth start did not return an authorization URL"); | |
| 241 | + | |
| 242 | + console.log("[agent-config-smoke] complete OAuth callback through browser URL"); | |
| 243 | + const callback = await Effect.runPromise( | |
| 244 | + oauth.completeAuthorizationCodeFlow({ | |
| 245 | + authorizationUrl: oauthStart.authorizationUrl, | |
| 246 | + }), | |
| 247 | + ); | |
| 248 | + const callbackResponse = await fetch(callback.callbackUrl); | |
| 249 | + const callbackHtml = await callbackResponse.text(); | |
| 250 | + assert(callbackResponse.ok, `OAuth callback failed: ${callbackHtml}`); | |
| 251 | + assert( | |
| 252 | + callbackHtml.includes("Authentication complete"), | |
| 253 | + "OAuth callback did not render success", | |
| 254 | + ); | |
| 255 | + | |
| 256 | + const callbackBaseUrl = new URL(callback.callbackUrl).origin; | |
| 257 | + const pollResponse = await fetch( | |
| 258 | + `${callbackBaseUrl}/api/oauth/await/${encodeURIComponent(callback.state)}`, | |
| 259 | + ); | |
| 260 | + const pollResult = (await pollResponse.json()) as { readonly ok?: boolean }; | |
| 261 | + assert(pollResult.ok === true, `OAuth await result was not ok: ${JSON.stringify(pollResult)}`); | |
| 262 | + | |
| 263 | + const connections = toolData<{ | |
| 264 | + readonly connections: readonly { readonly id: string }[]; | |
| 265 | + }>(parseJsonOutput(await callTool(ctx, ["executor", "coreTools", "connections", "list"]))); | |
| 266 | + assert( | |
| 267 | + connections.connections.some((connection) => connection.id === "agent-smoke-oauth"), | |
| 268 | + `OAuth connection was not listed: ${JSON.stringify(connections)}`, | |
| 269 | + ); | |
| 270 | + | |
| 271 | + console.log("[agent-config-smoke] add OAuth-backed GraphQL source through approval flow"); | |
| 272 | + const addSource = await approvePausedCall( | |
| 273 | + ctx, | |
| 274 | + await callTool(ctx, ["executor", "graphql", "addSource"], { | |
| 275 | + scope: ctx.scopeDir, | |
| 276 | + endpoint: graph.endpoint, | |
| 277 | + name: "Agent Smoke GraphQL", | |
| 278 | + namespace: "agent_smoke_graphql", | |
| 279 | + oauth2: { | |
| 280 | + kind: "oauth2", | |
| 281 | + securitySchemeName: "OAuth2", | |
| 282 | + flow: "authorizationCode", | |
| 283 | + tokenUrl: oauth.tokenEndpoint, | |
| 284 | + authorizationUrl: oauth.authorizationEndpoint, | |
| 285 | + clientIdSlot: "auth:oauth2:client-id", | |
| 286 | + clientSecretSlot: null, | |
| 287 | + connectionSlot: "auth:oauth2:connection", | |
| 288 | + scopes: ["read"], | |
| 289 | + }, | |
| 290 | + credentials: { | |
| 291 | + scope: scope.id, | |
| 292 | + auth: { | |
| 293 | + oauth2: { | |
| 294 | + connection: { kind: "connection", connectionId: "agent-smoke-oauth" }, | |
| 295 | + }, | |
| 296 | + }, | |
| 297 | + }, | |
| 298 | + }), | |
| 299 | + ); | |
| 300 | + const added = parseJsonOutput<{ readonly ok: boolean }>(addSource); | |
| 301 | + assert(added.ok === true, `GraphQL addSource failed: ${addSource.text}`); | |
| 302 | + | |
| 303 | + console.log("[agent-config-smoke] invoke configured OAuth-backed tool"); | |
| 304 | + const invoked = parseJsonOutput<{ readonly ok: boolean; readonly data?: unknown }>( | |
| 305 | + await callTool(ctx, ["agent_smoke_graphql", "query", "hello"], { name: "Ada" }), | |
| 306 | + ); | |
| 307 | + assert( | |
| 308 | + invoked.ok === true && | |
| 309 | + JSON.stringify(invoked.data) === JSON.stringify({ hello: "Hello Ada" }), | |
| 310 | + `GraphQL invocation failed: ${JSON.stringify(invoked)}`, | |
| 311 | + ); | |
| 312 | + | |
| 313 | + console.log("[agent-config-smoke] passed"); | |
| 314 | + } finally { | |
| 315 | + await runCli(ctx, ["daemon", "stop", "--base-url", ctx.baseUrl], { | |
| 316 | + allowFailure: true, | |
| 317 | + }); | |
| 318 | + await rm(root, { recursive: true, force: true }); | |
| 319 | + } | |
| 320 | +}; | |
| 321 | + | |
| 322 | +const main = async () => { | |
| 323 | + await Effect.runPromise( | |
| 324 | + Effect.scoped( | |
| 325 | + Effect.gen(function* () { | |
| 326 | + const oauth = yield* serveOAuthTestServer(); | |
| 327 | + const graph = yield* serveGraphqlTestServer({ | |
| 328 | + schema: makeGreetingGraphqlSchema(), | |
| 329 | + auth: { | |
| 330 | + validateAuthorization: oauth.acceptsAuthorizationHeader, | |
| 331 | + wwwAuthenticate: `Bearer resource_metadata="${oauth.protectedResourceMetadataUrl}/graphql"`, | |
| 332 | + }, | |
| 333 | + }); | |
| 334 | + yield* Effect.promise(() => runSmoke(oauth, graph)); | |
| 335 | + }), | |
| 336 | + ), | |
| 337 | + ); | |
| 338 | +}; | |
| 339 | + | |
| 340 | +await main(); | |