* feat(sdk): add built-in core-tools plugin with agent secret-create flow Introduces `coreToolsPlugin` in @executor-js/sdk exposing three static tools agents can call directly: - `scopes.list` — enumerate visible scopes by name; agents pick a scope by asking the user, not by guessing or defaulting. - `secrets.list` — visible secrets as `{id, name, provider}`; values never cross the agent boundary. - `secrets.create` — pre-allocates a secret id and returns a URL to the existing /secrets web form, pre-filled with name + id. The user enters the value in the browser; the agent confirms by re-listing. The plugin auto-registers when `coreTools: { webBaseUrl }` is set on ExecutorConfig, so hosts opt in with one line. apps/local wires it to the daemon's own port — same host as the API, no separate UI server. The /secrets web side reads the prefill params via TanStack Router `validateSearch`, opens the add modal pre-filled, and locks the id via a new `initialIdOverride` prop on SecretForm. URL prefilling works fully through the dev:cli vite proxy. * fix(sdk): use typed core tools errors * feat: expand agent executor configuration tools * feat: expose source presets to agents * fix: resolve agent source install scopes * fix: clarify secret creation for agents * fix: render open object schema previews * fix: expose typed source configure tools * fix: stabilize agent openapi preview flow * fix: let secret handoff choose scope * fix: explain secret handoff completion * fix: explain oauth handoff completion * fix: clarify oauth credential scope * fix: satisfy core tools ci * fix: clarify preset and resume guidance
Showing the changes to scripts (1 of 54 files). Show all
Unified · Split
| @@ -0,0 +1,340 @@ | |||
| 1 | #!/usr/bin/env bun | ||
| 2 | /** | ||
| 3 | * End-to-end smoke for agent-driven Executor configuration. | ||
| 4 | * | ||
| 5 | * This intentionally drives the public dev CLI for agent actions, while using | ||
| 6 | * the local HTTP browser/API boundary for sensitive values and OAuth callback | ||
| 7 | * completion. Fake test credentials are used, but the shape matches the real | ||
| 8 | * flow: agents get handoff URLs, users/browsers enter secrets and complete | ||
| 9 | * OAuth outside the context window, then agents verify and bind the resulting | ||
| 10 | * ids. | ||
| 11 | */ | ||
| 12 | import { mkdtemp, mkdir, rm } from "node:fs/promises"; | ||
| 13 | import { tmpdir } from "node:os"; | ||
| 14 | import { dirname, join, resolve } from "node:path"; | ||
| 15 | import { fileURLToPath } from "node:url"; | ||
| 16 | |||
| 17 | import { serveOAuthTestServer, type OAuthTestServerShape } from "../packages/core/sdk/src/testing"; | ||
| 18 | import { | ||
| 19 | type GraphqlTestServerShape, | ||
| 20 | makeGreetingGraphqlSchema, | ||
| 21 | serveGraphqlTestServer, | ||
| 22 | } from "../packages/plugins/graphql/src/testing/index"; | ||
| 23 | import { Effect } from "effect"; | ||
| 24 | |||
| 25 | const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); | ||
| 26 | const cliEntrypoint = join(repoRoot, "apps/cli/src/main.ts"); | ||
| 27 | |||
| 28 | type CliContext = { | ||
| 29 | readonly dataDir: string; | ||
| 30 | readonly scopeDir: string; | ||
| 31 | readonly baseUrl: string; | ||
| 32 | }; | ||
| 33 | |||
| 34 | type CliResult = { | ||
| 35 | readonly stdout: string; | ||
| 36 | readonly stderr: string; | ||
| 37 | readonly text: string; | ||
| 38 | readonly exitCode: number; | ||
| 39 | }; | ||
| 40 | |||
| 41 | const assert = (condition: unknown, message: string): asserts condition => { | ||
| 42 | if (!condition) throw new Error(message); | ||
| 43 | }; | ||
| 44 | |||
| 45 | const runCli = async ( | ||
| 46 | ctx: CliContext, | ||
| 47 | args: readonly string[], | ||
| 48 | options: { readonly allowFailure?: boolean } = {}, | ||
| 49 | ): Promise<CliResult> => { | ||
| 50 | const proc = Bun.spawn([process.execPath, "run", cliEntrypoint, ...args], { | ||
| 51 | cwd: repoRoot, | ||
| 52 | env: { | ||
| 53 | ...process.env, | ||
| 54 | EXECUTOR_DEV: "1", | ||
| 55 | EXECUTOR_DATA_DIR: ctx.dataDir, | ||
| 56 | EXECUTOR_SCOPE_DIR: ctx.scopeDir, | ||
| 57 | }, | ||
| 58 | stdout: "pipe", | ||
| 59 | stderr: "pipe", | ||
| 60 | }); | ||
| 61 | |||
| 62 | const [stdout, stderr, exitCode] = await Promise.all([ | ||
| 63 | new Response(proc.stdout).text(), | ||
| 64 | new Response(proc.stderr).text(), | ||
| 65 | proc.exited, | ||
| 66 | ]); | ||
| 67 | const result = { stdout, stderr, text: `${stdout}${stderr}`, exitCode }; | ||
| 68 | |||
| 69 | if (exitCode !== 0 && !options.allowFailure) { | ||
| 70 | throw new Error(`CLI failed (${exitCode}) for ${args.join(" ")}\n${result.text}`); | ||
| 71 | } | ||
| 72 | |||
| 73 | return result; | ||
| 74 | }; | ||
| 75 | |||
| 76 | const parseJsonOutput = <T>(result: CliResult): T => { | ||
| 77 | const start = result.stdout.indexOf("{"); | ||
| 78 | const end = result.stdout.lastIndexOf("}"); | ||
| 79 | assert(start >= 0 && end > start, `CLI output did not contain JSON:\n${result.text}`); | ||
| 80 | return JSON.parse(result.stdout.slice(start, end + 1)) as T; | ||
| 81 | }; | ||
| 82 | |||
| 83 | const toolData = <T>(value: unknown): T => { | ||
| 84 | const result = value as { readonly ok?: boolean; readonly data?: T; readonly error?: unknown }; | ||
| 85 | assert(result.ok === true, `Tool returned failure: ${JSON.stringify(value)}`); | ||
| 86 | return result.data as T; | ||
| 87 | }; | ||
| 88 | |||
| 89 | const extractExecutionId = (text: string): string => { | ||
| 90 | const match = /executionId:\s*(exec_[A-Za-z0-9_]+)/.exec(text); | ||
| 91 | assert(match?.[1], `CLI output did not contain an execution id:\n${text}`); | ||
| 92 | return match[1]; | ||
| 93 | }; | ||
| 94 | |||
| 95 | const callTool = (ctx: CliContext, path: readonly string[], args: unknown = {}) => | ||
| 96 | runCli(ctx, [ | ||
| 97 | "call", | ||
| 98 | ...path, | ||
| 99 | JSON.stringify(args), | ||
| 100 | "--base-url", | ||
| 101 | ctx.baseUrl, | ||
| 102 | "--scope", | ||
| 103 | ctx.scopeDir, | ||
| 104 | ]); | ||
| 105 | |||
| 106 | const approvePausedCall = async (ctx: CliContext, paused: CliResult): Promise<CliResult> => { | ||
| 107 | const executionId = extractExecutionId(paused.text); | ||
| 108 | return await runCli(ctx, [ | ||
| 109 | "resume", | ||
| 110 | "--execution-id", | ||
| 111 | executionId, | ||
| 112 | "--base-url", | ||
| 113 | ctx.baseUrl, | ||
| 114 | "--scope", | ||
| 115 | ctx.scopeDir, | ||
| 116 | ]); | ||
| 117 | }; | ||
| 118 | |||
| 119 | const postJson = async (url: string, payload: unknown): Promise<unknown> => { | ||
| 120 | const response = await fetch(url, { | ||
| 121 | method: "POST", | ||
| 122 | headers: { "content-type": "application/json" }, | ||
| 123 | body: JSON.stringify(payload), | ||
| 124 | }); | ||
| 125 | const bodyText = await response.text(); | ||
| 126 | if (!response.ok) { | ||
| 127 | throw new Error(`POST ${url} failed with HTTP ${response.status}: ${bodyText}`); | ||
| 128 | } | ||
| 129 | return bodyText.length > 0 ? JSON.parse(bodyText) : null; | ||
| 130 | }; | ||
| 131 | |||
| 132 | const firstScope = (value: unknown): { readonly id: string; readonly name: string } => { | ||
| 133 | const data = toolData<{ | ||
| 134 | readonly scopes: readonly { readonly id: string; readonly name: string }[]; | ||
| 135 | }>(value); | ||
| 136 | const scopes = data.scopes; | ||
| 137 | const scope = scopes?.[0]; | ||
| 138 | assert(scope, `scopes.list returned no scopes: ${JSON.stringify(value)}`); | ||
| 139 | return scope; | ||
| 140 | }; | ||
| 141 | |||
| 142 | const createSecretPlaceholder = async ( | ||
| 143 | ctx: CliContext, | ||
| 144 | scope: string, | ||
| 145 | expectedHandoffScope: string, | ||
| 146 | name: string, | ||
| 147 | ) => { | ||
| 148 | const result = toolData<{ readonly id: string; readonly url: string }>( | ||
| 149 | parseJsonOutput( | ||
| 150 | await callTool(ctx, ["executor", "coreTools", "secrets", "create"], { name, scope }), | ||
| 151 | ), | ||
| 152 | ); | ||
| 153 | const handoff = new URL(result.url); | ||
| 154 | assert(handoff.pathname === "/secrets", `Expected /secrets handoff URL, got ${result.url}`); | ||
| 155 | assert( | ||
| 156 | handoff.searchParams.get("scope") === expectedHandoffScope, | ||
| 157 | `Secret handoff omitted scope: ${result.url}`, | ||
| 158 | ); | ||
| 159 | return result; | ||
| 160 | }; | ||
| 161 | |||
| 162 | const setSecretViaBrowserBoundary = async ( | ||
| 163 | ctx: CliContext, | ||
| 164 | browserBaseUrl: string, | ||
| 165 | scopeId: string, | ||
| 166 | input: { readonly id: string; readonly name: string; readonly value: string }, | ||
| 167 | ) => { | ||
| 168 | await postJson(`${browserBaseUrl}/api/scopes/${encodeURIComponent(scopeId)}/secrets`, input); | ||
| 169 | const status = toolData<{ readonly status: "resolved" | "missing" }>( | ||
| 170 | parseJsonOutput( | ||
| 171 | await callTool(ctx, ["executor", "coreTools", "secrets", "status"], { id: input.id }), | ||
| 172 | ), | ||
| 173 | ); | ||
| 174 | assert(status.status === "resolved", `Secret ${input.id} was not resolved`); | ||
| 175 | }; | ||
| 176 | |||
| 177 | const runSmoke = async (oauth: OAuthTestServerShape, graph: GraphqlTestServerShape) => { | ||
| 178 | const root = await mkdtemp(join(tmpdir(), "executor-agent-config-smoke-")); | ||
| 179 | const ctx: CliContext = { | ||
| 180 | dataDir: join(root, "data"), | ||
| 181 | scopeDir: join(root, "scope"), | ||
| 182 | baseUrl: `http://127.0.0.1:${64180 + Math.floor(Math.random() * 1000)}`, | ||
| 183 | }; | ||
| 184 | await mkdir(ctx.dataDir, { recursive: true }); | ||
| 185 | await mkdir(ctx.scopeDir, { recursive: true }); | ||
| 186 | |||
| 187 | try { | ||
| 188 | console.log("[agent-config-smoke] start dev CLI daemon"); | ||
| 189 | const port = new URL(ctx.baseUrl).port; | ||
| 190 | await runCli(ctx, ["daemon", "run", "--port", port, "--scope", ctx.scopeDir]); | ||
| 191 | |||
| 192 | console.log("[agent-config-smoke] discover scope"); | ||
| 193 | const scope = firstScope( | ||
| 194 | parseJsonOutput(await callTool(ctx, ["executor", "coreTools", "scopes", "list"])), | ||
| 195 | ); | ||
| 196 | |||
| 197 | console.log("[agent-config-smoke] create browser secret handoffs"); | ||
| 198 | const clientId = await createSecretPlaceholder(ctx, ctx.scopeDir, scope.id, "OAuth client id"); | ||
| 199 | const clientSecret = await createSecretPlaceholder( | ||
| 200 | ctx, | ||
| 201 | ctx.scopeDir, | ||
| 202 | scope.id, | ||
| 203 | "OAuth client secret", | ||
| 204 | ); | ||
| 205 | const browserBaseUrl = new URL(clientId.url).origin; | ||
| 206 | await setSecretViaBrowserBoundary(ctx, browserBaseUrl, scope.id, { | ||
| 207 | id: clientId.id, | ||
| 208 | name: "OAuth client id", | ||
| 209 | value: "test-client", | ||
| 210 | }); | ||
| 211 | await setSecretViaBrowserBoundary(ctx, browserBaseUrl, scope.id, { | ||
| 212 | id: clientSecret.id, | ||
| 213 | name: "OAuth client secret", | ||
| 214 | value: "test-secret", | ||
| 215 | }); | ||
| 216 | |||
| 217 | console.log("[agent-config-smoke] start OAuth handoff through core tool"); | ||
| 218 | const oauthStart = toolData<{ | ||
| 219 | readonly sessionId: string; | ||
| 220 | readonly authorizationUrl: string | null; | ||
| 221 | }>( | ||
| 222 | parseJsonOutput( | ||
| 223 | await callTool(ctx, ["executor", "coreTools", "oauth", "start"], { | ||
| 224 | scope: ctx.scopeDir, | ||
| 225 | endpoint: oauth.resourceUrl, | ||
| 226 | connectionId: "agent-smoke-oauth", | ||
| 227 | pluginId: "graphql", | ||
| 228 | identityLabel: "Agent Smoke OAuth", | ||
| 229 | strategy: { | ||
| 230 | kind: "authorization-code", | ||
| 231 | authorizationEndpoint: oauth.authorizationEndpoint, | ||
| 232 | tokenEndpoint: oauth.tokenEndpoint, | ||
| 233 | clientIdSecretId: clientId.id, | ||
| 234 | clientSecretSecretId: clientSecret.id, | ||
| 235 | scopes: ["read"], | ||
| 236 | }, | ||
| 237 | }), | ||
| 238 | ), | ||
| 239 | ); | ||
| 240 | assert(oauthStart.authorizationUrl, "OAuth start did not return an authorization URL"); | ||
| 241 | |||
| 242 | console.log("[agent-config-smoke] complete OAuth callback through browser URL"); | ||
| 243 | const callback = await Effect.runPromise( | ||
| 244 | oauth.completeAuthorizationCodeFlow({ | ||
| 245 | authorizationUrl: oauthStart.authorizationUrl, | ||
| 246 | }), | ||
| 247 | ); | ||
| 248 | const callbackResponse = await fetch(callback.callbackUrl); | ||
| 249 | const callbackHtml = await callbackResponse.text(); | ||
| 250 | assert(callbackResponse.ok, `OAuth callback failed: ${callbackHtml}`); | ||
| 251 | assert( | ||
| 252 | callbackHtml.includes("Authentication complete"), | ||
| 253 | "OAuth callback did not render success", | ||
| 254 | ); | ||
| 255 | |||
| 256 | const callbackBaseUrl = new URL(callback.callbackUrl).origin; | ||
| 257 | const pollResponse = await fetch( | ||
| 258 | `${callbackBaseUrl}/api/oauth/await/${encodeURIComponent(callback.state)}`, | ||
| 259 | ); | ||
| 260 | const pollResult = (await pollResponse.json()) as { readonly ok?: boolean }; | ||
| 261 | assert(pollResult.ok === true, `OAuth await result was not ok: ${JSON.stringify(pollResult)}`); | ||
| 262 | |||
| 263 | const connections = toolData<{ | ||
| 264 | readonly connections: readonly { readonly id: string }[]; | ||
| 265 | }>(parseJsonOutput(await callTool(ctx, ["executor", "coreTools", "connections", "list"]))); | ||
| 266 | assert( | ||
| 267 | connections.connections.some((connection) => connection.id === "agent-smoke-oauth"), | ||
| 268 | `OAuth connection was not listed: ${JSON.stringify(connections)}`, | ||
| 269 | ); | ||
| 270 | |||
| 271 | console.log("[agent-config-smoke] add OAuth-backed GraphQL source through approval flow"); | ||
| 272 | const addSource = await approvePausedCall( | ||
| 273 | ctx, | ||
| 274 | await callTool(ctx, ["executor", "graphql", "addSource"], { | ||
| 275 | scope: ctx.scopeDir, | ||
| 276 | endpoint: graph.endpoint, | ||
| 277 | name: "Agent Smoke GraphQL", | ||
| 278 | namespace: "agent_smoke_graphql", | ||
| 279 | oauth2: { | ||
| 280 | kind: "oauth2", | ||
| 281 | securitySchemeName: "OAuth2", | ||
| 282 | flow: "authorizationCode", | ||
| 283 | tokenUrl: oauth.tokenEndpoint, | ||
| 284 | authorizationUrl: oauth.authorizationEndpoint, | ||
| 285 | clientIdSlot: "auth:oauth2:client-id", | ||
| 286 | clientSecretSlot: null, | ||
| 287 | connectionSlot: "auth:oauth2:connection", | ||
| 288 | scopes: ["read"], | ||
| 289 | }, | ||
| 290 | credentials: { | ||
| 291 | scope: scope.id, | ||
| 292 | auth: { | ||
| 293 | oauth2: { | ||
| 294 | connection: { kind: "connection", connectionId: "agent-smoke-oauth" }, | ||
| 295 | }, | ||
| 296 | }, | ||
| 297 | }, | ||
| 298 | }), | ||
| 299 | ); | ||
| 300 | const added = parseJsonOutput<{ readonly ok: boolean }>(addSource); | ||
| 301 | assert(added.ok === true, `GraphQL addSource failed: ${addSource.text}`); | ||
| 302 | |||
| 303 | console.log("[agent-config-smoke] invoke configured OAuth-backed tool"); | ||
| 304 | const invoked = parseJsonOutput<{ readonly ok: boolean; readonly data?: unknown }>( | ||
| 305 | await callTool(ctx, ["agent_smoke_graphql", "query", "hello"], { name: "Ada" }), | ||
| 306 | ); | ||
| 307 | assert( | ||
| 308 | invoked.ok === true && | ||
| 309 | JSON.stringify(invoked.data) === JSON.stringify({ hello: "Hello Ada" }), | ||
| 310 | `GraphQL invocation failed: ${JSON.stringify(invoked)}`, | ||
| 311 | ); | ||
| 312 | |||
| 313 | console.log("[agent-config-smoke] passed"); | ||
| 314 | } finally { | ||
| 315 | await runCli(ctx, ["daemon", "stop", "--base-url", ctx.baseUrl], { | ||
| 316 | allowFailure: true, | ||
| 317 | }); | ||
| 318 | await rm(root, { recursive: true, force: true }); | ||
| 319 | } | ||
| 320 | }; | ||
| 321 | |||
| 322 | const main = async () => { | ||
| 323 | await Effect.runPromise( | ||
| 324 | Effect.scoped( | ||
| 325 | Effect.gen(function* () { | ||
| 326 | const oauth = yield* serveOAuthTestServer(); | ||
| 327 | const graph = yield* serveGraphqlTestServer({ | ||
| 328 | schema: makeGreetingGraphqlSchema(), | ||
| 329 | auth: { | ||
| 330 | validateAuthorization: oauth.acceptsAuthorizationHeader, | ||
| 331 | wwwAuthenticate: `Bearer resource_metadata="${oauth.protectedResourceMetadataUrl}/graphql"`, | ||
| 332 | }, | ||
| 333 | }); | ||
| 334 | yield* Effect.promise(() => runSmoke(oauth, graph)); | ||
| 335 | }), | ||
| 336 | ), | ||
| 337 | ); | ||
| 338 | }; | ||
| 339 | |||
| 340 | await main(); | ||