Showing the changes to scripts (1 of 3 files). Show all
Unified · Split
| @@ -1,511 +0,0 @@ | |||
| 1 | import { spawn } from "node:child_process"; | ||
| 2 | import fs from "node:fs/promises"; | ||
| 3 | import path from "node:path"; | ||
| 4 | import readline from "node:readline"; | ||
| 5 | |||
| 6 | const DEFAULT_PROMPT_TIMEOUT_MS = 120_000; | ||
| 7 | const CANCEL_AFTER_MS = 1_500; | ||
| 8 | |||
| 9 | function parseArgv(argv) { | ||
| 10 | const parsed = { | ||
| 11 | outputDir: "", | ||
| 12 | workspace: process.cwd(), | ||
| 13 | model: "", | ||
| 14 | permissionOption: "allow-once", | ||
| 15 | }; | ||
| 16 | |||
| 17 | for (let i = 0; i < argv.length; i += 1) { | ||
| 18 | const token = argv[i]; | ||
| 19 | const next = argv[i + 1]; | ||
| 20 | if ((token === "--output-dir" || token === "-o") && next) { | ||
| 21 | parsed.outputDir = next; | ||
| 22 | i += 1; | ||
| 23 | continue; | ||
| 24 | } | ||
| 25 | if ((token === "--workspace" || token === "-w") && next) { | ||
| 26 | parsed.workspace = path.resolve(next); | ||
| 27 | i += 1; | ||
| 28 | continue; | ||
| 29 | } | ||
| 30 | if ((token === "--model" || token === "-m") && next) { | ||
| 31 | parsed.model = next; | ||
| 32 | i += 1; | ||
| 33 | continue; | ||
| 34 | } | ||
| 35 | if (token === "--permission-option" && next) { | ||
| 36 | parsed.permissionOption = next; | ||
| 37 | i += 1; | ||
| 38 | } | ||
| 39 | } | ||
| 40 | |||
| 41 | return parsed; | ||
| 42 | } | ||
| 43 | |||
| 44 | function nowIso() { | ||
| 45 | return new Date().toISOString(); | ||
| 46 | } | ||
| 47 | |||
| 48 | function safeJsonParse(line) { | ||
| 49 | try { | ||
| 50 | return { ok: true, value: JSON.parse(line) }; | ||
| 51 | } catch (error) { | ||
| 52 | return { ok: false, error }; | ||
| 53 | } | ||
| 54 | } | ||
| 55 | |||
| 56 | class AcpProbeClient { | ||
| 57 | #child; | ||
| 58 | #stdoutRl; | ||
| 59 | #stderrRl; | ||
| 60 | #nextId = 1; | ||
| 61 | #closed = false; | ||
| 62 | #pending = new Map(); | ||
| 63 | #onMessage; | ||
| 64 | #onServerRequest; | ||
| 65 | |||
| 66 | constructor({ onMessage, onServerRequest }) { | ||
| 67 | this.#onMessage = onMessage; | ||
| 68 | this.#onServerRequest = onServerRequest; | ||
| 69 | |||
| 70 | this.#child = spawn("agent", ["acp"], { | ||
| 71 | stdio: ["pipe", "pipe", "pipe"], | ||
| 72 | env: { | ||
| 73 | ...process.env, | ||
| 74 | NO_COLOR: "1", | ||
| 75 | }, | ||
| 76 | }); | ||
| 77 | |||
| 78 | this.#stdoutRl = readline.createInterface({ input: this.#child.stdout }); | ||
| 79 | this.#stderrRl = readline.createInterface({ input: this.#child.stderr }); | ||
| 80 | |||
| 81 | this.#stdoutRl.on("line", (line) => this.#handleStdoutLine(line)); | ||
| 82 | this.#stderrRl.on("line", (line) => { | ||
| 83 | this.#onMessage({ | ||
| 84 | ts: nowIso(), | ||
| 85 | channel: "stderr", | ||
| 86 | line, | ||
| 87 | }); | ||
| 88 | }); | ||
| 89 | |||
| 90 | this.#child.once("exit", (code, signal) => { | ||
| 91 | this.#closed = true; | ||
| 92 | const reason = `ACP process exited (code=${String(code)}, signal=${String(signal)})`; | ||
| 93 | for (const [id, pending] of this.#pending.entries()) { | ||
| 94 | this.#pending.delete(id); | ||
| 95 | pending.reject(new Error(reason)); | ||
| 96 | } | ||
| 97 | this.#onMessage({ | ||
| 98 | ts: nowIso(), | ||
| 99 | channel: "lifecycle", | ||
| 100 | event: "exit", | ||
| 101 | code, | ||
| 102 | signal, | ||
| 103 | }); | ||
| 104 | }); | ||
| 105 | } | ||
| 106 | |||
| 107 | async close() { | ||
| 108 | if (this.#closed) return; | ||
| 109 | try { | ||
| 110 | this.#child.stdin.end(); | ||
| 111 | } catch { | ||
| 112 | // ignored | ||
| 113 | } | ||
| 114 | this.#child.kill("SIGTERM"); | ||
| 115 | await new Promise((resolve) => setTimeout(resolve, 100)); | ||
| 116 | if (!this.#closed) { | ||
| 117 | this.#child.kill("SIGKILL"); | ||
| 118 | } | ||
| 119 | } | ||
| 120 | |||
| 121 | async send(method, params, { timeoutMs = DEFAULT_PROMPT_TIMEOUT_MS } = {}) { | ||
| 122 | if (this.#closed) { | ||
| 123 | throw new Error("Cannot send: ACP process is already closed."); | ||
| 124 | } | ||
| 125 | const id = this.#nextId; | ||
| 126 | this.#nextId += 1; | ||
| 127 | |||
| 128 | const message = { | ||
| 129 | jsonrpc: "2.0", | ||
| 130 | id, | ||
| 131 | method, | ||
| 132 | params, | ||
| 133 | }; | ||
| 134 | |||
| 135 | this.#onMessage({ | ||
| 136 | ts: nowIso(), | ||
| 137 | channel: "client->server", | ||
| 138 | message, | ||
| 139 | }); | ||
| 140 | this.#child.stdin.write(`${JSON.stringify(message)}\n`); | ||
| 141 | |||
| 142 | return new Promise((resolve, reject) => { | ||
| 143 | const timeout = setTimeout(() => { | ||
| 144 | this.#pending.delete(id); | ||
| 145 | reject(new Error(`Timed out waiting for response to '${method}' (id=${id}).`)); | ||
| 146 | }, timeoutMs); | ||
| 147 | |||
| 148 | this.#pending.set(id, { | ||
| 149 | resolve: (value) => { | ||
| 150 | clearTimeout(timeout); | ||
| 151 | resolve(value); | ||
| 152 | }, | ||
| 153 | reject: (error) => { | ||
| 154 | clearTimeout(timeout); | ||
| 155 | reject(error); | ||
| 156 | }, | ||
| 157 | }); | ||
| 158 | }); | ||
| 159 | } | ||
| 160 | |||
| 161 | #respond(id, result) { | ||
| 162 | if (this.#closed) return; | ||
| 163 | const message = { | ||
| 164 | jsonrpc: "2.0", | ||
| 165 | id, | ||
| 166 | result, | ||
| 167 | }; | ||
| 168 | this.#onMessage({ | ||
| 169 | ts: nowIso(), | ||
| 170 | channel: "client->server", | ||
| 171 | message, | ||
| 172 | }); | ||
| 173 | this.#child.stdin.write(`${JSON.stringify(message)}\n`); | ||
| 174 | } | ||
| 175 | |||
| 176 | #handleStdoutLine(line) { | ||
| 177 | const parsed = safeJsonParse(line); | ||
| 178 | if (!parsed.ok) { | ||
| 179 | this.#onMessage({ | ||
| 180 | ts: nowIso(), | ||
| 181 | channel: "stdout-non-json", | ||
| 182 | line, | ||
| 183 | }); | ||
| 184 | return; | ||
| 185 | } | ||
| 186 | |||
| 187 | const message = parsed.value; | ||
| 188 | this.#onMessage({ | ||
| 189 | ts: nowIso(), | ||
| 190 | channel: "server->client", | ||
| 191 | message, | ||
| 192 | }); | ||
| 193 | |||
| 194 | if ( | ||
| 195 | Object.prototype.hasOwnProperty.call(message, "id") && | ||
| 196 | (Object.prototype.hasOwnProperty.call(message, "result") || | ||
| 197 | Object.prototype.hasOwnProperty.call(message, "error")) && | ||
| 198 | !Object.prototype.hasOwnProperty.call(message, "method") | ||
| 199 | ) { | ||
| 200 | const pending = this.#pending.get(message.id); | ||
| 201 | if (!pending) return; | ||
| 202 | this.#pending.delete(message.id); | ||
| 203 | if (Object.prototype.hasOwnProperty.call(message, "error")) { | ||
| 204 | pending.reject(message.error); | ||
| 205 | return; | ||
| 206 | } | ||
| 207 | pending.resolve(message.result); | ||
| 208 | return; | ||
| 209 | } | ||
| 210 | |||
| 211 | if ( | ||
| 212 | Object.prototype.hasOwnProperty.call(message, "id") && | ||
| 213 | Object.prototype.hasOwnProperty.call(message, "method") | ||
| 214 | ) { | ||
| 215 | this.#onServerRequest?.(message, (result) => this.#respond(message.id, result)); | ||
| 216 | } | ||
| 217 | } | ||
| 218 | } | ||
| 219 | |||
| 220 | function summarizeTranscript(entries) { | ||
| 221 | const summary = { | ||
| 222 | counts: { | ||
| 223 | notificationsByMethod: {}, | ||
| 224 | sessionUpdateByType: {}, | ||
| 225 | serverRequestsByMethod: {}, | ||
| 226 | permissionDecisions: {}, | ||
| 227 | }, | ||
| 228 | samples: { | ||
| 229 | initializeResult: null, | ||
| 230 | authenticateResult: null, | ||
| 231 | sessionNewResult: null, | ||
| 232 | sessionPromptResultByScenario: {}, | ||
| 233 | sessionUpdateByType: {}, | ||
| 234 | serverRequestByMethod: {}, | ||
| 235 | }, | ||
| 236 | stderr: [], | ||
| 237 | }; | ||
| 238 | |||
| 239 | for (const entry of entries) { | ||
| 240 | if (entry.channel === "stderr") { | ||
| 241 | summary.stderr.push(entry.line); | ||
| 242 | continue; | ||
| 243 | } | ||
| 244 | if (entry.channel !== "server->client") continue; | ||
| 245 | |||
| 246 | const message = entry.message; | ||
| 247 | if (!message || typeof message !== "object") continue; | ||
| 248 | |||
| 249 | if (typeof message.method === "string" && !Object.prototype.hasOwnProperty.call(message, "id")) { | ||
| 250 | summary.counts.notificationsByMethod[message.method] = | ||
| 251 | (summary.counts.notificationsByMethod[message.method] ?? 0) + 1; | ||
| 252 | if (message.method === "session/update") { | ||
| 253 | const updateType = message.params?.update?.sessionUpdate; | ||
| 254 | if (typeof updateType === "string") { | ||
| 255 | summary.counts.sessionUpdateByType[updateType] = | ||
| 256 | (summary.counts.sessionUpdateByType[updateType] ?? 0) + 1; | ||
| 257 | if (!summary.samples.sessionUpdateByType[updateType]) { | ||
| 258 | summary.samples.sessionUpdateByType[updateType] = message; | ||
| 259 | } | ||
| 260 | } | ||
| 261 | } | ||
| 262 | continue; | ||
| 263 | } | ||
| 264 | |||
| 265 | if (typeof message.method === "string" && Object.prototype.hasOwnProperty.call(message, "id")) { | ||
| 266 | summary.counts.serverRequestsByMethod[message.method] = | ||
| 267 | (summary.counts.serverRequestsByMethod[message.method] ?? 0) + 1; | ||
| 268 | if (!summary.samples.serverRequestByMethod[message.method]) { | ||
| 269 | summary.samples.serverRequestByMethod[message.method] = message; | ||
| 270 | } | ||
| 271 | continue; | ||
| 272 | } | ||
| 273 | } | ||
| 274 | |||
| 275 | for (const entry of entries) { | ||
| 276 | if (entry.channel !== "scenario-result") continue; | ||
| 277 | if (entry.scenario === "initialize" && !summary.samples.initializeResult) { | ||
| 278 | summary.samples.initializeResult = entry.result; | ||
| 279 | continue; | ||
| 280 | } | ||
| 281 | if (entry.scenario === "authenticate" && !summary.samples.authenticateResult) { | ||
| 282 | summary.samples.authenticateResult = entry.result; | ||
| 283 | continue; | ||
| 284 | } | ||
| 285 | if (entry.scenario === "session/new" && !summary.samples.sessionNewResult) { | ||
| 286 | summary.samples.sessionNewResult = entry.result; | ||
| 287 | continue; | ||
| 288 | } | ||
| 289 | if (entry.scenarioName) { | ||
| 290 | summary.samples.sessionPromptResultByScenario[entry.scenarioName] = entry.result; | ||
| 291 | } | ||
| 292 | } | ||
| 293 | |||
| 294 | for (const entry of entries) { | ||
| 295 | if (entry.channel !== "permission-decision") continue; | ||
| 296 | const optionId = entry.optionId; | ||
| 297 | summary.counts.permissionDecisions[optionId] = | ||
| 298 | (summary.counts.permissionDecisions[optionId] ?? 0) + 1; | ||
| 299 | } | ||
| 300 | |||
| 301 | return summary; | ||
| 302 | } | ||
| 303 | |||
| 304 | async function run() { | ||
| 305 | const args = parseArgv(process.argv.slice(2)); | ||
| 306 | const allowedPermissionOptions = new Set(["allow-once", "allow-always", "reject-once"]); | ||
| 307 | if (!allowedPermissionOptions.has(args.permissionOption)) { | ||
| 308 | throw new Error( | ||
| 309 | `Invalid --permission-option '${args.permissionOption}'. Expected one of: ${Array.from(allowedPermissionOptions).join(", ")}`, | ||
| 310 | ); | ||
| 311 | } | ||
| 312 | const stamp = nowIso().replaceAll(":", "-"); | ||
| 313 | const outputDir = args.outputDir | ||
| 314 | ? path.resolve(args.outputDir) | ||
| 315 | : path.join(process.cwd(), ".tmp", "acp-probe", stamp); | ||
| 316 | await fs.mkdir(outputDir, { recursive: true }); | ||
| 317 | |||
| 318 | const transcript = []; | ||
| 319 | const pushEntry = (entry) => transcript.push(entry); | ||
| 320 | |||
| 321 | let activeScenarioName = null; | ||
| 322 | |||
| 323 | const client = new AcpProbeClient({ | ||
| 324 | onMessage: (entry) => { | ||
| 325 | if ( | ||
| 326 | activeScenarioName && | ||
| 327 | entry.channel === "server->client" && | ||
| 328 | entry.message?.method === "session/update" | ||
| 329 | ) { | ||
| 330 | transcript.push({ ...entry, scenarioName: activeScenarioName }); | ||
| 331 | return; | ||
| 332 | } | ||
| 333 | transcript.push(entry); | ||
| 334 | }, | ||
| 335 | onServerRequest: (message, respond) => { | ||
| 336 | if (message.method === "session/request_permission") { | ||
| 337 | const defaultChoice = args.permissionOption; | ||
| 338 | pushEntry({ | ||
| 339 | ts: nowIso(), | ||
| 340 | channel: "permission-decision", | ||
| 341 | requestId: message.id, | ||
| 342 | optionId: defaultChoice, | ||
| 343 | params: message.params, | ||
| 344 | }); | ||
| 345 | respond({ | ||
| 346 | outcome: { | ||
| 347 | outcome: "selected", | ||
| 348 | optionId: defaultChoice, | ||
| 349 | }, | ||
| 350 | }); | ||
| 351 | return; | ||
| 352 | } | ||
| 353 | |||
| 354 | respond({ | ||
| 355 | outcome: { | ||
| 356 | outcome: "selected", | ||
| 357 | optionId: "deny", | ||
| 358 | }, | ||
| 359 | }); | ||
| 360 | pushEntry({ | ||
| 361 | ts: nowIso(), | ||
| 362 | channel: "server-request-unhandled", | ||
| 363 | method: message.method, | ||
| 364 | requestId: message.id, | ||
| 365 | params: message.params, | ||
| 366 | }); | ||
| 367 | }, | ||
| 368 | }); | ||
| 369 | |||
| 370 | try { | ||
| 371 | const initializeResult = await client.send("initialize", { | ||
| 372 | protocolVersion: 1, | ||
| 373 | clientCapabilities: { | ||
| 374 | fs: { readTextFile: false, writeTextFile: false }, | ||
| 375 | terminal: false, | ||
| 376 | }, | ||
| 377 | clientInfo: { | ||
| 378 | name: "t3-cursor-acp-probe", | ||
| 379 | version: "0.1.0", | ||
| 380 | }, | ||
| 381 | }); | ||
| 382 | pushEntry({ | ||
| 383 | ts: nowIso(), | ||
| 384 | channel: "scenario-result", | ||
| 385 | scenario: "initialize", | ||
| 386 | result: initializeResult, | ||
| 387 | }); | ||
| 388 | |||
| 389 | const authenticateResult = await client.send("authenticate", { | ||
| 390 | methodId: "cursor_login", | ||
| 391 | }); | ||
| 392 | pushEntry({ | ||
| 393 | ts: nowIso(), | ||
| 394 | channel: "scenario-result", | ||
| 395 | scenario: "authenticate", | ||
| 396 | result: authenticateResult, | ||
| 397 | }); | ||
| 398 | |||
| 399 | const sessionParams = { | ||
| 400 | cwd: args.workspace, | ||
| 401 | mcpServers: [], | ||
| 402 | ...(args.model ? { model: args.model } : {}), | ||
| 403 | }; | ||
| 404 | const sessionResult = await client.send("session/new", sessionParams); | ||
| 405 | pushEntry({ | ||
| 406 | ts: nowIso(), | ||
| 407 | channel: "scenario-result", | ||
| 408 | scenario: "session/new", | ||
| 409 | result: sessionResult, | ||
| 410 | }); | ||
| 411 | |||
| 412 | const sessionId = sessionResult?.sessionId; | ||
| 413 | if (typeof sessionId !== "string" || sessionId.length === 0) { | ||
| 414 | throw new Error(`Missing sessionId from session/new response: ${JSON.stringify(sessionResult)}`); | ||
| 415 | } | ||
| 416 | |||
| 417 | const scenarios = [ | ||
| 418 | { | ||
| 419 | name: "hello", | ||
| 420 | prompt: "Say hello in one sentence.", | ||
| 421 | }, | ||
| 422 | { | ||
| 423 | name: "tooling", | ||
| 424 | prompt: | ||
| 425 | "Use tools to run `pwd` and then `ls -1 | head -n 8`, and summarize what you found in one paragraph.", | ||
| 426 | }, | ||
| 427 | { | ||
| 428 | name: "cancel", | ||
| 429 | prompt: | ||
| 430 | "Think for a while and draft a long detailed migration plan with at least 20 bullet points before answering.", | ||
| 431 | cancelAfterMs: CANCEL_AFTER_MS, | ||
| 432 | }, | ||
| 433 | ]; | ||
| 434 | |||
| 435 | for (const scenario of scenarios) { | ||
| 436 | activeScenarioName = scenario.name; | ||
| 437 | const promptParams = { | ||
| 438 | sessionId, | ||
| 439 | prompt: [{ type: "text", text: scenario.prompt }], | ||
| 440 | }; | ||
| 441 | |||
| 442 | const promptPromise = client.send("session/prompt", promptParams, { | ||
| 443 | timeoutMs: DEFAULT_PROMPT_TIMEOUT_MS, | ||
| 444 | }); | ||
| 445 | |||
| 446 | if (scenario.cancelAfterMs) { | ||
| 447 | setTimeout(() => { | ||
| 448 | client | ||
| 449 | .send("session/cancel", { sessionId }, { timeoutMs: 15_000 }) | ||
| 450 | .then((cancelResult) => { | ||
| 451 | pushEntry({ | ||
| 452 | ts: nowIso(), | ||
| 453 | channel: "scenario-result", | ||
| 454 | scenario: "session/cancel", | ||
| 455 | scenarioName: scenario.name, | ||
| 456 | result: cancelResult, | ||
| 457 | }); | ||
| 458 | }) | ||
| 459 | .catch((error) => { | ||
| 460 | pushEntry({ | ||
| 461 | ts: nowIso(), | ||
| 462 | channel: "scenario-error", | ||
| 463 | scenario: "session/cancel", | ||
| 464 | scenarioName: scenario.name, | ||
| 465 | error: | ||
| 466 | error instanceof Error | ||
| 467 | ? error.message | ||
| 468 | : typeof error === "string" | ||
| 469 | ? error | ||
| 470 | : JSON.stringify(error), | ||
| 471 | }); | ||
| 472 | }); | ||
| 473 | }, scenario.cancelAfterMs); | ||
| 474 | } | ||
| 475 | |||
| 476 | const promptResult = await promptPromise; | ||
| 477 | pushEntry({ | ||
| 478 | ts: nowIso(), | ||
| 479 | channel: "scenario-result", | ||
| 480 | scenario: "session/prompt", | ||
| 481 | scenarioName: scenario.name, | ||
| 482 | result: promptResult, | ||
| 483 | }); | ||
| 484 | activeScenarioName = null; | ||
| 485 | await new Promise((resolve) => setTimeout(resolve, 500)); | ||
| 486 | } | ||
| 487 | } finally { | ||
| 488 | await client.close(); | ||
| 489 | } | ||
| 490 | |||
| 491 | const summary = summarizeTranscript(transcript); | ||
| 492 | const transcriptPath = path.join(outputDir, "transcript.ndjson"); | ||
| 493 | const summaryPath = path.join(outputDir, "summary.json"); | ||
| 494 | |||
| 495 | await fs.writeFile( | ||
| 496 | transcriptPath, | ||
| 497 | `${transcript.map((entry) => JSON.stringify(entry)).join("\n")}\n`, | ||
| 498 | "utf8", | ||
| 499 | ); | ||
| 500 | await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); | ||
| 501 | |||
| 502 | process.stdout.write(`ACP probe complete.\n`); | ||
| 503 | process.stdout.write(` outputDir: ${outputDir}\n`); | ||
| 504 | process.stdout.write(` transcript: ${transcriptPath}\n`); | ||
| 505 | process.stdout.write(` summary: ${summaryPath}\n`); | ||
| 506 | } | ||
| 507 | |||
| 508 | run().catch((error) => { | ||
| 509 | process.stderr.write(`ACP probe failed: ${String(error)}\n`); | ||
| 510 | process.exitCode = 1; | ||
| 511 | }); | ||