History

chore: move cursor acp probe artifacts to cursor stack

Julius Marminge committed 8 months ago

Commit 2435a9c1 parent 78d53063 files · +1 −840Browse files

Showing the changes to scripts (1 of 3 files). Show all

Unified · Split

scripts/cursor-acp-probe.mjs+0−511
@@ -1,511 +0,0 @@
1−import { spawn } from "node:child_process";
2−import fs from "node:fs/promises";
3−import path from "node:path";
4−import readline from "node:readline";
5−
6−const DEFAULT_PROMPT_TIMEOUT_MS = 120_000;
7−const CANCEL_AFTER_MS = 1_500;
8−
9−function parseArgv(argv) {
10− const parsed = {
11− outputDir: "",
12− workspace: process.cwd(),
13− model: "",
14− permissionOption: "allow-once",
15− };
16−
17− for (let i = 0; i < argv.length; i += 1) {
18− const token = argv[i];
19− const next = argv[i + 1];
20− if ((token === "--output-dir" || token === "-o") && next) {
21− parsed.outputDir = next;
22− i += 1;
23− continue;
24− }
25− if ((token === "--workspace" || token === "-w") && next) {
26− parsed.workspace = path.resolve(next);
27− i += 1;
28− continue;
29− }
30− if ((token === "--model" || token === "-m") && next) {
31− parsed.model = next;
32− i += 1;
33− continue;
34− }
35− if (token === "--permission-option" && next) {
36− parsed.permissionOption = next;
37− i += 1;
38− }
39− }
40−
41− return parsed;
42−}
43−
44−function nowIso() {
45− return new Date().toISOString();
46−}
47−
48−function safeJsonParse(line) {
49− try {
50− return { ok: true, value: JSON.parse(line) };
51− } catch (error) {
52− return { ok: false, error };
53− }
54−}
55−
56−class AcpProbeClient {
57− #child;
58− #stdoutRl;
59− #stderrRl;
60− #nextId = 1;
61− #closed = false;
62− #pending = new Map();
63− #onMessage;
64− #onServerRequest;
65−
66− constructor({ onMessage, onServerRequest }) {
67− this.#onMessage = onMessage;
68− this.#onServerRequest = onServerRequest;
69−
70− this.#child = spawn("agent", ["acp"], {
71− stdio: ["pipe", "pipe", "pipe"],
72− env: {
73− ...process.env,
74− NO_COLOR: "1",
75− },
76− });
77−
78− this.#stdoutRl = readline.createInterface({ input: this.#child.stdout });
79− this.#stderrRl = readline.createInterface({ input: this.#child.stderr });
80−
81− this.#stdoutRl.on("line", (line) => this.#handleStdoutLine(line));
82− this.#stderrRl.on("line", (line) => {
83− this.#onMessage({
84− ts: nowIso(),
85− channel: "stderr",
86− line,
87− });
88− });
89−
90− this.#child.once("exit", (code, signal) => {
91− this.#closed = true;
92− const reason = `ACP process exited (code=${String(code)}, signal=${String(signal)})`;
93− for (const [id, pending] of this.#pending.entries()) {
94− this.#pending.delete(id);
95− pending.reject(new Error(reason));
96− }
97− this.#onMessage({
98− ts: nowIso(),
99− channel: "lifecycle",
100− event: "exit",
101− code,
102− signal,
103− });
104− });
105− }
106−
107− async close() {
108− if (this.#closed) return;
109− try {
110− this.#child.stdin.end();
111− } catch {
112− // ignored
113− }
114− this.#child.kill("SIGTERM");
115− await new Promise((resolve) => setTimeout(resolve, 100));
116− if (!this.#closed) {
117− this.#child.kill("SIGKILL");
118− }
119− }
120−
121− async send(method, params, { timeoutMs = DEFAULT_PROMPT_TIMEOUT_MS } = {}) {
122− if (this.#closed) {
123− throw new Error("Cannot send: ACP process is already closed.");
124− }
125− const id = this.#nextId;
126− this.#nextId += 1;
127−
128− const message = {
129− jsonrpc: "2.0",
130− id,
131− method,
132− params,
133− };
134−
135− this.#onMessage({
136− ts: nowIso(),
137− channel: "client->server",
138− message,
139− });
140− this.#child.stdin.write(`${JSON.stringify(message)}\n`);
141−
142− return new Promise((resolve, reject) => {
143− const timeout = setTimeout(() => {
144− this.#pending.delete(id);
145− reject(new Error(`Timed out waiting for response to '${method}' (id=${id}).`));
146− }, timeoutMs);
147−
148− this.#pending.set(id, {
149− resolve: (value) => {
150− clearTimeout(timeout);
151− resolve(value);
152− },
153− reject: (error) => {
154− clearTimeout(timeout);
155− reject(error);
156− },
157− });
158− });
159− }
160−
161− #respond(id, result) {
162− if (this.#closed) return;
163− const message = {
164− jsonrpc: "2.0",
165− id,
166− result,
167− };
168− this.#onMessage({
169− ts: nowIso(),
170− channel: "client->server",
171− message,
172− });
173− this.#child.stdin.write(`${JSON.stringify(message)}\n`);
174− }
175−
176− #handleStdoutLine(line) {
177− const parsed = safeJsonParse(line);
178− if (!parsed.ok) {
179− this.#onMessage({
180− ts: nowIso(),
181− channel: "stdout-non-json",
182− line,
183− });
184− return;
185− }
186−
187− const message = parsed.value;
188− this.#onMessage({
189− ts: nowIso(),
190− channel: "server->client",
191− message,
192− });
193−
194− if (
195− Object.prototype.hasOwnProperty.call(message, "id") &&
196− (Object.prototype.hasOwnProperty.call(message, "result") ||
197− Object.prototype.hasOwnProperty.call(message, "error")) &&
198− !Object.prototype.hasOwnProperty.call(message, "method")
199− ) {
200− const pending = this.#pending.get(message.id);
201− if (!pending) return;
202− this.#pending.delete(message.id);
203− if (Object.prototype.hasOwnProperty.call(message, "error")) {
204− pending.reject(message.error);
205− return;
206− }
207− pending.resolve(message.result);
208− return;
209− }
210−
211− if (
212− Object.prototype.hasOwnProperty.call(message, "id") &&
213− Object.prototype.hasOwnProperty.call(message, "method")
214− ) {
215− this.#onServerRequest?.(message, (result) => this.#respond(message.id, result));
216− }
217− }
218−}
219−
220−function summarizeTranscript(entries) {
221− const summary = {
222− counts: {
223− notificationsByMethod: {},
224− sessionUpdateByType: {},
225− serverRequestsByMethod: {},
226− permissionDecisions: {},
227− },
228− samples: {
229− initializeResult: null,
230− authenticateResult: null,
231− sessionNewResult: null,
232− sessionPromptResultByScenario: {},
233− sessionUpdateByType: {},
234− serverRequestByMethod: {},
235− },
236− stderr: [],
237− };
238−
239− for (const entry of entries) {
240− if (entry.channel === "stderr") {
241− summary.stderr.push(entry.line);
242− continue;
243− }
244− if (entry.channel !== "server->client") continue;
245−
246− const message = entry.message;
247− if (!message || typeof message !== "object") continue;
248−
249− if (typeof message.method === "string" && !Object.prototype.hasOwnProperty.call(message, "id")) {
250− summary.counts.notificationsByMethod[message.method] =
251− (summary.counts.notificationsByMethod[message.method] ?? 0) + 1;
252− if (message.method === "session/update") {
253− const updateType = message.params?.update?.sessionUpdate;
254− if (typeof updateType === "string") {
255− summary.counts.sessionUpdateByType[updateType] =
256− (summary.counts.sessionUpdateByType[updateType] ?? 0) + 1;
257− if (!summary.samples.sessionUpdateByType[updateType]) {
258− summary.samples.sessionUpdateByType[updateType] = message;
259− }
260− }
261− }
262− continue;
263− }
264−
265− if (typeof message.method === "string" && Object.prototype.hasOwnProperty.call(message, "id")) {
266− summary.counts.serverRequestsByMethod[message.method] =
267− (summary.counts.serverRequestsByMethod[message.method] ?? 0) + 1;
268− if (!summary.samples.serverRequestByMethod[message.method]) {
269− summary.samples.serverRequestByMethod[message.method] = message;
270− }
271− continue;
272− }
273− }
274−
275− for (const entry of entries) {
276− if (entry.channel !== "scenario-result") continue;
277− if (entry.scenario === "initialize" && !summary.samples.initializeResult) {
278− summary.samples.initializeResult = entry.result;
279− continue;
280− }
281− if (entry.scenario === "authenticate" && !summary.samples.authenticateResult) {
282− summary.samples.authenticateResult = entry.result;
283− continue;
284− }
285− if (entry.scenario === "session/new" && !summary.samples.sessionNewResult) {
286− summary.samples.sessionNewResult = entry.result;
287− continue;
288− }
289− if (entry.scenarioName) {
290− summary.samples.sessionPromptResultByScenario[entry.scenarioName] = entry.result;
291− }
292− }
293−
294− for (const entry of entries) {
295− if (entry.channel !== "permission-decision") continue;
296− const optionId = entry.optionId;
297− summary.counts.permissionDecisions[optionId] =
298− (summary.counts.permissionDecisions[optionId] ?? 0) + 1;
299− }
300−
301− return summary;
302−}
303−
304−async function run() {
305− const args = parseArgv(process.argv.slice(2));
306− const allowedPermissionOptions = new Set(["allow-once", "allow-always", "reject-once"]);
307− if (!allowedPermissionOptions.has(args.permissionOption)) {
308− throw new Error(
309− `Invalid --permission-option '${args.permissionOption}'. Expected one of: ${Array.from(allowedPermissionOptions).join(", ")}`,
310− );
311− }
312− const stamp = nowIso().replaceAll(":", "-");
313− const outputDir = args.outputDir
314− ? path.resolve(args.outputDir)
315− : path.join(process.cwd(), ".tmp", "acp-probe", stamp);
316− await fs.mkdir(outputDir, { recursive: true });
317−
318− const transcript = [];
319− const pushEntry = (entry) => transcript.push(entry);
320−
321− let activeScenarioName = null;
322−
323− const client = new AcpProbeClient({
324− onMessage: (entry) => {
325− if (
326− activeScenarioName &&
327− entry.channel === "server->client" &&
328− entry.message?.method === "session/update"
329− ) {
330− transcript.push({ ...entry, scenarioName: activeScenarioName });
331− return;
332− }
333− transcript.push(entry);
334− },
335− onServerRequest: (message, respond) => {
336− if (message.method === "session/request_permission") {
337− const defaultChoice = args.permissionOption;
338− pushEntry({
339− ts: nowIso(),
340− channel: "permission-decision",
341− requestId: message.id,
342− optionId: defaultChoice,
343− params: message.params,
344− });
345− respond({
346− outcome: {
347− outcome: "selected",
348− optionId: defaultChoice,
349− },
350− });
351− return;
352− }
353−
354− respond({
355− outcome: {
356− outcome: "selected",
357− optionId: "deny",
358− },
359− });
360− pushEntry({
361− ts: nowIso(),
362− channel: "server-request-unhandled",
363− method: message.method,
364− requestId: message.id,
365− params: message.params,
366− });
367− },
368− });
369−
370− try {
371− const initializeResult = await client.send("initialize", {
372− protocolVersion: 1,
373− clientCapabilities: {
374− fs: { readTextFile: false, writeTextFile: false },
375− terminal: false,
376− },
377− clientInfo: {
378− name: "t3-cursor-acp-probe",
379− version: "0.1.0",
380− },
381− });
382− pushEntry({
383− ts: nowIso(),
384− channel: "scenario-result",
385− scenario: "initialize",
386− result: initializeResult,
387− });
388−
389− const authenticateResult = await client.send("authenticate", {
390− methodId: "cursor_login",
391− });
392− pushEntry({
393− ts: nowIso(),
394− channel: "scenario-result",
395− scenario: "authenticate",
396− result: authenticateResult,
397− });
398−
399− const sessionParams = {
400− cwd: args.workspace,
401− mcpServers: [],
402− ...(args.model ? { model: args.model } : {}),
403− };
404− const sessionResult = await client.send("session/new", sessionParams);
405− pushEntry({
406− ts: nowIso(),
407− channel: "scenario-result",
408− scenario: "session/new",
409− result: sessionResult,
410− });
411−
412− const sessionId = sessionResult?.sessionId;
413− if (typeof sessionId !== "string" || sessionId.length === 0) {
414− throw new Error(`Missing sessionId from session/new response: ${JSON.stringify(sessionResult)}`);
415− }
416−
417− const scenarios = [
418− {
419− name: "hello",
420− prompt: "Say hello in one sentence.",
421− },
422− {
423− name: "tooling",
424− prompt:
425− "Use tools to run `pwd` and then `ls -1 | head -n 8`, and summarize what you found in one paragraph.",
426− },
427− {
428− name: "cancel",
429− prompt:
430− "Think for a while and draft a long detailed migration plan with at least 20 bullet points before answering.",
431− cancelAfterMs: CANCEL_AFTER_MS,
432− },
433− ];
434−
435− for (const scenario of scenarios) {
436− activeScenarioName = scenario.name;
437− const promptParams = {
438− sessionId,
439− prompt: [{ type: "text", text: scenario.prompt }],
440− };
441−
442− const promptPromise = client.send("session/prompt", promptParams, {
443− timeoutMs: DEFAULT_PROMPT_TIMEOUT_MS,
444− });
445−
446− if (scenario.cancelAfterMs) {
447− setTimeout(() => {
448− client
449− .send("session/cancel", { sessionId }, { timeoutMs: 15_000 })
450− .then((cancelResult) => {
451− pushEntry({
452− ts: nowIso(),
453− channel: "scenario-result",
454− scenario: "session/cancel",
455− scenarioName: scenario.name,
456− result: cancelResult,
457− });
458− })
459− .catch((error) => {
460− pushEntry({
461− ts: nowIso(),
462− channel: "scenario-error",
463− scenario: "session/cancel",
464− scenarioName: scenario.name,
465− error:
466− error instanceof Error
467− ? error.message
468− : typeof error === "string"
469− ? error
470− : JSON.stringify(error),
471− });
472− });
473− }, scenario.cancelAfterMs);
474− }
475−
476− const promptResult = await promptPromise;
477− pushEntry({
478− ts: nowIso(),
479− channel: "scenario-result",
480− scenario: "session/prompt",
481− scenarioName: scenario.name,
482− result: promptResult,
483− });
484− activeScenarioName = null;
485− await new Promise((resolve) => setTimeout(resolve, 500));
486− }
487− } finally {
488− await client.close();
489− }
490−
491− const summary = summarizeTranscript(transcript);
492− const transcriptPath = path.join(outputDir, "transcript.ndjson");
493− const summaryPath = path.join(outputDir, "summary.json");
494−
495− await fs.writeFile(
496− transcriptPath,
497− `${transcript.map((entry) => JSON.stringify(entry)).join("\n")}\n`,
498− "utf8",
499− );
500− await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
501−
502− process.stdout.write(`ACP probe complete.\n`);
503− process.stdout.write(` outputDir: ${outputDir}\n`);
504− process.stdout.write(` transcript: ${transcriptPath}\n`);
505− process.stdout.write(` summary: ${summaryPath}\n`);
506−}
507−
508−run().catch((error) => {
509− process.stderr.write(`ACP probe failed: ${String(error)}\n`);
510− process.exitCode = 1;
511−});