import { query, type SDKMessage, type ThinkingConfig } from "@anthropic-ai/claude-agent-sdk"; type ProbeCase = | "default" | "effort-low" | "effort-high" | "settings-off" | "settings-on" | "thinking-disabled" | "thinking-enabled"; function parseArgs(argv: ReadonlyArray): { readonly model: string; readonly cwd?: string; readonly prompt: string; readonly cases: ReadonlyArray; } { let model = "claude-haiku-4-5"; let cwd: string | undefined; let prompt = "Think step by step about this before answering: what is 27 multiplied by 43? Return the final number only."; let cases: ReadonlyArray = [ "default", "effort-low", "effort-high", "settings-off", "settings-on", "thinking-disabled", "thinking-enabled", ]; for (let index = 0; index < argv.length; index += 1) { const value = argv[index]; if (!value) { continue; } if (value === "--model") { model = argv[index + 1] ?? model; index += 1; continue; } if (value === "--cwd") { cwd = argv[index + 1] ?? cwd; index += 1; continue; } if (value === "--prompt") { prompt = argv[index + 1] ?? prompt; index += 1; continue; } if (value === "--cases") { const next = argv[index + 1]; if (next) { const requested = next .split(",") .map((entry) => entry.trim()) .filter( (entry): entry is ProbeCase => entry === "default" || entry === "effort-low" || entry === "effort-high" || entry === "settings-off" || entry === "settings-on" || entry === "thinking-disabled" || entry === "thinking-enabled", ); if (requested.length > 0) { cases = requested; } } index += 1; } } return { model, prompt, ...(cwd ? { cwd } : {}), cases, }; } function caseOptions(caseName: ProbeCase): { readonly label: ProbeCase; readonly effort?: "low" | "high"; readonly settings?: { alwaysThinkingEnabled: boolean }; readonly thinking?: ThinkingConfig; } { switch (caseName) { case "effort-low": return { label: caseName, effort: "low", }; case "effort-high": return { label: caseName, effort: "high", }; case "settings-off": return { label: caseName, settings: { alwaysThinkingEnabled: false, }, }; case "settings-on": return { label: caseName, settings: { alwaysThinkingEnabled: true, }, }; case "thinking-disabled": return { label: caseName, thinking: { type: "disabled", }, }; case "thinking-enabled": return { label: caseName, thinking: { type: "enabled", budgetTokens: 1024, }, }; default: return { label: caseName, }; } } type ProbeSummary = { readonly case: ProbeCase; readonly model: string; initModelInfo: { value: string | null; supportsEffort: boolean | null; supportsAdaptiveThinking: boolean | null; supportedEffortLevels: ReadonlyArray | null; } | null; resultSubtype: string | null; resultText: string | null; assistantThinkingBlockCount: number; streamThinkingEventTypes: Record; error: string | null; }; async function runCase(input: { readonly caseName: ProbeCase; readonly model: string; readonly cwd?: string; readonly prompt: string; }): Promise { const caseConfig = caseOptions(input.caseName); const summary: ProbeSummary = { case: input.caseName, model: input.model, initModelInfo: null, resultSubtype: null, resultText: null, assistantThinkingBlockCount: 0, streamThinkingEventTypes: {}, error: null, }; try { const messages = query({ prompt: input.prompt, options: { model: input.model, ...(input.cwd ? { cwd: input.cwd } : {}), persistSession: false, tools: [], permissionMode: "bypassPermissions", includePartialMessages: true, maxTurns: 1, ...(caseConfig.effort ? { effort: caseConfig.effort } : {}), ...(caseConfig.settings ? { settings: caseConfig.settings } : {}), ...(caseConfig.thinking ? { thinking: caseConfig.thinking } : {}), }, }); const initialization = await messages.initializationResult(); const initModel = initialization.models.find((candidate) => candidate.value === input.model) ?? initialization.models.find((candidate) => candidate.value.includes("haiku")) ?? null; summary.initModelInfo = initModel ? { value: initModel.value, supportsEffort: initModel.supportsEffort ?? null, supportsAdaptiveThinking: initModel.supportsAdaptiveThinking ?? null, supportedEffortLevels: initModel.supportedEffortLevels ?? null, } : null; for await (const message of messages) { handleMessage(summary, message); } } catch (error) { summary.error = error instanceof Error ? error.message : String(error); } return summary; } function handleMessage(summary: ProbeSummary, message: SDKMessage): void { if (message.type === "stream_event") { const event = message.event as { type?: string; delta?: { type?: string; }; }; const eventType = event.delta?.type ?? event.type; if (typeof eventType === "string" && eventType.includes("thinking")) { summary.streamThinkingEventTypes[eventType] = (summary.streamThinkingEventTypes[eventType] ?? 0) + 1; } return; } if (message.type === "assistant") { for (const block of message.message.content) { if (typeof block?.type === "string" && block.type.includes("thinking")) { summary.assistantThinkingBlockCount += 1; } } return; } if (message.type === "result") { summary.resultSubtype = message.subtype; summary.resultText = message.subtype === "success" ? message.result : message.errors.join("\n"); } } async function main(): Promise { const args = parseArgs(process.argv.slice(2)); const results: ProbeSummary[] = []; for (const caseName of args.cases) { results.push( await runCase({ caseName, model: args.model, ...(args.cwd ? { cwd: args.cwd } : {}), prompt: args.prompt, }), ); } console.log(JSON.stringify(results, null, 2)); } await main();