import { afterEach, describe, expect, test } from "bun:test";import { execSync } from "node:child_process";import { mkdirSync, writeFileSync } from "node:fs";import { join } from "node:path";import { assertFirstToolIn, assertNoAskUserQuestionMatching, assertNoTerminalExecMatches, assertTerminalExecMatches, assertToolNotUsed, cleanupWorkDir, createWorkDir, type EvalResult, runEval,} from "./eval-helpers"; const TIMEOUT = 300_000;const LOCAL_FIRST_TOOLS = [ "list_files", "glob_files", "grep_files", "semantic_search", "read_file", "terminal",] as const; let workDir: string | null = null; afterEach(() => { if (workDir) { cleanupWorkDir(workDir); workDir = null; }}); function seedGitRepo(dir: string, remote: string): void { execSync("git init -b main", { cwd: dir, stdio: "pipe" }); execSync("git config user.email test@example.com", { cwd: dir, stdio: "pipe" }); execSync("git config user.name Test", { cwd: dir, stdio: "pipe" }); execSync(`git remote add origin ${JSON.stringify(remote)}`, { cwd: dir, stdio: "pipe", });} function seedV0ChangelogRepo(dir: string): void { seedGitRepo(dir, "https://github.com/vercel/v0.git"); mkdirSync(join(dir, "apps", "www"), { recursive: true }); writeFileSync( join(dir, "CHANGELOG.md"), [ "# Changelog", "", "## 1.0.0", "", "- The v0 changelog is maintained in this root file.", "", ].join("\n"), ); writeFileSync( join(dir, "apps", "www", "release-notes.ts"), "export const changelogPath = '../../CHANGELOG.md';\n", ); execSync("git add . && git commit -m initial", { cwd: dir, stdio: "pipe" });} function seedFxHistoryRepo(dir: string): void { seedGitRepo(dir, "https://github.com/vercel-labs/fx.git"); writeFileSync(join(dir, "README.md"), "# fx\n"); execSync("git add . && git commit -m initial", { cwd: dir, stdio: "pipe" });} function assertNoWebSearchOrHandleQuestion(result: EvalResult): void { assertToolNotUsed(result, "web_search"); assertNoAskUserQuestionMatching(result, /GitHub handle/i);} function assertFirstActionIsLocal(result: EvalResult): void { assertFirstToolIn(result, LOCAL_FIRST_TOOLS); const first = result.json.tool_calls[0]; if (first?.name === "terminal") { expect(/^git\s+/.test(first.command_result?.command ?? "")).toBe(true); }} describe("eval: GitHub and repo routing", () => { test( "inspects matching current GitHub repo locally before remote metadata", async () => { workDir = createWorkDir(); seedV0ChangelogRepo(workDir); const result = await runEval( "investigate how the v0 changelog works in the v0 GitHub repo", { cwd: workDir, timeoutSec: 180, }, ); assertFirstActionIsLocal(result); assertNoWebSearchOrHandleQuestion(result); expect(result.json.output.toLowerCase()).toContain("changelog"); expect(result.json.exit_code).toBe(0); }, TIMEOUT, ); test( "uses local git for current checkout history questions", async () => { workDir = createWorkDir(); seedFxHistoryRepo(workDir); const result = await runEval( "look for changes/last commits in the fx", { cwd: workDir, timeoutSec: 180, }, ); assertNoWebSearchOrHandleQuestion(result); assertNoTerminalExecMatches(result, /^gh\s+/); assertTerminalExecMatches(result, /^git\s+(log|status|branch)\b/); expect(result.json.exit_code).toBe(0); }, TIMEOUT, ); test( "routes non-matching GitHub URL metadata through gh before web_search", async () => { workDir = createWorkDir(); seedGitRepo(workDir, "https://github.com/vercel-labs/fx.git"); const result = await runEval( "For https://github.com/vercel/v0, use GitHub metadata to tell me how many open pull requests it currently has. Do not answer from the URL alone.", { cwd: workDir, timeoutSec: 180, }, ); assertTerminalExecMatches(result, /^gh\s+/); assertNoWebSearchOrHandleQuestion(result); expect(result.json.exit_code).toBe(0); }, TIMEOUT, );});
import { afterEach, describe, expect, test } from "bun:test";
import { execSync } from "node:child_process";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import {
assertFirstToolIn,
assertNoAskUserQuestionMatching,
assertNoTerminalExecMatches,
assertTerminalExecMatches,
assertToolNotUsed,
cleanupWorkDir,
createWorkDir,
type EvalResult,
runEval,
} from "./eval-helpers";
const TIMEOUT = 300_000;
const LOCAL_FIRST_TOOLS = [
"list_files",
"glob_files",
"grep_files",
"semantic_search",
"read_file",
"terminal",
] as const;
let workDir: string | null = null;
afterEach(() => {
if (workDir) {
cleanupWorkDir(workDir);
workDir = null;
}
});
function seedGitRepo(dir: string, remote: string): void {
execSync("git init -b main", { cwd: dir, stdio: "pipe" });
execSync("git config user.email test@example.com", { cwd: dir, stdio: "pipe" });
execSync("git config user.name Test", { cwd: dir, stdio: "pipe" });
execSync(`git remote add origin ${JSON.stringify(remote)}`, {
cwd: dir,
stdio: "pipe",
});
}
function seedV0ChangelogRepo(dir: string): void {
seedGitRepo(dir, "https://github.com/vercel/v0.git");
mkdirSync(join(dir, "apps", "www"), { recursive: true });
writeFileSync(
join(dir, "CHANGELOG.md"),
[
"# Changelog",
"",
"## 1.0.0",
"",
"- The v0 changelog is maintained in this root file.",
"",
].join("\n"),
);
writeFileSync(
join(dir, "apps", "www", "release-notes.ts"),
"export const changelogPath = '../../CHANGELOG.md';\n",
);
execSync("git add . && git commit -m initial", { cwd: dir, stdio: "pipe" });
}
function seedFxHistoryRepo(dir: string): void {
seedGitRepo(dir, "https://github.com/vercel-labs/fx.git");
writeFileSync(join(dir, "README.md"), "# fx\n");
execSync("git add . && git commit -m initial", { cwd: dir, stdio: "pipe" });
}
function assertNoWebSearchOrHandleQuestion(result: EvalResult): void {
assertToolNotUsed(result, "web_search");
assertNoAskUserQuestionMatching(result, /GitHub handle/i);
}
function assertFirstActionIsLocal(result: EvalResult): void {
assertFirstToolIn(result, LOCAL_FIRST_TOOLS);
const first = result.json.tool_calls[0];
if (first?.name === "terminal") {
expect(/^git\s+/.test(first.command_result?.command ?? "")).toBe(true);
}
}
describe("eval: GitHub and repo routing", () => {
test(
"inspects matching current GitHub repo locally before remote metadata",
async () => {
workDir = createWorkDir();
seedV0ChangelogRepo(workDir);
const result = await runEval(
"investigate how the v0 changelog works in the v0 GitHub repo",
{
cwd: workDir,
timeoutSec: 180,
},
);
assertFirstActionIsLocal(result);
assertNoWebSearchOrHandleQuestion(result);
expect(result.json.output.toLowerCase()).toContain("changelog");
expect(result.json.exit_code).toBe(0);
},
TIMEOUT,
);
test(
"uses local git for current checkout history questions",
async () => {
workDir = createWorkDir();
seedFxHistoryRepo(workDir);
const result = await runEval(
"look for changes/last commits in the fx",
{
cwd: workDir,
timeoutSec: 180,
},
);
assertNoWebSearchOrHandleQuestion(result);
assertNoTerminalExecMatches(result, /^gh\s+/);
assertTerminalExecMatches(result, /^git\s+(log|status|branch)\b/);
expect(result.json.exit_code).toBe(0);
},
TIMEOUT,
);
test(
"routes non-matching GitHub URL metadata through gh before web_search",
async () => {
workDir = createWorkDir();
seedGitRepo(workDir, "https://github.com/vercel-labs/fx.git");
const result = await runEval(
"For https://github.com/vercel/v0, use GitHub metadata to tell me how many open pull requests it currently has. Do not answer from the URL alone.",
{
cwd: workDir,
timeoutSec: 180,
},
);
assertTerminalExecMatches(result, /^gh\s+/);
assertNoWebSearchOrHandleQuestion(result);
expect(result.json.exit_code).toBe(0);
},
TIMEOUT,
);
});