use super::*;use crate::{ ReadFileToolInput, edit_file_tool::{EditFileMode, EditFileToolInput}, grep_tool::GrepToolInput, list_directory_tool::ListDirectoryToolInput,};use Role::*;use assistant_tool::ToolRegistry;use client::{Client, UserStore};use collections::HashMap;use fs::FakeFs;use futures::{FutureExt, future::LocalBoxFuture};use gpui::{AppContext, TestAppContext};use indoc::{formatdoc, indoc};use language_model::{ LanguageModelRegistry, LanguageModelRequestTool, LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, SelectedModel,};use project::Project;use prompt_store::{ModelContext, ProjectContext, PromptBuilder, WorktreeContext};use rand::prelude::*;use reqwest_client::ReqwestClient;use serde_json::json;use std::{ cmp::Reverse, fmt::{self, Display}, io::Write as _, str::FromStr, sync::mpsc,};use util::path; #[test]#[cfg_attr(not(feature = "eval"), ignore)]fn eval_extract_handle_command_output() { // Test how well agent generates multiple edit hunks. // // Model | Pass rate // ----------------------------|---------- // claude-3.7-sonnet | 0.98 // gemini-2.5-pro | 0.86 // gemini-2.5-flash | 0.11 // gpt-4.1 | 1.00 let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/extract_handle_command_output/before.rs"); let possible_diffs = vec![ include_str!("evals/fixtures/extract_handle_command_output/possible-01.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-02.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-03.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-04.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-05.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-06.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-07.diff"), ]; let edit_description = "Extract `handle_command_output` method from `run_git_blame`."; eval( 100, 0.7, // Taking the lower bar for Gemini EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and extract a method in the final stanza of `run_git_blame` to deal with command failures, call it `handle_command_output` and take the std::process::Output as the only parameter. Do not document the method and do not add any comments. Add it right next to `run_git_blame` and copy it verbatim from `run_git_blame`. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_diff_any(possible_diffs), ), );} #[test]#[cfg_attr(not(feature = "eval"), ignore)]fn eval_delete_run_git_blame() { let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/delete_run_git_blame/before.rs"); let output_file_content = include_str!("evals/fixtures/delete_run_git_blame/after.rs"); let edit_description = "Delete the `run_git_blame` function."; eval( 100, 0.95, EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and delete `run_git_blame`. Just that one function, not its usages. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_eq(output_file_content), ), );} #[test]#[cfg_attr(not(feature = "eval"), ignore)]fn eval_translate_doc_comments() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | // gemini-2.5-pro-preview-03-25 | 1.0 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/canvas.rs"; let input_file_content = include_str!("evals/fixtures/translate_doc_comments/before.rs"); let edit_description = "Translate all doc comments to Italian"; eval( 200, 1., EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the {input_file_path} file and edit it (without overwriting it), translating all the doc comments to italian. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff("Doc comments were translated to Italian"), ), );} #[test]#[cfg_attr(not(feature = "eval"), ignore)]fn eval_use_wasi_sdk_in_compile_parser_to_wasm() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 0.98 // gemini-2.5-pro-preview-03-25 | 0.99 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/lib.rs"; let input_file_content = include_str!("evals/fixtures/use_wasi_sdk_in_compile_parser_to_wasm/before.rs"); let edit_description = "Update compile_parser_to_wasm to use wasi-sdk instead of emscripten"; eval( 100, 0.95, EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and change `compile_parser_to_wasm` to use `wasi-sdk` instead of emscripten. Use `ureq` to download the SDK for the current platform and architecture. Extract the archive into a sibling of `lib` inside the `tree-sitter` directory in the cache_dir. Compile the parser to wasm using the `bin/clang` executable (or `bin/clang.exe` on windows) that's inside of the archive. Don't re-download the SDK if that executable already exists. Use these clang flags: -fPIC -shared -Os -Wl,--export=tree_sitter_{{language_name}} Here are the available wasi-sdk assets: - wasi-sdk-25.0-x86_64-macos.tar.gz - wasi-sdk-25.0-arm64-macos.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-windows.tar.gz "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: Some(971), end_line: Some(1050), }, )], ), message( User, [tool_result( "tool_1", "read_file", lines(input_file_content, 971..1050), )], ), message( Assistant, [tool_use( "tool_2", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: Some(1050), end_line: Some(1100), }, )], ), message( User, [tool_result( "tool_2", "read_file", lines(input_file_content, 1050..1100), )], ), message( Assistant, [tool_use( "tool_3", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: Some(1100), end_line: Some(1150), }, )], ), message( User, [tool_result( "tool_3", "read_file", lines(input_file_content, 1100..1150), )], ), message( Assistant, [tool_use( "tool_4", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff(indoc! {" - The compile_parser_to_wasm method has been changed to use wasi-sdk - ureq is used to download the SDK for current platform and architecture "}), ), );} #[test]#[cfg_attr(not(feature = "eval"), ignore)]fn eval_disable_cursor_blinking() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | // gemini-2.5-pro-preview-03-25 | 1.0 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/editor.rs"; let input_file_content = include_str!("evals/fixtures/disable_cursor_blinking/before.rs"); let edit_description = "Comment out the call to `BlinkManager::enable`"; eval( 100, 0.95, EvalInput::from_conversation( vec![ message(User, [text("Let's research how to cursor blinking works.")]), message( Assistant, [tool_use( "tool_1", "grep", GrepToolInput { regex: "blink".into(), include_pattern: None, offset: 0, case_sensitive: false, }, )], ), message( User, [tool_result( "tool_1", "grep", [ lines(input_file_content, 100..400), lines(input_file_content, 800..1300), lines(input_file_content, 1600..2000), lines(input_file_content, 5000..5500), lines(input_file_content, 8000..9000), lines(input_file_content, 18455..18470), lines(input_file_content, 20000..20500), lines(input_file_content, 21000..21300), ] .join("Match found:\n\n"), )], ), message( User, [text(indoc! {" Comment out the lines that interact with the BlinkManager. Keep the outer `update` blocks, but comments everything that's inside (including if statements). Don't add additional comments. "})], ), message( Assistant, [tool_use( "tool_4", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff(indoc! {" - Calls to BlinkManager in `observe_window_activation` were commented out - The call to `blink_manager.enable` above the call to show_cursor_names was commented out - All the edits have valid indentation "}), ), );} #[test]#[cfg_attr(not(feature = "eval"), ignore)]fn eval_from_pixels_constructor() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | // gemini-2.5-pro-preview-03-25 | 0.94 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/canvas.rs"; let input_file_content = include_str!("evals/fixtures/from_pixels_constructor/before.rs"); let edit_description = "Implement from_pixels constructor and add tests."; eval( 100, 0.95, EvalInput::from_conversation( vec![ message( User, [text(indoc! {" Introduce a new `from_pixels` constructor in Canvas and also add tests for it in the same file. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "grep", GrepToolInput { regex: "mod\\s+tests".into(), include_pattern: Some("font-kit/src/canvas.rs".into()), offset: 0, case_sensitive: false, }, )], ), message(User, [tool_result("tool_2", "grep", "No matches found")]), message( Assistant, [tool_use( "tool_3", "grep", GrepToolInput { regex: "mod\\s+tests".into(), include_pattern: Some("font-kit/src/**/*.rs".into()), offset: 0, case_sensitive: false, },Lines 1–500 of 1621. Load more
use super::*; use crate::{ ReadFileToolInput, edit_file_tool::{EditFileMode, EditFileToolInput}, grep_tool::GrepToolInput, list_directory_tool::ListDirectoryToolInput, }; use Role::*; use assistant_tool::ToolRegistry; use client::{Client, UserStore}; use collections::HashMap; use fs::FakeFs; use futures::{FutureExt, future::LocalBoxFuture}; use gpui::{AppContext, TestAppContext}; use indoc::{formatdoc, indoc}; use language_model::{ LanguageModelRegistry, LanguageModelRequestTool, LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, SelectedModel, }; use project::Project; use prompt_store::{ModelContext, ProjectContext, PromptBuilder, WorktreeContext}; use rand::prelude::*; use reqwest_client::ReqwestClient; use serde_json::json; use std::{ cmp::Reverse, fmt::{self, Display}, io::Write as _, str::FromStr, sync::mpsc, }; use util::path; #[test] #[cfg_attr(not(feature = "eval"), ignore)] fn eval_extract_handle_command_output() { // Test how well agent generates multiple edit hunks. // // Model | Pass rate // ----------------------------|---------- // claude-3.7-sonnet | 0.98 // gemini-2.5-pro | 0.86 // gemini-2.5-flash | 0.11 // gpt-4.1 | 1.00 let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/extract_handle_command_output/before.rs"); let possible_diffs = vec![ include_str!("evals/fixtures/extract_handle_command_output/possible-01.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-02.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-03.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-04.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-05.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-06.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-07.diff"), ]; let edit_description = "Extract `handle_command_output` method from `run_git_blame`."; eval( 100, 0.7, // Taking the lower bar for Gemini EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and extract a method in the final stanza of `run_git_blame` to deal with command failures, call it `handle_command_output` and take the std::process::Output as the only parameter. Do not document the method and do not add any comments. Add it right next to `run_git_blame` and copy it verbatim from `run_git_blame`. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_diff_any(possible_diffs), ), ); } #[test] #[cfg_attr(not(feature = "eval"), ignore)] fn eval_delete_run_git_blame() { let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/delete_run_git_blame/before.rs"); let output_file_content = include_str!("evals/fixtures/delete_run_git_blame/after.rs"); let edit_description = "Delete the `run_git_blame` function."; eval( 100, 0.95, EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and delete `run_git_blame`. Just that one function, not its usages. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_eq(output_file_content), ), ); } #[test] #[cfg_attr(not(feature = "eval"), ignore)] fn eval_translate_doc_comments() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | // gemini-2.5-pro-preview-03-25 | 1.0 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/canvas.rs"; let input_file_content = include_str!("evals/fixtures/translate_doc_comments/before.rs"); let edit_description = "Translate all doc comments to Italian"; eval( 200, 1., EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the {input_file_path} file and edit it (without overwriting it), translating all the doc comments to italian. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff("Doc comments were translated to Italian"), ), ); } #[test] #[cfg_attr(not(feature = "eval"), ignore)] fn eval_use_wasi_sdk_in_compile_parser_to_wasm() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 0.98 // gemini-2.5-pro-preview-03-25 | 0.99 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/lib.rs"; let input_file_content = include_str!("evals/fixtures/use_wasi_sdk_in_compile_parser_to_wasm/before.rs"); let edit_description = "Update compile_parser_to_wasm to use wasi-sdk instead of emscripten"; eval( 100, 0.95, EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and change `compile_parser_to_wasm` to use `wasi-sdk` instead of emscripten. Use `ureq` to download the SDK for the current platform and architecture. Extract the archive into a sibling of `lib` inside the `tree-sitter` directory in the cache_dir. Compile the parser to wasm using the `bin/clang` executable (or `bin/clang.exe` on windows) that's inside of the archive. Don't re-download the SDK if that executable already exists. Use these clang flags: -fPIC -shared -Os -Wl,--export=tree_sitter_{{language_name}} Here are the available wasi-sdk assets: - wasi-sdk-25.0-x86_64-macos.tar.gz - wasi-sdk-25.0-arm64-macos.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-windows.tar.gz "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: Some(971), end_line: Some(1050), }, )], ), message( User, [tool_result( "tool_1", "read_file", lines(input_file_content, 971..1050), )], ), message( Assistant, [tool_use( "tool_2", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: Some(1050), end_line: Some(1100), }, )], ), message( User, [tool_result( "tool_2", "read_file", lines(input_file_content, 1050..1100), )], ), message( Assistant, [tool_use( "tool_3", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: Some(1100), end_line: Some(1150), }, )], ), message( User, [tool_result( "tool_3", "read_file", lines(input_file_content, 1100..1150), )], ), message( Assistant, [tool_use( "tool_4", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff(indoc! {" - The compile_parser_to_wasm method has been changed to use wasi-sdk - ureq is used to download the SDK for current platform and architecture "}), ), ); } #[test] #[cfg_attr(not(feature = "eval"), ignore)] fn eval_disable_cursor_blinking() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | // gemini-2.5-pro-preview-03-25 | 1.0 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/editor.rs"; let input_file_content = include_str!("evals/fixtures/disable_cursor_blinking/before.rs"); let edit_description = "Comment out the call to `BlinkManager::enable`"; eval( 100, 0.95, EvalInput::from_conversation( vec![ message(User, [text("Let's research how to cursor blinking works.")]), message( Assistant, [tool_use( "tool_1", "grep", GrepToolInput { regex: "blink".into(), include_pattern: None, offset: 0, case_sensitive: false, }, )], ), message( User, [tool_result( "tool_1", "grep", [ lines(input_file_content, 100..400), lines(input_file_content, 800..1300), lines(input_file_content, 1600..2000), lines(input_file_content, 5000..5500), lines(input_file_content, 8000..9000), lines(input_file_content, 18455..18470), lines(input_file_content, 20000..20500), lines(input_file_content, 21000..21300), ] .join("Match found:\n\n"), )], ), message( User, [text(indoc! {" Comment out the lines that interact with the BlinkManager. Keep the outer `update` blocks, but comments everything that's inside (including if statements). Don't add additional comments. "})], ), message( Assistant, [tool_use( "tool_4", "edit_file", EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff(indoc! {" - Calls to BlinkManager in `observe_window_activation` were commented out - The call to `blink_manager.enable` above the call to show_cursor_names was commented out - All the edits have valid indentation "}), ), ); } #[test] #[cfg_attr(not(feature = "eval"), ignore)] fn eval_from_pixels_constructor() { // Results for 2025-05-22 // // Model | Pass rate // ============================================ // // claude-3.7-sonnet | // gemini-2.5-pro-preview-03-25 | 0.94 // gemini-2.5-flash-preview-04-17 | // gpt-4.1 | let input_file_path = "root/canvas.rs"; let input_file_content = include_str!("evals/fixtures/from_pixels_constructor/before.rs"); let edit_description = "Implement from_pixels constructor and add tests."; eval( 100, 0.95, EvalInput::from_conversation( vec![ message( User, [text(indoc! {" Introduce a new `from_pixels` constructor in Canvas and also add tests for it in the same file. "})], ), message( Assistant, [tool_use( "tool_1", "read_file", ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result("tool_1", "read_file", input_file_content)], ), message( Assistant, [tool_use( "tool_2", "grep", GrepToolInput { regex: "mod\\s+tests".into(), include_pattern: Some("font-kit/src/canvas.rs".into()), offset: 0, case_sensitive: false, }, )], ), message(User, [tool_result("tool_2", "grep", "No matches found")]), message( Assistant, [tool_use( "tool_3", "grep", GrepToolInput { regex: "mod\\s+tests".into(), include_pattern: Some("font-kit/src/**/*.rs".into()), offset: 0, case_sensitive: false, },