use super::*;use crate::{ AgentTool, EditFileMode, EditFileTool, EditFileToolInput, GrepTool, GrepToolInput, ListDirectoryTool, ListDirectoryToolInput, ReadFileTool, ReadFileToolInput,};use Role::*;use client::{Client, UserStore};use eval_utils::{EvalOutput, EvalOutputProcessor, OutcomeKind};use fs::FakeFs;use futures::{FutureExt, future::LocalBoxFuture};use gpui::{AppContext, TestAppContext};use http_client::StatusCode;use indoc::{formatdoc, indoc};use language_model::{ LanguageModelRegistry, LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, SelectedModel,};use project::Project;use prompt_store::{ProjectContext, WorktreeContext};use rand::prelude::*;use reqwest_client::ReqwestClient;use serde_json::json;use std::{ fmt::{self, Display}, path::Path, str::FromStr, time::Duration,};use util::path; #[derive(Default, Clone, Debug)]struct EditAgentOutputProcessor { mismatched_tag_threshold: f32, cumulative_tags: usize, cumulative_mismatched_tags: usize, eval_outputs: Vec<EvalOutput<EditEvalMetadata>>,} fn mismatched_tag_threshold(mismatched_tag_threshold: f32) -> EditAgentOutputProcessor { EditAgentOutputProcessor { mismatched_tag_threshold, cumulative_tags: 0, cumulative_mismatched_tags: 0, eval_outputs: Vec::new(), }} #[derive(Clone, Debug)]struct EditEvalMetadata { tags: usize, mismatched_tags: usize,} impl EvalOutputProcessor for EditAgentOutputProcessor { type Metadata = EditEvalMetadata; fn process(&mut self, output: &EvalOutput<Self::Metadata>) { if matches!(output.outcome, OutcomeKind::Passed | OutcomeKind::Failed) { self.cumulative_mismatched_tags += output.metadata.mismatched_tags; self.cumulative_tags += output.metadata.tags; self.eval_outputs.push(output.clone()); } } fn assert(&mut self) { let mismatched_tag_ratio = self.cumulative_mismatched_tags as f32 / self.cumulative_tags as f32; if mismatched_tag_ratio > self.mismatched_tag_threshold { for eval_output in &self.eval_outputs { println!("{}", eval_output.data); } panic!( "Too many mismatched tags: {:?}", self.cumulative_mismatched_tags ); } }} #[test]#[cfg_attr(not(feature = "unit-eval"), ignore)]fn eval_extract_handle_command_output() { // Test how well agent generates multiple edit hunks. // // Model | Pass rate // ----------------------------|---------- // claude-3.7-sonnet | 0.99 (2025-06-14) // claude-sonnet-4 | 0.97 (2025-06-14) // gemini-2.5-pro-06-05 | 0.98 (2025-06-16) // gemini-2.5-flash | 0.11 (2025-05-22) let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/extract_handle_command_output/before.rs"); let possible_diffs = vec![ include_str!("evals/fixtures/extract_handle_command_output/possible-01.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-02.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-03.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-04.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-05.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-06.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-07.diff"), ]; let edit_description = "Extract `handle_command_output` method from `run_git_blame`."; eval_utils::eval(100, 0.95, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and extract a method in the final stanza of `run_git_blame` to deal with command failures, call it `handle_command_output` and take the std::process::Output as the only parameter. Do not document the method and do not add any comments. Add it right next to `run_git_blame` and copy it verbatim from `run_git_blame`. "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, input_file_content, )], ), message( Assistant, [tool_use( "tool_2", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_diff_any(possible_diffs.clone()), )) });} #[test]#[cfg_attr(not(feature = "unit-eval"), ignore)]fn eval_delete_run_git_blame() { // Model | Pass rate // ----------------------------|---------- // claude-3.7-sonnet | 1.0 (2025-06-14) // claude-sonnet-4 | 0.96 (2025-06-14) // gemini-2.5-pro-06-05 | 1.0 (2025-06-16) // gemini-2.5-flash | let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/delete_run_git_blame/before.rs"); let output_file_content = include_str!("evals/fixtures/delete_run_git_blame/after.rs"); let edit_description = "Delete the `run_git_blame` function."; eval_utils::eval(100, 0.95, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and delete `run_git_blame`. Just that one function, not its usages. "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, input_file_content, )], ), message( Assistant, [tool_use( "tool_2", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_eq(output_file_content), )) });} #[test]#[cfg_attr(not(feature = "unit-eval"), ignore)]fn eval_translate_doc_comments() { // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 1.0 (2025-06-14) // claude-sonnet-4 | 1.0 (2025-06-14) // gemini-2.5-pro-preview-03-25 | 1.0 (2025-05-22) // gemini-2.5-flash-preview-04-17 | let input_file_path = "root/canvas.rs"; let input_file_content = include_str!("evals/fixtures/translate_doc_comments/before.rs"); let edit_description = "Translate all doc comments to Italian"; eval_utils::eval(200, 1., mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the {input_file_path} file and edit it (without overwriting it), translating all the doc comments to italian. "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, input_file_content, )], ), message( Assistant, [tool_use( "tool_2", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff("Doc comments were translated to Italian"), )) });} #[test]#[cfg_attr(not(feature = "unit-eval"), ignore)]fn eval_use_wasi_sdk_in_compile_parser_to_wasm() { // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 0.96 (2025-06-14) // claude-sonnet-4 | 0.11 (2025-06-14) // gemini-2.5-pro-preview-latest | 0.99 (2025-06-16) // gemini-2.5-flash-preview-04-17 | let input_file_path = "root/lib.rs"; let input_file_content = include_str!("evals/fixtures/use_wasi_sdk_in_compile_parser_to_wasm/before.rs"); let edit_description = "Update compile_parser_to_wasm to use wasi-sdk instead of emscripten"; eval_utils::eval(100, 0.95, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and change `compile_parser_to_wasm` to use `wasi-sdk` instead of emscripten. Use `ureq` to download the SDK for the current platform and architecture. Extract the archive into a sibling of `lib` inside the `tree-sitter` directory in the cache_dir. Compile the parser to wasm using the `bin/clang` executable (or `bin/clang.exe` on windows) that's inside of the archive. Don't re-download the SDK if that executable already exists. Use these clang flags: -fPIC -shared -Os -Wl,--export=tree_sitter_{{language_name}} Here are the available wasi-sdk assets: - wasi-sdk-25.0-x86_64-macos.tar.gz - wasi-sdk-25.0-arm64-macos.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-windows.tar.gz "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: Some(971), end_line: Some(1050), }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, lines(input_file_content, 971..1050), )], ), message( Assistant, [tool_use( "tool_2", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: Some(1050), end_line: Some(1100), }, )], ), message( User, [tool_result( "tool_2", ReadFileTool::NAME, lines(input_file_content, 1050..1100), )], ), message( Assistant, [tool_use( "tool_3", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: Some(1100), end_line: Some(1150), }, )], ), message( User, [tool_result( "tool_3", ReadFileTool::NAME, lines(input_file_content, 1100..1150), )], ), message( Assistant, [tool_use( "tool_4", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff(indoc! {" - The compile_parser_to_wasm method has been changed to use wasi-sdk - ureq is used to download the SDK for current platform and architecture "}), )) });} #[test]#[cfg_attr(not(feature = "unit-eval"), ignore)]fn eval_disable_cursor_blinking() { // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 0.59 (2025-07-14) // claude-sonnet-4 | 0.81 (2025-07-14) // gemini-2.5-pro | 0.95 (2025-07-14) // gemini-2.5-flash-preview-04-17 | 0.78 (2025-07-14) let input_file_path = "root/editor.rs"; let input_file_content = include_str!("evals/fixtures/disable_cursor_blinking/before.rs"); let edit_description = "Comment out the call to `BlinkManager::enable`"; let possible_diffs = vec![ include_str!("evals/fixtures/disable_cursor_blinking/possible-01.diff"), include_str!("evals/fixtures/disable_cursor_blinking/possible-02.diff"), include_str!("evals/fixtures/disable_cursor_blinking/possible-03.diff"), include_str!("evals/fixtures/disable_cursor_blinking/possible-04.diff"), ]; eval_utils::eval(100, 0.51, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message(User, [text("Let's research how to cursor blinking works.")]), message( Assistant, [tool_use( "tool_1", GrepTool::NAME, GrepToolInput { regex: "blink".into(), include_pattern: None, offset: 0, case_sensitive: false, }, )], ), message( User, [tool_result( "tool_1", GrepTool::NAME, [ lines(input_file_content, 100..400), lines(input_file_content, 800..1300), lines(input_file_content, 1600..2000), lines(input_file_content, 5000..5500), lines(input_file_content, 8000..9000), lines(input_file_content, 18455..18470), lines(input_file_content, 20000..20500), lines(input_file_content, 21000..21300), ] .join("Match found:\n\n"), )], ), message( User, [text(indoc! {" Comment out the lines that interact with the BlinkManager. Keep the outer `update` blocks, but comments everything that's inside (including if statements). Don't add additional comments. "})], ), message( Assistant, [tool_use( "tool_4", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_diff_any(possible_diffs.clone()), )) });} #[test]#[cfg_attr(not(feature = "unit-eval"), ignore)]fn eval_from_pixels_constructor() { // Results for 2025-06-13 // // The outcome of this evaluation depends heavily on the LINE_HINT_TOLERANCE // value. Higher values improve the pass rate but may sometimes cause // edits to be misapplied. In the context of this eval, this means // the agent might add from_pixels tests in incorrect locations // (e.g., at the beginning of the file), yet the evaluation may still // rate it highly.Lines 1–500 of 1700. Load more
use super::*; use crate::{ AgentTool, EditFileMode, EditFileTool, EditFileToolInput, GrepTool, GrepToolInput, ListDirectoryTool, ListDirectoryToolInput, ReadFileTool, ReadFileToolInput, }; use Role::*; use client::{Client, UserStore}; use eval_utils::{EvalOutput, EvalOutputProcessor, OutcomeKind}; use fs::FakeFs; use futures::{FutureExt, future::LocalBoxFuture}; use gpui::{AppContext, TestAppContext}; use http_client::StatusCode; use indoc::{formatdoc, indoc}; use language_model::{ LanguageModelRegistry, LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, SelectedModel, }; use project::Project; use prompt_store::{ProjectContext, WorktreeContext}; use rand::prelude::*; use reqwest_client::ReqwestClient; use serde_json::json; use std::{ fmt::{self, Display}, path::Path, str::FromStr, time::Duration, }; use util::path; #[derive(Default, Clone, Debug)] struct EditAgentOutputProcessor { mismatched_tag_threshold: f32, cumulative_tags: usize, cumulative_mismatched_tags: usize, eval_outputs: Vec<EvalOutput<EditEvalMetadata>>, } fn mismatched_tag_threshold(mismatched_tag_threshold: f32) -> EditAgentOutputProcessor { EditAgentOutputProcessor { mismatched_tag_threshold, cumulative_tags: 0, cumulative_mismatched_tags: 0, eval_outputs: Vec::new(), } } #[derive(Clone, Debug)] struct EditEvalMetadata { tags: usize, mismatched_tags: usize, } impl EvalOutputProcessor for EditAgentOutputProcessor { type Metadata = EditEvalMetadata; fn process(&mut self, output: &EvalOutput<Self::Metadata>) { if matches!(output.outcome, OutcomeKind::Passed | OutcomeKind::Failed) { self.cumulative_mismatched_tags += output.metadata.mismatched_tags; self.cumulative_tags += output.metadata.tags; self.eval_outputs.push(output.clone()); } } fn assert(&mut self) { let mismatched_tag_ratio = self.cumulative_mismatched_tags as f32 / self.cumulative_tags as f32; if mismatched_tag_ratio > self.mismatched_tag_threshold { for eval_output in &self.eval_outputs { println!("{}", eval_output.data); } panic!( "Too many mismatched tags: {:?}", self.cumulative_mismatched_tags ); } } } #[test] #[cfg_attr(not(feature = "unit-eval"), ignore)] fn eval_extract_handle_command_output() { // Test how well agent generates multiple edit hunks. // // Model | Pass rate // ----------------------------|---------- // claude-3.7-sonnet | 0.99 (2025-06-14) // claude-sonnet-4 | 0.97 (2025-06-14) // gemini-2.5-pro-06-05 | 0.98 (2025-06-16) // gemini-2.5-flash | 0.11 (2025-05-22) let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/extract_handle_command_output/before.rs"); let possible_diffs = vec![ include_str!("evals/fixtures/extract_handle_command_output/possible-01.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-02.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-03.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-04.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-05.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-06.diff"), include_str!("evals/fixtures/extract_handle_command_output/possible-07.diff"), ]; let edit_description = "Extract `handle_command_output` method from `run_git_blame`."; eval_utils::eval(100, 0.95, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and extract a method in the final stanza of `run_git_blame` to deal with command failures, call it `handle_command_output` and take the std::process::Output as the only parameter. Do not document the method and do not add any comments. Add it right next to `run_git_blame` and copy it verbatim from `run_git_blame`. "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, input_file_content, )], ), message( Assistant, [tool_use( "tool_2", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_diff_any(possible_diffs.clone()), )) }); } #[test] #[cfg_attr(not(feature = "unit-eval"), ignore)] fn eval_delete_run_git_blame() { // Model | Pass rate // ----------------------------|---------- // claude-3.7-sonnet | 1.0 (2025-06-14) // claude-sonnet-4 | 0.96 (2025-06-14) // gemini-2.5-pro-06-05 | 1.0 (2025-06-16) // gemini-2.5-flash | let input_file_path = "root/blame.rs"; let input_file_content = include_str!("evals/fixtures/delete_run_git_blame/before.rs"); let output_file_content = include_str!("evals/fixtures/delete_run_git_blame/after.rs"); let edit_description = "Delete the `run_git_blame` function."; eval_utils::eval(100, 0.95, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and delete `run_git_blame`. Just that one function, not its usages. "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, input_file_content, )], ), message( Assistant, [tool_use( "tool_2", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_eq(output_file_content), )) }); } #[test] #[cfg_attr(not(feature = "unit-eval"), ignore)] fn eval_translate_doc_comments() { // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 1.0 (2025-06-14) // claude-sonnet-4 | 1.0 (2025-06-14) // gemini-2.5-pro-preview-03-25 | 1.0 (2025-05-22) // gemini-2.5-flash-preview-04-17 | let input_file_path = "root/canvas.rs"; let input_file_content = include_str!("evals/fixtures/translate_doc_comments/before.rs"); let edit_description = "Translate all doc comments to Italian"; eval_utils::eval(200, 1., mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the {input_file_path} file and edit it (without overwriting it), translating all the doc comments to italian. "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: None, end_line: None, }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, input_file_content, )], ), message( Assistant, [tool_use( "tool_2", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff("Doc comments were translated to Italian"), )) }); } #[test] #[cfg_attr(not(feature = "unit-eval"), ignore)] fn eval_use_wasi_sdk_in_compile_parser_to_wasm() { // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 0.96 (2025-06-14) // claude-sonnet-4 | 0.11 (2025-06-14) // gemini-2.5-pro-preview-latest | 0.99 (2025-06-16) // gemini-2.5-flash-preview-04-17 | let input_file_path = "root/lib.rs"; let input_file_content = include_str!("evals/fixtures/use_wasi_sdk_in_compile_parser_to_wasm/before.rs"); let edit_description = "Update compile_parser_to_wasm to use wasi-sdk instead of emscripten"; eval_utils::eval(100, 0.95, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message( User, [text(formatdoc! {" Read the `{input_file_path}` file and change `compile_parser_to_wasm` to use `wasi-sdk` instead of emscripten. Use `ureq` to download the SDK for the current platform and architecture. Extract the archive into a sibling of `lib` inside the `tree-sitter` directory in the cache_dir. Compile the parser to wasm using the `bin/clang` executable (or `bin/clang.exe` on windows) that's inside of the archive. Don't re-download the SDK if that executable already exists. Use these clang flags: -fPIC -shared -Os -Wl,--export=tree_sitter_{{language_name}} Here are the available wasi-sdk assets: - wasi-sdk-25.0-x86_64-macos.tar.gz - wasi-sdk-25.0-arm64-macos.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-linux.tar.gz - wasi-sdk-25.0-arm64-linux.tar.gz - wasi-sdk-25.0-x86_64-windows.tar.gz "})], ), message( Assistant, [tool_use( "tool_1", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: Some(971), end_line: Some(1050), }, )], ), message( User, [tool_result( "tool_1", ReadFileTool::NAME, lines(input_file_content, 971..1050), )], ), message( Assistant, [tool_use( "tool_2", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: Some(1050), end_line: Some(1100), }, )], ), message( User, [tool_result( "tool_2", ReadFileTool::NAME, lines(input_file_content, 1050..1100), )], ), message( Assistant, [tool_use( "tool_3", ReadFileTool::NAME, ReadFileToolInput { path: input_file_path.into(), start_line: Some(1100), end_line: Some(1150), }, )], ), message( User, [tool_result( "tool_3", ReadFileTool::NAME, lines(input_file_content, 1100..1150), )], ), message( Assistant, [tool_use( "tool_4", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::judge_diff(indoc! {" - The compile_parser_to_wasm method has been changed to use wasi-sdk - ureq is used to download the SDK for current platform and architecture "}), )) }); } #[test] #[cfg_attr(not(feature = "unit-eval"), ignore)] fn eval_disable_cursor_blinking() { // Model | Pass rate // ============================================ // // claude-3.7-sonnet | 0.59 (2025-07-14) // claude-sonnet-4 | 0.81 (2025-07-14) // gemini-2.5-pro | 0.95 (2025-07-14) // gemini-2.5-flash-preview-04-17 | 0.78 (2025-07-14) let input_file_path = "root/editor.rs"; let input_file_content = include_str!("evals/fixtures/disable_cursor_blinking/before.rs"); let edit_description = "Comment out the call to `BlinkManager::enable`"; let possible_diffs = vec![ include_str!("evals/fixtures/disable_cursor_blinking/possible-01.diff"), include_str!("evals/fixtures/disable_cursor_blinking/possible-02.diff"), include_str!("evals/fixtures/disable_cursor_blinking/possible-03.diff"), include_str!("evals/fixtures/disable_cursor_blinking/possible-04.diff"), ]; eval_utils::eval(100, 0.51, mismatched_tag_threshold(0.05), move || { run_eval(EvalInput::from_conversation( vec![ message(User, [text("Let's research how to cursor blinking works.")]), message( Assistant, [tool_use( "tool_1", GrepTool::NAME, GrepToolInput { regex: "blink".into(), include_pattern: None, offset: 0, case_sensitive: false, }, )], ), message( User, [tool_result( "tool_1", GrepTool::NAME, [ lines(input_file_content, 100..400), lines(input_file_content, 800..1300), lines(input_file_content, 1600..2000), lines(input_file_content, 5000..5500), lines(input_file_content, 8000..9000), lines(input_file_content, 18455..18470), lines(input_file_content, 20000..20500), lines(input_file_content, 21000..21300), ] .join("Match found:\n\n"), )], ), message( User, [text(indoc! {" Comment out the lines that interact with the BlinkManager. Keep the outer `update` blocks, but comments everything that's inside (including if statements). Don't add additional comments. "})], ), message( Assistant, [tool_use( "tool_4", EditFileTool::NAME, EditFileToolInput { display_description: edit_description.into(), path: input_file_path.into(), mode: EditFileMode::Edit, }, )], ), ], Some(input_file_content.into()), EvalAssertion::assert_diff_any(possible_diffs.clone()), )) }); } #[test] #[cfg_attr(not(feature = "unit-eval"), ignore)] fn eval_from_pixels_constructor() { // Results for 2025-06-13 // // The outcome of this evaluation depends heavily on the LINE_HINT_TOLERANCE // value. Higher values improve the pass rate but may sometimes cause // edits to be misapplied. In the context of this eval, this means // the agent might add from_pixels tests in incorrect locations // (e.g., at the beginning of the file), yet the evaluation may still // rate it highly.