//! Headless CLI binary for running Zed's agent in evaluation/benchmark environments.//!//! Designed to work inside containerized environments (like Harbor/termbench) where://! - The repository is already checked out at the working directory//! - The model API key is provided via environment variables//! - Results are written to an output directory (default: `/logs/agent/`)//!//! ## Usage//!//! ```text//! eval-cli --workdir /testbed --model anthropic/claude-sonnet-4-6-latest \//!          --instruction "Fix the bug described in..." --timeout 600//! ```//!//! ## Output//!//! Writes to `--output-dir` (default `/logs/agent/`)://!   - `result.json`  — structured result with status, timing, token usage,//!     step count, and tool-call counts (total and per tool)//!   - `thread.md`    — full conversation as markdown//!   - `thread.json`  — raw thread state as JSON//!//! ## Exit codes//!//! | Code | Meaning |//! |------|---------|//! | 0    | Agent finished |//! | 1    | Error (model/auth/runtime failure) |//! | 2    | Timeout |//! | 3    | Interrupted (SIGTERM/SIGINT) | mod headless; use std::path::PathBuf;use std::process;use std::rc::Rc;use std::str::FromStr;use std::sync::Arc;use std::sync::atomic::{AtomicBool, Ordering};use std::time::{Duration, Instant}; use acp_thread::AgentConnection as _;use agent::{NativeAgent, NativeAgentConnection, Templates, ThreadStore};use agent_client_protocol::schema::v1 as acp;use anyhow::{Context, Result};use clap::Parser;use feature_flags::FeatureFlagAppExt as _; use futures::{FutureExt, select_biased};use gpui::{AppContext as _, AsyncApp, Entity, UpdateGlobal};use language_model::{    ANTHROPIC_PROVIDER_ID, LanguageModel, LanguageModelId, LanguageModelProviderId,    LanguageModelRegistry, SelectedModel,};use project::Project;use settings::SettingsStore;use util::path_list::PathList; use crate::headless::AgentCliAppState; #[derive(Parser, Debug)]#[command(    name = "eval-cli",    about = "Run Zed's agent headlessly in evaluation/benchmark environments")]struct Args {    /// Output current environment variables as JSON to stdout.    /// Used internally by Zed's shell environment capture.    #[arg(long, hide = true)]    printenv: bool,     /// Path to the repository working directory. Defaults to the current directory.    #[arg(long, default_value = ".")]    workdir: PathBuf,     /// Instruction/prompt text. If omitted, read from stdin.    #[arg(long, allow_hyphen_values = true)]    instruction: Option<String>,     /// File containing additional instruction text appended after the task prompt.    #[arg(long)]    instruction_suffix_file: Option<PathBuf>,     /// Language model to use, in `provider/model` format.    #[arg(long, default_value = "anthropic/claude-sonnet-4-6-latest")]    model: String,     /// Maximum wall-clock time in seconds for the agent run.    #[arg(long)]    timeout: Option<u64>,     /// Directory for output artifacts (result.json, thread.md, thread.json).    #[arg(long, default_value = ".")]    output_dir: PathBuf,     /// Disable staff mode (staff mode is enabled by default).    #[arg(long)]    no_staff: bool,     /// Reasoning effort level for models that support thinking (low, medium, high).    /// Defaults to "high" for thinking-capable models.    #[arg(long)]    reasoning_effort: Option<String>,     /// Enable or disable extended thinking. Defaults to model auto-detection if omitted.    #[arg(long)]    thinking: Option<bool>,} enum AgentOutcome {    Completed,    Timeout { seconds: u64 },    Interrupted,} #[derive(serde::Serialize)]struct EvalResult {    status: String,    #[serde(skip_serializing_if = "Option::is_none")]    error: Option<String>,    duration_secs: f64,    #[serde(skip_serializing_if = "Option::is_none")]    timeout_secs: Option<u64>,    model: String,    #[serde(skip_serializing_if = "Option::is_none")]    input_tokens: Option<u64>,    #[serde(skip_serializing_if = "Option::is_none")]    output_tokens: Option<u64>,    #[serde(skip_serializing_if = "Option::is_none")]    cache_creation_input_tokens: Option<u64>,    #[serde(skip_serializing_if = "Option::is_none")]    cache_read_input_tokens: Option<u64>,    /// Number of agent (assistant) turns, i.e. model round-trips in the agentic    /// loop. Reported as "steps" by the eval harness.    #[serde(skip_serializing_if = "Option::is_none")]    step_count: Option<u64>,    /// Total number of tool calls across all steps.    #[serde(skip_serializing_if = "Option::is_none")]    tool_call_count: Option<u64>,    /// Tool calls broken down by tool name.    #[serde(skip_serializing_if = "Option::is_none")]    tool_calls: Option<std::collections::BTreeMap<String, u64>>,} /// Per-run statistics collected from the finished thread, written into/// `result.json` so the post-hoc report can compute success-conditioned metrics.#[derive(Default)]struct RunStats {    token_usage: Option<language_model::TokenUsage>,    step_count: Option<u64>,    tool_call_count: Option<u64>,    tool_calls: Option<std::collections::BTreeMap<String, u64>>,} const EXIT_OK: i32 = 0;const EXIT_ERROR: i32 = 1;const EXIT_TIMEOUT: i32 = 2;const EXIT_INTERRUPTED: i32 = 3;const MODEL_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30);const MODEL_DISCOVERY_POLL_INTERVAL: Duration = Duration::from_millis(100); static TERMINATED: AtomicBool = AtomicBool::new(false); fn main() {    let args = Args::parse();     if args.printenv {        util::shell_env::print_env();        return;    }     env_logger::init();     ctrlc::set_handler(|| {        TERMINATED.store(true, Ordering::SeqCst);    })    .expect("failed to set signal handler");     let instruction = read_instruction(&args).unwrap_or_else(|e| {        eprintln!("Error reading instruction: {e}");        process::exit(EXIT_ERROR);    });     let workdir = args.workdir.canonicalize().unwrap_or_else(|e| {        eprintln!("Invalid --workdir {:?}: {e}", args.workdir);        process::exit(EXIT_ERROR);    });     let output_dir = args.output_dir.clone();    if let Err(e) = std::fs::create_dir_all(&output_dir) {        eprintln!("Error creating output dir {}: {e}", output_dir.display());        process::exit(EXIT_ERROR);    }    let output_dir = output_dir.canonicalize().unwrap_or_else(|e| {        eprintln!("Invalid --output-dir {:?}: {e}", output_dir);        process::exit(EXIT_ERROR);    });     let http_client = Arc::new(reqwest_client::ReqwestClient::new());    let app = gpui_platform::headless().with_http_client(http_client);     app.run(move |cx| {        let app_state = headless::init(cx);        cx.set_staff(!args.no_staff);         // Eval hook: enable additional feature-flag-gated tools (e.g. the LSP        // navigation tools behind `lsp-tool` / `rename-tool`) so experiments can        // measure the agent with tools that aren't yet GA. Comma-separated flag        // names; unset in production.        if let Ok(raw_flags) = std::env::var("ZED_EVAL_ENABLE_FLAGS") {            let flags: Vec<String> = raw_flags                .split(',')                .map(|flag| flag.trim().to_string())                .filter(|flag| !flag.is_empty())                .collect();            if !flags.is_empty() {                cx.update_flags(!args.no_staff, flags);            }        }         let openai_compatible_providers_json = openai_compatible_providers_override();        let anthropic_available_models_json = anthropic_available_models_override();         let model_name = args.model.clone();        let timeout = args.timeout;        let thinking_override = args.thinking;        let reasoning_effort = args.reasoning_effort.clone();         cx.spawn(async move |cx| {            // Each settings change below is applied in its own `cx.update` call (rather than            // inline in the synchronous body of `app.run`) so that GPUI flushes the resulting            // `NotifyGlobalObservers` effect before we move on. Without this, the            // openai_compatible/anthropic provider registration (driven by an            // `observe_global::<SettingsStore>` callback in language_models::init) wouldn't            // have run yet by the time we collect `auth_tasks`, so a newly-added provider's            // `authenticate()` would never get called and it would be permanently stuck            // unauthenticated.            if let Some(providers_json) = &openai_compatible_providers_json {                let result = cx.update(|cx| apply_openai_compatible_providers(providers_json, cx));                if let Err(e) = result {                    eprintln!("Error applying {OPENAI_COMPATIBLE_PROVIDERS_ENV}: {e:#}");                    process::exit(EXIT_ERROR);                }            }             if let Some(models_json) = &anthropic_available_models_json {                let result = cx.update(|cx| apply_anthropic_available_models(models_json, cx));                if let Err(e) = result {                    eprintln!("Error applying {ANTHROPIC_AVAILABLE_MODELS_ENV}: {e:#}");                    process::exit(EXIT_ERROR);                }            }             let auth_tasks = cx.update(|cx| {                LanguageModelRegistry::global(cx).update(cx, |registry, cx| {                    registry                        .providers()                        .iter()                        .map(|p| p.authenticate(cx))                        .collect::<Vec<_>>()                })            });            futures::future::join_all(auth_tasks).await;             let start = Instant::now();             let (outcome, stats) = run_agent(                &app_state,                &workdir,                &instruction,                &model_name,                timeout,                thinking_override,                reasoning_effort.as_deref(),                Some(&output_dir),                openai_compatible_providers_json.as_deref(),                anthropic_available_models_json.as_deref(),                cx,            )            .await;             let duration = start.elapsed();             let (status, error, exit_code) = match &outcome {                Ok(AgentOutcome::Completed) => ("completed".to_string(), None, EXIT_OK),                Ok(AgentOutcome::Timeout { seconds }) => {                    eprintln!("Timeout: agent exceeded {seconds}s time limit");                    ("timeout".to_string(), None, EXIT_TIMEOUT)                }                Ok(AgentOutcome::Interrupted) => {                    eprintln!("Interrupted: received SIGTERM, saved partial output");                    ("interrupted".to_string(), None, EXIT_INTERRUPTED)                }                Err(e) => {                    eprintln!("Error: {e:#}");                    ("error".to_string(), Some(format!("{e:#}")), EXIT_ERROR)                }            };             let token_usage = stats.token_usage;            let result = EvalResult {                status,                error,                duration_secs: duration.as_secs_f64(),                timeout_secs: timeout,                model: model_name.clone(),                input_tokens: token_usage.as_ref().map(|u| u.input_tokens),                output_tokens: token_usage.as_ref().map(|u| u.output_tokens),                cache_creation_input_tokens: token_usage                    .as_ref()                    .filter(|u| u.cache_creation_input_tokens > 0)                    .map(|u| u.cache_creation_input_tokens),                cache_read_input_tokens: token_usage                    .as_ref()                    .filter(|u| u.cache_read_input_tokens > 0)                    .map(|u| u.cache_read_input_tokens),                step_count: stats.step_count,                tool_call_count: stats.tool_call_count,                tool_calls: stats.tool_calls,            };             match serde_json::to_string_pretty(&result) {                Ok(json) => {                    if let Err(e) = std::fs::write(output_dir.join("result.json"), &json) {                        eprintln!("Error writing result.json: {e:#}");                    }                    eprintln!("[eval-cli] result: {json}");                }                Err(e) => eprintln!("Error serializing result: {e:#}"),            }             cx.update(|cx| cx.quit());            process::exit(exit_code);        })        .detach();    });} /// Name of the env var carrying a JSON object to merge into/// `language_models.openai_compatible` user settings before model discovery, in/// the same shape as Zed's `openai_compatible` settings key (provider id ->/// `{ "api_url": ..., "available_models": [...] }`). Lets zed-eval route the/// agent itself through an OpenAI-compatible endpoint (e.g. Baseten) that isn't/// one of Zed's built-in providers, without hardcoding it into eval-cli.const OPENAI_COMPATIBLE_PROVIDERS_ENV: &str = "ZED_OPENAI_COMPATIBLE_PROVIDERS"; fn openai_compatible_providers_override() -> Option<String> {    let raw = std::env::var(OPENAI_COMPATIBLE_PROVIDERS_ENV).ok()?;    if raw.trim().is_empty() {        return None;    }    Some(raw)} fn apply_openai_compatible_providers(providers_json: &str, cx: &mut gpui::App) -> Result<()> {    let settings = format!(r#"{{"language_models": {{"openai_compatible": {providers_json}}}}}"#);    SettingsStore::update_global(cx, |store, cx| {        store.set_user_settings(&settings, cx).result()    })    .context("applying openai_compatible provider settings")?;    Ok(())} /// Name of the env var carrying a JSON array to merge into/// `language_models.anthropic.available_models` user settings before model/// discovery, in the same shape as Zed's `anthropic.available_models` settings/// key (a list of `{ "name": ..., "max_tokens": ..., ... }` entries). Lets/// zed-eval run models that exist on the Anthropic API for the configured/// key (e.g. early-access-program models) but aren't returned by the live/// `/v1/models` listing, without hardcoding them into eval-cli.const ANTHROPIC_AVAILABLE_MODELS_ENV: &str = "ZED_ANTHROPIC_AVAILABLE_MODELS"; fn anthropic_available_models_override() -> Option<String> {    let raw = std::env::var(ANTHROPIC_AVAILABLE_MODELS_ENV).ok()?;    if raw.trim().is_empty() {        return None;    }    Some(raw)} fn apply_anthropic_available_models(models_json: &str, cx: &mut gpui::App) -> Result<()> {    let settings =        format!(r#"{{"language_models": {{"anthropic": {{"available_models": {models_json}}}}}}}"#);    SettingsStore::update_global(cx, |store, cx| {        store.set_user_settings(&settings, cx).result()    })    .context("applying anthropic available_models settings")?;    Ok(())} fn read_instruction(args: &Args) -> Result<String> {    let mut text = if let Some(text) = &args.instruction {        text.clone()    } else {        use std::io::Read;        let mut buf = String::new();        std::io::stdin()            .read_to_string(&mut buf)            .context("reading instruction from stdin")?;        buf    };    anyhow::ensure!(!text.trim().is_empty(), "instruction is empty");     if let Some(path) = &args.instruction_suffix_file {        let suffix = read_instruction_suffix_file(path)?;        text.push_str("\n\n");        text.push_str(&suffix);    }    Ok(text)} fn read_instruction_suffix_file(path: &PathBuf) -> Result<String> {    let suffix = std::fs::read_to_string(path)        .with_context(|| format!("reading instruction suffix file {}", path.display()))?;    let suffix = suffix.trim().to_string();    anyhow::ensure!(!suffix.is_empty(), "instruction suffix file is empty");    Ok(suffix)} async fn wait_for_model(selected: &SelectedModel, cx: &mut AsyncApp) -> Result<()> {    let started_at = Instant::now();     loop {        let found = cx.update(|cx| find_available_model(selected, cx).is_some());        if found {            return Ok(());        }         cx.update(|cx| ensure_provider_authenticated(selected, cx))?;         let selected_provider_has_models = cx.update(|cx| {            LanguageModelRegistry::global(cx)                .read(cx)                .available_models(cx)                .any(|model| model.provider_id() == selected.provider)        });        let should_wait_for_discovery =            selected.provider == ANTHROPIC_PROVIDER_ID || !selected_provider_has_models;         if !should_wait_for_discovery || started_at.elapsed() >= MODEL_DISCOVERY_TIMEOUT {            return Err(cx.update(|cx| model_not_found_error(&selected_model_name(selected), cx)));        }         cx.background_executor()            .timer(MODEL_DISCOVERY_POLL_INTERVAL)            .await;    }} fn ensure_provider_authenticated(selected: &SelectedModel, cx: &gpui::App) -> Result<()> {    let registry = LanguageModelRegistry::global(cx);    let provider = registry        .read(cx)        .provider(&selected.provider)        .ok_or_else(|| anyhow::anyhow!("Provider {} not found", selected.provider.0))?;     anyhow::ensure!(        provider.is_authenticated(cx),        "Provider {} is not authenticated",        selected.provider.0    );     Ok(())} fn find_available_model(    selected: &SelectedModel,    cx: &gpui::App,) -> Option<Arc<dyn LanguageModel>> {    let registry = LanguageModelRegistry::global(cx);    let models = registry.read(cx).available_models(cx).collect::<Vec<_>>();     if let Some(model) = models        .iter()        .find(|model| model.provider_id() == selected.provider && model.id() == selected.model)    {        return Some(model.clone());    }     models        .into_iter()        .filter(|model| {            model.provider_id() == selected.provider                && model_id_matches_selected(&model.provider_id(), &model.id(), &selected.model)        })        .max_by(|left, right| left.id().0.to_string().cmp(&right.id().0.to_string()))} fn model_id_matches_selected(    provider_id: &LanguageModelProviderId,    available: &LanguageModelId,    selected: &LanguageModelId,) -> bool {    if available == selected {        return true;    }     if provider_id != &ANTHROPIC_PROVIDER_ID {        return false;    }

Lines 1–500 of 1020. Load more