| @@ -1,83 +1,161 @@ |
| 1 | 1 | use agent::{RequestKind, ThreadEvent, ThreadStore}; |
| 2 | | −use anyhow::{Result, anyhow}; |
| 2 | +use anyhow::{Context as _, Result, anyhow}; |
| 3 | 3 | use assistant_tool::ToolWorkingSet; |
| 4 | +use client::proto::LspWorkProgress; |
| 4 | 5 | use dap::DapRegistry; |
| 5 | | −use futures::channel::oneshot; |
| 6 | | −use gpui::{App, Task}; |
| 7 | | −use language_model::{LanguageModel, StopReason}; |
| 8 | | −use project::Project; |
| 9 | | −use serde::Deserialize; |
| 10 | | −use std::process::Command; |
| 11 | | −use std::sync::Arc; |
| 6 | +use futures::channel::{mpsc, oneshot}; |
| 7 | +use futures::{FutureExt, StreamExt as _}; |
| 8 | +use gpui::{App, AsyncApp, Entity, Task}; |
| 9 | +use handlebars::Handlebars; |
| 10 | +use language::{DiagnosticSeverity, OffsetRangeExt}; |
| 11 | +use language_model::{ |
| 12 | + LanguageModel, LanguageModelRequest, LanguageModelRequestMessage, MessageContent, Role, |
| 13 | + StopReason, TokenUsage, |
| 14 | +}; |
| 15 | +use project::{LspStore, Project, ProjectPath}; |
| 16 | +use serde::{Deserialize, Serialize}; |
| 17 | +use std::fmt::Write as _; |
| 18 | +use std::fs::File; |
| 19 | +use std::io::Write as _; |
| 20 | +use std::sync::{Arc, Mutex}; |
| 21 | +use std::time::Duration; |
| 12 | 22 | use std::{ |
| 13 | 23 | fs, |
| 14 | 24 | path::{Path, PathBuf}, |
| 15 | 25 | }; |
| 26 | +use unindent::Unindent as _; |
| 27 | +use util::ResultExt as _; |
| 28 | +use util::command::new_smol_command; |
| 29 | +use util::serde::default_true; |
| 16 | 30 | |
| 17 | 31 | use crate::AgentAppState; |
| 18 | 32 | |
| 19 | | −#[derive(Debug, Deserialize)] |
| 33 | +pub const EXAMPLES_DIR: &str = "./crates/eval/examples"; |
| 34 | +pub const REPOS_DIR: &str = "./crates/eval/repos"; |
| 35 | +pub const WORKTREES_DIR: &str = "./crates/eval/worktrees"; |
| 36 | + |
| 37 | +#[derive(Clone, Debug, Deserialize)] |
| 20 | 38 | pub struct ExampleBase { |
| 21 | | − pub path: PathBuf, |
| 39 | + pub url: String, |
| 22 | 40 | pub revision: String, |
| 41 | + pub language_extension: Option<String>, |
| 42 | + pub insert_id: Option<String>, |
| 43 | + #[serde(default = "default_true")] |
| 44 | + pub require_lsp: bool, |
| 23 | 45 | } |
| 24 | 46 | |
| 25 | | −#[derive(Debug)] |
| 47 | +#[derive(Clone, Debug)] |
| 26 | 48 | pub struct Example { |
| 49 | + pub name: String, |
| 50 | + /// Content of `base.toml` |
| 27 | 51 | pub base: ExampleBase, |
| 28 | | − |
| 29 | | − /// Content of the prompt.md file |
| 52 | + /// Content of `prompt.md` |
| 30 | 53 | pub prompt: String, |
| 54 | + /// Content of `criteria.md` |
| 55 | + pub criteria: String, |
| 56 | + /// Markdown log file to append to |
| 57 | + pub log_file: Arc<Mutex<File>>, |
| 58 | +} |
| 59 | + |
| 60 | +#[derive(Debug, Serialize, Deserialize, Clone)] |
| 61 | +pub struct RunOutput { |
| 62 | + pub repository_diff: String, |
| 63 | + pub diagnostics: String, |
| 64 | + pub response_count: usize, |
| 65 | + pub token_usage: TokenUsage, |
| 66 | +} |
| 31 | 67 | |
| 32 | | − /// Content of the rubric.md file |
| 33 | | − pub _rubric: String, |
| 68 | +#[derive(Debug, Clone, Serialize, Deserialize)] |
| 69 | +pub struct JudgeInput { |
| 70 | + pub repository_diff: String, |
| 71 | + pub criteria: String, |
| 72 | +} |
| 73 | + |
| 74 | +#[derive(Debug, Clone, Serialize, Deserialize)] |
| 75 | +pub struct JudgeOutput { |
| 76 | + pub analysis: String, |
| 77 | + pub score: u32, |
| 34 | 78 | } |
| 35 | 79 | |
| 36 | 80 | impl Example { |
| 37 | | − /// Load an example from a directory containing base.toml, prompt.md, and rubric.md |
| 38 | | − pub fn load_from_directory<P: AsRef<Path>>(dir_path: P) -> Result<Self> { |
| 39 | | − let base_path = dir_path.as_ref().join("base.toml"); |
| 40 | | − let prompt_path = dir_path.as_ref().join("prompt.md"); |
| 41 | | − let rubric_path = dir_path.as_ref().join("rubric.md"); |
| 81 | + /// Load an example from a directory containing base.toml, prompt.md, and criteria.md |
| 82 | + pub fn load_from_directory(dir_path: &Path, run_dir: &Path) -> Result<Self> { |
| 83 | + let name = dir_path.file_name().unwrap().to_string_lossy().to_string(); |
| 84 | + let base_path = dir_path.join("base.toml"); |
| 85 | + let prompt_path = dir_path.join("prompt.md"); |
| 86 | + let criteria_path = dir_path.join("criteria.md"); |
| 42 | 87 | |
| 43 | | − let mut base: ExampleBase = toml::from_str(&fs::read_to_string(&base_path)?)?; |
| 44 | | − base.path = base.path.canonicalize()?; |
| 88 | + let log_file_path = run_dir.join(format!( |
| 89 | + "{}.md", |
| 90 | + dir_path.file_name().unwrap().to_str().unwrap() |
| 91 | + )); |
| 92 | + let log_file = Arc::new(Mutex::new(File::create(&log_file_path).unwrap())); |
| 93 | + println!("{}> Logging to {:?}", name, log_file_path); |
| 45 | 94 | |
| 46 | 95 | Ok(Example { |
| 47 | | − base, |
| 48 | | − prompt: fs::read_to_string(prompt_path)?, |
| 49 | | − _rubric: fs::read_to_string(rubric_path)?, |
| 96 | + name, |
| 97 | + base: toml::from_str(&fs::read_to_string(&base_path)?)?, |
| 98 | + prompt: fs::read_to_string(prompt_path.clone())?, |
| 99 | + criteria: fs::read_to_string(criteria_path.clone())?, |
| 100 | + log_file, |
| 50 | 101 | }) |
| 51 | 102 | } |
| 52 | 103 | |
| 104 | + pub fn worktree_path(&self) -> PathBuf { |
| 105 | + Path::new(WORKTREES_DIR) |
| 106 | + .canonicalize() |
| 107 | + .context(format!("No such directory {WORKTREES_DIR}")) |
| 108 | + .unwrap() |
| 109 | + .join(&self.name) |
| 110 | + } |
| 111 | + |
| 53 | 112 | /// Set up the example by checking out the specified Git revision |
| 54 | | − pub fn setup(&self) -> Result<()> { |
| 55 | | − // Check if the directory exists |
| 56 | | − let path = Path::new(&self.base.path); |
| 57 | | − anyhow::ensure!(path.exists(), "Path does not exist: {:?}", self.base.path); |
| 58 | | − |
| 59 | | − // Change to the project directory and checkout the specified revision |
| 60 | | − let output = Command::new("git") |
| 61 | | − .current_dir(&self.base.path) |
| 62 | | − .arg("checkout") |
| 63 | | − .arg(&self.base.revision) |
| 64 | | − .output()?; |
| 65 | | − anyhow::ensure!( |
| 66 | | − output.status.success(), |
| 67 | | − "Failed to checkout revision {}: {}", |
| 68 | | − self.base.revision, |
| 69 | | − String::from_utf8_lossy(&output.stderr), |
| 70 | | − ); |
| 113 | + pub async fn setup(&self) -> Result<()> { |
| 114 | + let repo_path = repo_path_for_url(&self.base.url); |
| 115 | + |
| 116 | + run_git( |
| 117 | + &repo_path, |
| 118 | + &["fetch", "--depth", "1", "origin", &self.base.revision], |
| 119 | + ) |
| 120 | + .await?; |
| 121 | + |
| 122 | + let worktree_path = self.worktree_path(); |
| 123 | + |
| 124 | + if worktree_path.is_dir() { |
| 125 | + println!("{}> Resetting existing worktree", self.name); |
| 126 | + |
| 127 | + // TODO: consider including "-x" to remove ignored files. The downside of this is that |
| 128 | + // it will also remove build artifacts, and so prevent incremental reuse there. |
| 129 | + run_git(&worktree_path, &["clean", "--force", "-d"]).await?; |
| 130 | + run_git(&worktree_path, &["reset", "--hard", "HEAD"]).await?; |
| 131 | + run_git(&worktree_path, &["checkout", &self.base.revision]).await?; |
| 132 | + } else { |
| 133 | + println!("{}> Creating worktree", self.name); |
| 134 | + |
| 135 | + let worktree_path_string = worktree_path.to_string_lossy().to_string(); |
| 136 | + |
| 137 | + run_git( |
| 138 | + &repo_path, |
| 139 | + &[ |
| 140 | + "worktree", |
| 141 | + "add", |
| 142 | + "-f", |
| 143 | + &worktree_path_string, |
| 144 | + &self.base.revision, |
| 145 | + ], |
| 146 | + ) |
| 147 | + .await?; |
| 148 | + } |
| 71 | 149 | |
| 72 | 150 | Ok(()) |
| 73 | 151 | } |
| 74 | 152 | |
| 75 | 153 | pub fn run( |
| 76 | | − self, |
| 154 | + &self, |
| 77 | 155 | model: Arc<dyn LanguageModel>, |
| 78 | 156 | app_state: Arc<AgentAppState>, |
| 79 | 157 | cx: &mut App, |
| 80 | | − ) -> Task<Result<()>> { |
| 158 | + ) -> Task<Result<RunOutput>> { |
| 81 | 159 | let project = Project::local( |
| 82 | 160 | app_state.client.clone(), |
| 83 | 161 | app_state.node_runtime.clone(), |
| @@ -89,30 +167,119 @@ impl Example { · 5 unmodified lines |
| 89 | 167 | cx, |
| 90 | 168 | ); |
| 91 | 169 | |
| 170 | + let worktree_path = self.worktree_path(); |
| 92 | 171 | let worktree = project.update(cx, |project, cx| { |
| 93 | | − project.create_worktree(self.base.path, true, cx) |
| 172 | + project.create_worktree(&worktree_path, true, cx) |
| 94 | 173 | }); |
| 95 | 174 | |
| 96 | 175 | let tools = Arc::new(ToolWorkingSet::default()); |
| 97 | 176 | let thread_store = |
| 98 | 177 | ThreadStore::load(project.clone(), tools, app_state.prompt_builder.clone(), cx); |
| 178 | + let this = self.clone(); |
| 99 | 179 | |
| 100 | | − println!("USER:"); |
| 101 | | − println!("{}", self.prompt); |
| 102 | | − println!("ASSISTANT:"); |
| 103 | 180 | cx.spawn(async move |cx| { |
| 104 | | − worktree.await?; |
| 181 | + let worktree = worktree.await?; |
| 182 | + |
| 183 | + // Wait for worktree scan to finish before choosing a file to open. |
| 184 | + worktree |
| 185 | + .update(cx, |worktree, _cx| { |
| 186 | + worktree.as_local().unwrap().scan_complete() |
| 187 | + })? |
| 188 | + .await; |
| 189 | + |
| 190 | + let lsp_open_handle_and_store = if this.base.require_lsp { |
| 191 | + let language_extension = this.base.language_extension.as_deref().context( |
| 192 | + "language_extension field is required in base.toml when `require_lsp == true`", |
| 193 | + )?; |
| 194 | + |
| 195 | + // Open a file that matches the language to cause LSP to start. |
| 196 | + let language_file = worktree.read_with(cx, |worktree, _cx| { |
| 197 | + worktree |
| 198 | + .files(false, 0) |
| 199 | + .find_map(|e| { |
| 200 | + if e.path.clone().extension().and_then(|ext| ext.to_str()) |
| 201 | + == Some(language_extension) |
| 202 | + { |
| 203 | + Some(ProjectPath { |
| 204 | + worktree_id: worktree.id(), |
| 205 | + path: e.path.clone(), |
| 206 | + }) |
| 207 | + } else { |
| 208 | + None |
| 209 | + } |
| 210 | + }) |
| 211 | + .context("Failed to find a file for example language") |
| 212 | + })??; |
| 213 | + |
| 214 | + let open_language_file_buffer_task = project.update(cx, |project, cx| { |
| 215 | + project.open_buffer(language_file.clone(), cx) |
| 216 | + })?; |
| 217 | + |
| 218 | + let language_file_buffer = open_language_file_buffer_task.await?; |
| 219 | + |
| 220 | + let (lsp_open_handle, lsp_store) = project.update(cx, |project, cx| { |
| 221 | + ( |
| 222 | + project.register_buffer_with_language_servers(&language_file_buffer, cx), |
| 223 | + project.lsp_store().clone(), |
| 224 | + ) |
| 225 | + })?; |
| 226 | + |
| 227 | + // TODO: remove this once the diagnostics tool waits for new diagnostics |
| 228 | + cx.background_executor().timer(Duration::new(5, 0)).await; |
| 229 | + wait_for_lang_server(&lsp_store, this.name.clone(), cx).await?; |
| 230 | + |
| 231 | + lsp_store.update(cx, |lsp_store, cx| { |
| 232 | + lsp_open_handle.update(cx, |buffer, cx| { |
| 233 | + buffer.update(cx, |buffer, cx| { |
| 234 | + let has_language_server = lsp_store |
| 235 | + .language_servers_for_local_buffer(buffer, cx) |
| 236 | + .next() |
| 237 | + .is_some(); |
| 238 | + if has_language_server { |
| 239 | + Ok(()) |
| 240 | + } else { |
| 241 | + Err(anyhow!( |
| 242 | + "`{:?}` was opened to cause the language server to start, \ |
| 243 | + but no language servers are registered for its buffer. \ |
| 244 | + Set `require_lsp = false` in `base.toml` to skip this.", |
| 245 | + language_file |
| 246 | + )) |
| 247 | + } |
| 248 | + }) |
| 249 | + }) |
| 250 | + })??; |
| 251 | + |
| 252 | + Some((lsp_open_handle, lsp_store)) |
| 253 | + } else { |
| 254 | + None |
| 255 | + }; |
| 256 | + |
| 257 | + if std::env::var("ZED_EVAL_SETUP_ONLY").is_ok() { |
| 258 | + return Err(anyhow!("Setup only mode")); |
| 259 | + } |
| 260 | + |
| 105 | 261 | let thread_store = thread_store.await; |
| 106 | 262 | let thread = |
| 107 | 263 | thread_store.update(cx, |thread_store, cx| thread_store.create_thread(cx))?; |
| 108 | 264 | |
| 265 | + { |
| 266 | + let mut log_file = this.log_file.lock().unwrap(); |
| 267 | + writeln!(&mut log_file, "👤 USER:").log_err(); |
| 268 | + writeln!(&mut log_file, "{}", this.prompt).log_err(); |
| 269 | + writeln!(&mut log_file, "🤖 ASSISTANT:").log_err(); |
| 270 | + log_file.flush().log_err(); |
| 271 | + } |
| 272 | + |
| 109 | 273 | let (tx, rx) = oneshot::channel(); |
| 110 | 274 | let mut tx = Some(tx); |
| 111 | 275 | |
| 112 | | − let _subscription = |
| 113 | | − cx.subscribe( |
| 114 | | − &thread, |
| 115 | | − move |thread, event: &ThreadEvent, cx| match event { |
| 276 | + let _subscription = cx.subscribe(&thread, { |
| 277 | + let log_file = this.log_file.clone(); |
| 278 | + let name = this.name.clone(); |
| 279 | + move |thread, event: &ThreadEvent, cx| { |
| 280 | + let mut log_file = log_file.lock().unwrap(); |
| 281 | + |
| 282 | + match event { |
| 116 | 283 | ThreadEvent::Stopped(reason) => match reason { |
| 117 | 284 | Ok(StopReason::EndTurn) => { |
| 118 | 285 | if let Some(tx) = tx.take() { |
| @@ -137,15 +304,16 @@ impl Example { · 18 unmodified lines |
| 137 | 304 | } |
| 138 | 305 | } |
| 139 | 306 | ThreadEvent::StreamedAssistantText(_, chunk) => { |
| 140 | | − print!("{}", chunk); |
| 307 | + write!(&mut log_file, "{}", chunk).log_err(); |
| 141 | 308 | } |
| 142 | 309 | ThreadEvent::StreamedAssistantThinking(_, chunk) => { |
| 143 | | − print!("{}", chunk); |
| 310 | + write!(&mut log_file, "{}", chunk).log_err(); |
| 144 | 311 | } |
| 145 | 312 | ThreadEvent::UsePendingTools { tool_uses } => { |
| 146 | | − println!("\n\nUSING TOOLS:"); |
| 313 | + writeln!(&mut log_file, "\n\nUSING TOOLS:").log_err(); |
| 147 | 314 | for tool_use in tool_uses { |
| 148 | | − println!("{}: {}", tool_use.name, tool_use.input); |
| 315 | + writeln!(&mut log_file, "{}: {}", tool_use.name, tool_use.input) |
| 316 | + .log_err(); |
| 149 | 317 | } |
| 150 | 318 | } |
| 151 | 319 | ThreadEvent::ToolFinished { |
| @@ -154,25 +322,331 @@ impl Example { · 2 unmodified lines |
| 154 | 322 | .. |
| 155 | 323 | } => { |
| 156 | 324 | if let Some(tool_use) = pending_tool_use { |
| 157 | | − println!("\nTOOL FINISHED: {}", tool_use.name); |
| 325 | + let message = format!("TOOL FINISHED: {}", tool_use.name); |
| 326 | + println!("{name}> {message}"); |
| 327 | + writeln!(&mut log_file, "\n{}", message).log_err(); |
| 158 | 328 | } |
| 159 | 329 | if let Some(tool_result) = thread.read(cx).tool_result(tool_use_id) { |
| 160 | | − println!("\n{}\n", tool_result.content); |
| 330 | + let message = format!("\n{}\n", tool_result.content); |
| 331 | + writeln!(&mut log_file, "{}", message).log_err(); |
| 161 | 332 | } |
| 162 | 333 | } |
| 163 | 334 | _ => {} |
| 164 | | − }, |
| 165 | | − )?; |
| 335 | + } |
| 336 | + |
| 337 | + log_file.flush().log_err(); |
| 338 | + } |
| 339 | + })?; |
| 166 | 340 | |
| 167 | 341 | thread.update(cx, |thread, cx| { |
| 168 | 342 | let context = vec![]; |
| 169 | | − thread.insert_user_message(self.prompt.clone(), context, None, cx); |
| 343 | + thread.insert_user_message(this.prompt.clone(), context, None, cx); |
| 170 | 344 | thread.send_to_model(model, RequestKind::Chat, cx); |
| 171 | 345 | })?; |
| 172 | 346 | |
| 173 | 347 | rx.await??; |
| 174 | 348 | |
| 175 | | − Ok(()) |
| 349 | + if let Some((_, lsp_store)) = lsp_open_handle_and_store.as_ref() { |
| 350 | + wait_for_lang_server(lsp_store, this.name.clone(), cx).await?; |
| 351 | + } |
| 352 | + |
| 353 | + let repository_diff = this.repository_diff().await?; |
| 354 | + let diagnostics = cx |
| 355 | + .update(move |cx| { |
| 356 | + cx.spawn(async move |cx| query_lsp_diagnostics(project, cx).await) |
| 357 | + })? |
| 358 | + .await?; |
| 359 | + |
| 360 | + drop(lsp_open_handle_and_store); |
| 361 | + |
| 362 | + thread.update(cx, |thread, _cx| { |
| 363 | + let response_count = thread |
| 364 | + .messages() |
| 365 | + .filter(|message| message.role == language_model::Role::Assistant) |
| 366 | + .count(); |
| 367 | + RunOutput { |
| 368 | + repository_diff, |
| 369 | + diagnostics, |
| 370 | + response_count, |
| 371 | + token_usage: thread.cumulative_token_usage(), |
| 372 | + } |
| 373 | + }) |
| 176 | 374 | }) |
| 177 | 375 | } |
| 376 | + |
| 377 | + pub async fn judge( |
| 378 | + &mut self, |
| 379 | + model: Arc<dyn LanguageModel>, |
| 380 | + repository_diff: String, |
| 381 | + cx: &AsyncApp, |
| 382 | + ) -> Result<JudgeOutput> { |
| 383 | + let judge_prompt = include_str!("judge_prompt.hbs"); |
| 384 | + let judge_prompt_name = "judge_prompt"; |
| 385 | + let mut handlebars = Handlebars::new(); |
| 386 | + handlebars.register_template_string(judge_prompt_name, judge_prompt)?; |
| 387 | + let prompt = handlebars.render( |
| 388 | + judge_prompt_name, |
| 389 | + &JudgeInput { |
| 390 | + repository_diff, |
| 391 | + criteria: self.criteria.clone(), |
| 392 | + }, |
| 393 | + )?; |
| 394 | + |
| 395 | + let request = LanguageModelRequest { |
| 396 | + messages: vec![LanguageModelRequestMessage { |
| 397 | + role: Role::User, |
| 398 | + content: vec![MessageContent::Text(prompt)], |
| 399 | + cache: false, |
| 400 | + }], |
| 401 | + temperature: None, |
| 402 | + tools: Vec::new(), |
| 403 | + stop: Vec::new(), |
| 404 | + }; |
| 405 | + |
| 406 | + let response = send_language_model_request(model, request, cx).await?; |
| 407 | + |
| 408 | + let mut log_file = self.log_file.lock().unwrap(); |
| 409 | + |
| 410 | + writeln!(&mut log_file, "\n\n").log_err(); |
| 411 | + writeln!(&mut log_file, "========================================").log_err(); |
| 412 | + writeln!(&mut log_file, " JUDGE OUTPUT ").log_err(); |
| 413 | + writeln!(&mut log_file, "========================================").log_err(); |
| 414 | + writeln!(&mut log_file, "\n{}", &response).log_err(); |
| 415 | + |
| 416 | + parse_judge_output(&response) |
| 417 | + } |
| 418 | + |
| 419 | + pub async fn repository_diff(&self) -> Result<String> { |
| 420 | + let worktree_path = self.worktree_path(); |
| 421 | + run_git(&worktree_path, &["add", "-N"]).await?; |
| 422 | + run_git(&worktree_path, &["diff"]).await |
| 423 | + } |
| 424 | +} |
| 425 | + |
| 426 | +fn wait_for_lang_server( |
| 427 | + lsp_store: &Entity<LspStore>, |
| 428 | + name: String, |
| 429 | + cx: &mut AsyncApp, |
| 430 | +) -> Task<Result<()>> { |
| 431 | + if cx |
| 432 | + .update(|cx| !has_pending_lang_server_work(lsp_store, cx)) |
| 433 | + .unwrap() |
| 434 | + || std::env::var("ZED_EVAL_SKIP_LS_WAIT").is_ok() |
| 435 | + { |
| 436 | + return Task::ready(anyhow::Ok(())); |
| 437 | + } |
| 438 | + |
| 439 | + println!("{}> ⏵ Waiting for language server", name); |
| 440 | + |
| 441 | + let (mut tx, mut rx) = mpsc::channel(1); |
| 442 | + |
| 443 | + let subscription = |
| 444 | + cx.subscribe(&lsp_store, { |
| 445 | + let name = name.clone(); |
| 446 | + move |lsp_store, event, cx| { |
| 447 | + match event { |
| 448 | + project::LspStoreEvent::LanguageServerUpdate { |
| 449 | + message: |
| 450 | + client::proto::update_language_server::Variant::WorkProgress( |
| 451 | + LspWorkProgress { |
| 452 | + message: Some(message), |
| 453 | + .. |
| 454 | + }, |
| 455 | + ), |
| 456 | + .. |
| 457 | + } => println!("{name}> ⟲ {message}"), |
| 458 | + _ => {} |
| 459 | + } |
| 460 | + |
| 461 | + if !has_pending_lang_server_work(&lsp_store, cx) { |
| 462 | + tx.try_send(()).ok(); |
| 463 | + } |
| 464 | + } |
| 465 | + }); |
| 466 | + |
| 467 | + cx.spawn(async move |cx| { |
| 468 | + let timeout = cx.background_executor().timer(Duration::new(60 * 5, 0)); |
| 469 | + let result = futures::select! { |
| 470 | + _ = rx.next() => { |
| 471 | + println!("{}> ⚑ Language server idle", name); |
| 472 | + anyhow::Ok(()) |
| 473 | + }, |
| 474 | + _ = timeout.fuse() => { |
| 475 | + Err(anyhow!("LSP wait timed out after 5 minutes")) |
| 476 | + } |
| 477 | + }; |
| 478 | + drop(subscription); |
| 479 | + result |
| 480 | + }) |
| 481 | +} |
| 482 | + |
| 483 | +fn has_pending_lang_server_work(lsp_store: &Entity<LspStore>, cx: &App) -> bool { |
| 484 | + lsp_store |
| 485 | + .read(cx) |
| 486 | + .language_server_statuses() |
| 487 | + .any(|(_, status)| !status.pending_work.is_empty()) |
| 488 | +} |
| 489 | + |
| 490 | +async fn query_lsp_diagnostics(project: Entity<Project>, cx: &mut AsyncApp) -> Result<String> { |
| 491 | + let paths_with_diagnostics = project.update(cx, |project, cx| { |
| 492 | + project |
| 493 | + .diagnostic_summaries(true, cx) |
| 494 | + .filter(|(_, _, summary)| summary.error_count > 0 || summary.warning_count > 0) |
| 495 | + .map(|(project_path, _, _)| project_path) |
| 496 | + .collect::<Vec<_>>() |
| 497 | + })?; |
| 498 | + |
| 499 | + let mut output = String::new(); |
| 500 | + for project_path in paths_with_diagnostics { |
| 501 | + let buffer = project |
| 502 | + .update(cx, |project, cx| project.open_buffer(project_path, cx))? |
| 503 | + .await?; |
| 504 | + let snapshot = buffer.read_with(cx, |buffer, _cx| buffer.snapshot())?; |
| 505 | + |
| 506 | + for (_, group) in snapshot.diagnostic_groups(None) { |
| 507 | + let entry = &group.entries[group.primary_ix]; |
| 508 | + let range = entry.range.to_point(&snapshot); |
| 509 | + let severity = match entry.diagnostic.severity { |
| 510 | + DiagnosticSeverity::ERROR => "error", |
| 511 | + DiagnosticSeverity::WARNING => "warning", |
| 512 | + _ => continue, |
| 513 | + }; |
| 514 | + |
| 515 | + writeln!( |
| 516 | + output, |
| 517 | + "{} at line {}: {}", |
| 518 | + severity, |
| 519 | + range.start.row + 1, |
| 520 | + entry.diagnostic.message |
| 521 | + )?; |
| 522 | + } |
| 523 | + } |
| 524 | + anyhow::Ok(output) |
| 525 | +} |
| 526 | + |
| 527 | +fn parse_judge_output(response: &str) -> Result<JudgeOutput> { |
| 528 | + let analysis = get_tag("analysis", response)?.to_string(); |
| 529 | + let score = get_tag("score", response)? |
| 530 | + .parse() |
| 531 | + .context("error parsing score")?; |
| 532 | + |
| 533 | + Ok(JudgeOutput { analysis, score }) |
| 534 | +} |
| 535 | + |
| 536 | +fn get_tag(name: &'static str, response: &str) -> Result<String> { |
| 537 | + let start_tag = format!("<{}>", name); |
| 538 | + let end_tag = format!("</{}>", name); |
| 539 | + |
| 540 | + let start_ix = response |
| 541 | + .find(&start_tag) |
| 542 | + .context(format!("{} start tag not found", name))?; |
| 543 | + let content_start_ix = start_ix + start_tag.len(); |
| 544 | + |
| 545 | + let end_ix = content_start_ix |
| 546 | + + response[content_start_ix..] |
| 547 | + .find(&end_tag) |
| 548 | + .context(format!("{} end tag not found", name))?; |
| 549 | + |
| 550 | + let content = response[content_start_ix..end_ix].trim().unindent(); |
| 551 | + |
| 552 | + anyhow::Ok(content) |
| 553 | +} |
| 554 | + |
| 555 | +pub fn repo_path_for_url(repo_url: &str) -> PathBuf { |
| 556 | + let repo_name = repo_url |
| 557 | + .trim_start_matches("https://") |
| 558 | + .replace(|c: char| !c.is_alphanumeric(), "-"); |
| 559 | + Path::new(REPOS_DIR) |
| 560 | + .canonicalize() |
| 561 | + .context(format!("No such directory {REPOS_DIR}")) |
| 562 | + .unwrap() |
| 563 | + .join(repo_name) |
| 564 | +} |
| 565 | + |
| 566 | +pub async fn run_git(repo_path: &Path, args: &[&str]) -> Result<String> { |
| 567 | + let output = new_smol_command("git") |
| 568 | + .current_dir(repo_path) |
| 569 | + .args(args) |
| 570 | + .output() |
| 571 | + .await?; |
| 572 | + |
| 573 | + if output.status.success() { |
| 574 | + Ok(String::from_utf8(output.stdout)?.trim().to_string()) |
| 575 | + } else { |
| 576 | + Err(anyhow!( |
| 577 | + "`git {}` within `{}` failed with status: {}\nstderr:\n{}\nstdout:\n{}", |
| 578 | + args.join(" "), |
| 579 | + repo_path.display(), |
| 580 | + output.status, |
| 581 | + String::from_utf8_lossy(&output.stderr), |
| 582 | + String::from_utf8_lossy(&output.stdout), |
| 583 | + )) |
| 584 | + } |
| 585 | +} |
| 586 | + |
| 587 | +pub async fn send_language_model_request( |
| 588 | + model: Arc<dyn LanguageModel>, |
| 589 | + request: LanguageModelRequest, |
| 590 | + cx: &AsyncApp, |
| 591 | +) -> anyhow::Result<String> { |
| 592 | + match model.stream_completion_text(request, &cx).await { |
| 593 | + Ok(mut stream) => { |
| 594 | + let mut full_response = String::new(); |
| 595 | + while let Some(chunk_result) = stream.stream.next().await { |
| 596 | + match chunk_result { |
| 597 | + Ok(chunk_str) => { |
| 598 | + print!("{}", &chunk_str); |
| 599 | + full_response.push_str(&chunk_str); |
| 600 | + } |
| 601 | + Err(err) => { |
| 602 | + return Err(anyhow!( |
| 603 | + "Error receiving response from language model: {err}" |
| 604 | + )); |
| 605 | + } |
| 606 | + } |
| 607 | + } |
| 608 | + Ok(full_response) |
| 609 | + } |
| 610 | + Err(err) => Err(anyhow!( |
| 611 | + "Failed to get response from language model. Error was: {err}" |
| 612 | + )), |
| 613 | + } |
| 614 | +} |
| 615 | + |
| 616 | +#[cfg(test)] |
| 617 | +mod test { |
| 618 | + use super::*; |
| 619 | + |
| 620 | + #[test] |
| 621 | + fn test_parse_judge_output() { |
| 622 | + let response = r#" |
| 623 | + <analysis>The model did a good job but there were still compilations errors.</analysis> |
| 624 | + <score>3</score> |
| 625 | + "# |
| 626 | + .unindent(); |
| 627 | + |
| 628 | + let output = parse_judge_output(&response).unwrap(); |
| 629 | + assert_eq!( |
| 630 | + output.analysis, |
| 631 | + "The model did a good job but there were still compilations errors." |
| 632 | + ); |
| 633 | + assert_eq!(output.score, 3); |
| 634 | + |
| 635 | + let response = r#" |
| 636 | + Text around ignored |
| 637 | + |
| 638 | + <analysis> |
| 639 | + Failed to compile: |
| 640 | + - Error 1 |
| 641 | + - Error 2 |
| 642 | + </analysis> |
| 643 | + |
| 644 | + <score>1</score> |
| 645 | + "# |
| 646 | + .unindent(); |
| 647 | + |
| 648 | + let output = parse_judge_output(&response).unwrap(); |
| 649 | + assert_eq!(output.analysis, "Failed to compile:\n- Error 1\n- Error 2"); |
| 650 | + assert_eq!(output.score, 1); |
| 651 | + } |
| 178 | 652 | } |