| @@ -20,7 +20,6 @@ use parsing::{CodeContextRetriever, ParsedFile}; · 19 unmodified lines |
| 20 | 20 | use project::{Fs, PathChange, Project, ProjectEntryId, WorktreeId}; |
| 21 | 21 | use smol::channel; |
| 22 | 22 | use std::{ |
| 23 | | − cmp::Ordering, |
| 24 | 23 | collections::HashMap, |
| 25 | 24 | path::{Path, PathBuf}, |
| 26 | 25 | sync::Arc, |
| @@ -112,10 +111,10 @@ pub struct VectorStore { · 85 unmodified lines |
| 112 | 111 | database_url: Arc<PathBuf>, |
| 113 | 112 | embedding_provider: Arc<dyn EmbeddingProvider>, |
| 114 | 113 | language_registry: Arc<LanguageRegistry>, |
| 115 | | − db_update_tx: channel::Sender<DbWrite>, |
| 114 | + db_update_tx: channel::Sender<DbOperation>, |
| 116 | 115 | parsing_files_tx: channel::Sender<PendingFile>, |
| 117 | 116 | _db_update_task: Task<()>, |
| 118 | | − _embed_batch_task: Vec<Task<()>>, |
| 117 | + _embed_batch_task: Task<()>, |
| 119 | 118 | _batch_files_task: Task<()>, |
| 120 | 119 | _parsing_files_tasks: Vec<Task<()>>, |
| 121 | 120 | projects: HashMap<WeakModelHandle<Project>, ProjectState>, |
| @@ -128,6 +127,30 @@ struct ProjectState { · 6 unmodified lines |
| 128 | 127 | } |
| 129 | 128 | |
| 130 | 129 | impl ProjectState { |
| 130 | + fn db_id_for_worktree_id(&self, id: WorktreeId) -> Option<i64> { |
| 131 | + self.worktree_db_ids |
| 132 | + .iter() |
| 133 | + .find_map(|(worktree_id, db_id)| { |
| 134 | + if *worktree_id == id { |
| 135 | + Some(*db_id) |
| 136 | + } else { |
| 137 | + None |
| 138 | + } |
| 139 | + }) |
| 140 | + } |
| 141 | + |
| 142 | + fn worktree_id_for_db_id(&self, id: i64) -> Option<WorktreeId> { |
| 143 | + self.worktree_db_ids |
| 144 | + .iter() |
| 145 | + .find_map(|(worktree_id, db_id)| { |
| 146 | + if *db_id == id { |
| 147 | + Some(*worktree_id) |
| 148 | + } else { |
| 149 | + None |
| 150 | + } |
| 151 | + }) |
| 152 | + } |
| 153 | + |
| 131 | 154 | fn update_pending_files(&mut self, pending_file: PendingFile, indexing_time: SystemTime) { |
| 132 | 155 | // If Pending File Already Exists, Replace it with the new one |
| 133 | 156 | // but keep the old indexing time |
| @@ -185,7 +208,7 @@ pub struct SearchResult { · 51 unmodified lines |
| 185 | 208 | pub file_path: PathBuf, |
| 186 | 209 | } |
| 187 | 210 | |
| 188 | | −enum DbWrite { |
| 211 | +enum DbOperation { |
| 189 | 212 | InsertFile { |
| 190 | 213 | worktree_id: i64, |
| 191 | 214 | indexed_file: ParsedFile, |
| @@ -198,6 +221,10 @@ enum DbWrite { · 6 unmodified lines |
| 198 | 221 | path: PathBuf, |
| 199 | 222 | sender: oneshot::Sender<Result<i64>>, |
| 200 | 223 | }, |
| 224 | + FileMTimes { |
| 225 | + worktree_id: i64, |
| 226 | + sender: oneshot::Sender<Result<HashMap<PathBuf, SystemTime>>>, |
| 227 | + }, |
| 201 | 228 | } |
| 202 | 229 | |
| 203 | 230 | enum EmbeddingJob { |
| @@ -243,20 +270,27 @@ impl VectorStore { · 39 unmodified lines |
| 243 | 270 | let _db_update_task = cx.background().spawn(async move { |
| 244 | 271 | while let Ok(job) = db_update_rx.recv().await { |
| 245 | 272 | match job { |
| 246 | | − DbWrite::InsertFile { |
| 273 | + DbOperation::InsertFile { |
| 247 | 274 | worktree_id, |
| 248 | 275 | indexed_file, |
| 249 | 276 | } => { |
| 250 | 277 | log::info!("Inserting Data for {:?}", &indexed_file.path); |
| 251 | 278 | db.insert_file(worktree_id, indexed_file).log_err(); |
| 252 | 279 | } |
| 253 | | − DbWrite::Delete { worktree_id, path } => { |
| 280 | + DbOperation::Delete { worktree_id, path } => { |
| 254 | 281 | db.delete_file(worktree_id, path).log_err(); |
| 255 | 282 | } |
| 256 | | − DbWrite::FindOrCreateWorktree { path, sender } => { |
| 283 | + DbOperation::FindOrCreateWorktree { path, sender } => { |
| 257 | 284 | let id = db.find_or_create_worktree(&path); |
| 258 | 285 | sender.send(id).ok(); |
| 259 | 286 | } |
| 287 | + DbOperation::FileMTimes { |
| 288 | + worktree_id: worktree_db_id, |
| 289 | + sender, |
| 290 | + } => { |
| 291 | + let file_mtimes = db.get_file_mtimes(worktree_db_id); |
| 292 | + sender.send(file_mtimes).ok(); |
| 293 | + } |
| 260 | 294 | } |
| 261 | 295 | } |
| 262 | 296 | }); |
| @@ -264,24 +298,18 @@ impl VectorStore { · 1 unmodified line |
| 264 | 298 | // embed_tx/rx: Embed Batch and Send to Database |
| 265 | 299 | let (embed_batch_tx, embed_batch_rx) = |
| 266 | 300 | channel::unbounded::<Vec<(i64, ParsedFile, Vec<String>)>>(); |
| 267 | | − let mut _embed_batch_task = Vec::new(); |
| 268 | | − for _ in 0..1 { |
| 269 | | − //cx.background().num_cpus() { |
| 301 | + let _embed_batch_task = cx.background().spawn({ |
| 270 | 302 | let db_update_tx = db_update_tx.clone(); |
| 271 | | − let embed_batch_rx = embed_batch_rx.clone(); |
| 272 | 303 | let embedding_provider = embedding_provider.clone(); |
| 273 | | − _embed_batch_task.push(cx.background().spawn(async move { |
| 274 | | − while let Ok(embeddings_queue) = embed_batch_rx.recv().await { |
| 304 | + async move { |
| 305 | + while let Ok(mut embeddings_queue) = embed_batch_rx.recv().await { |
| 275 | 306 | // Construct Batch |
| 276 | | − let mut embeddings_queue = embeddings_queue.clone(); |
| 277 | 307 | let mut document_spans = vec![]; |
| 278 | | − for (_, _, document_span) in embeddings_queue.clone().into_iter() { |
| 279 | | − document_spans.extend(document_span); |
| 308 | + for (_, _, document_span) in embeddings_queue.iter() { |
| 309 | + document_spans.extend(document_span.iter().map(|s| s.as_str())); |
| 280 | 310 | } |
| 281 | 311 | |
| 282 | | − if let Ok(embeddings) = embedding_provider |
| 283 | | − .embed_batch(document_spans.iter().map(|x| &**x).collect()) |
| 284 | | − .await |
| 312 | + if let Ok(embeddings) = embedding_provider.embed_batch(document_spans).await |
| 285 | 313 | { |
| 286 | 314 | let mut i = 0; |
| 287 | 315 | let mut j = 0; |
| @@ -306,7 +334,7 @@ impl VectorStore { · 18 unmodified lines |
| 306 | 334 | } |
| 307 | 335 | |
| 308 | 336 | db_update_tx |
| 309 | | − .send(DbWrite::InsertFile { |
| 337 | + .send(DbOperation::InsertFile { |
| 310 | 338 | worktree_id, |
| 311 | 339 | indexed_file, |
| 312 | 340 | }) |
| @@ -315,8 +343,9 @@ impl VectorStore { · 2 unmodified lines |
| 315 | 343 | } |
| 316 | 344 | } |
| 317 | 345 | } |
| 318 | | − })) |
| 319 | | − } |
| 346 | + } |
| 347 | + }); |
| 348 | + |
| 320 | 349 | // batch_tx/rx: Batch Files to Send for Embeddings |
| 321 | 350 | let (batch_files_tx, batch_files_rx) = channel::unbounded::<EmbeddingJob>(); |
| 322 | 351 | let _batch_files_task = cx.background().spawn(async move { |
| @@ -398,7 +427,21 @@ impl VectorStore { · 75 unmodified lines |
| 398 | 427 | fn find_or_create_worktree(&self, path: PathBuf) -> impl Future<Output = Result<i64>> { |
| 399 | 428 | let (tx, rx) = oneshot::channel(); |
| 400 | 429 | self.db_update_tx |
| 401 | | − .try_send(DbWrite::FindOrCreateWorktree { path, sender: tx }) |
| 430 | + .try_send(DbOperation::FindOrCreateWorktree { path, sender: tx }) |
| 431 | + .unwrap(); |
| 432 | + async move { rx.await? } |
| 433 | + } |
| 434 | + |
| 435 | + fn get_file_mtimes( |
| 436 | + &self, |
| 437 | + worktree_id: i64, |
| 438 | + ) -> impl Future<Output = Result<HashMap<PathBuf, SystemTime>>> { |
| 439 | + let (tx, rx) = oneshot::channel(); |
| 440 | + self.db_update_tx |
| 441 | + .try_send(DbOperation::FileMTimes { |
| 442 | + worktree_id, |
| 443 | + sender: tx, |
| 444 | + }) |
| 402 | 445 | .unwrap(); |
| 403 | 446 | async move { rx.await? } |
| 404 | 447 | } |
| @@ -450,26 +493,17 @@ impl VectorStore { · 45 unmodified lines |
| 450 | 493 | .collect::<Vec<_>>() |
| 451 | 494 | }); |
| 452 | 495 | |
| 453 | | − // Here we query the worktree ids, and yet we dont have them elsewhere |
| 454 | | − // We likely want to clean up these datastructures |
| 455 | | − let (mut worktree_file_times, db_ids_by_worktree_id) = cx |
| 456 | | − .background() |
| 457 | | − .spawn({ |
| 458 | | − let worktrees = worktrees.clone(); |
| 459 | | − async move { |
| 460 | | − let db = VectorDatabase::new(database_url.to_string_lossy().into())?; |
| 461 | | − let mut db_ids_by_worktree_id = HashMap::new(); |
| 462 | | − let mut file_times: HashMap<WorktreeId, HashMap<PathBuf, SystemTime>> = |
| 463 | | − HashMap::new(); |
| 464 | | − for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) { |
| 465 | | − let db_id = db_id?; |
| 466 | | − db_ids_by_worktree_id.insert(worktree.id(), db_id); |
| 467 | | − file_times.insert(worktree.id(), db.get_file_mtimes(db_id)?); |
| 468 | | − } |
| 469 | | − anyhow::Ok((file_times, db_ids_by_worktree_id)) |
| 470 | | − } |
| 471 | | − }) |
| 472 | | − .await?; |
| 496 | + let mut worktree_file_times = HashMap::new(); |
| 497 | + let mut db_ids_by_worktree_id = HashMap::new(); |
| 498 | + for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) { |
| 499 | + let db_id = db_id?; |
| 500 | + db_ids_by_worktree_id.insert(worktree.id(), db_id); |
| 501 | + worktree_file_times.insert( |
| 502 | + worktree.id(), |
| 503 | + this.read_with(&cx, |this, _| this.get_file_mtimes(db_id)) |
| 504 | + .await?, |
| 505 | + ); |
| 506 | + } |
| 473 | 507 | |
| 474 | 508 | cx.background() |
| 475 | 509 | .spawn({ |
| @@ -520,7 +554,7 @@ impl VectorStore { · 44 unmodified lines |
| 520 | 554 | } |
| 521 | 555 | for file in file_mtimes.keys() { |
| 522 | 556 | db_update_tx |
| 523 | | − .try_send(DbWrite::Delete { |
| 557 | + .try_send(DbOperation::Delete { |
| 524 | 558 | worktree_id: db_ids_by_worktree_id[&worktree.id()], |
| 525 | 559 | path: file.to_owned(), |
| 526 | 560 | }) |
| @@ -542,7 +576,7 @@ impl VectorStore { · 15 unmodified lines |
| 542 | 576 | // greater than the previous embedded time by the REINDEXING_DELAY variable, we will send the file off to be indexed. |
| 543 | 577 | let _subscription = cx.subscribe(&project, |this, project, event, cx| { |
| 544 | 578 | if let project::Event::WorktreeUpdatedEntries(worktree_id, changes) = event { |
| 545 | | − this.project_entries_changed(project, changes, cx, worktree_id); |
| 579 | + this.project_entries_changed(project, changes.clone(), cx, worktree_id); |
| 546 | 580 | } |
| 547 | 581 | }); |
| 548 | 582 | |
| @@ -578,16 +612,7 @@ impl VectorStore { · 29 unmodified lines |
| 578 | 612 | .worktrees(cx) |
| 579 | 613 | .filter_map(|worktree| { |
| 580 | 614 | let worktree_id = worktree.read(cx).id(); |
| 581 | | − project_state |
| 582 | | − .worktree_db_ids |
| 583 | | − .iter() |
| 584 | | − .find_map(|(id, db_id)| { |
| 585 | | − if *id == worktree_id { |
| 586 | | − Some(*db_id) |
| 587 | | − } else { |
| 588 | | − None |
| 589 | | − } |
| 590 | | − }) |
| 615 | + project_state.db_id_for_worktree_id(worktree_id) |
| 591 | 616 | }) |
| 592 | 617 | .collect::<Vec<_>>(); |
| 593 | 618 | |
| @@ -606,24 +631,12 @@ impl VectorStore { · 12 unmodified lines |
| 606 | 631 | .next() |
| 607 | 632 | .unwrap(); |
| 608 | 633 | |
| 609 | | − let mut results = Vec::<(i64, f32)>::with_capacity(limit + 1); |
| 610 | | − database.for_each_document(&worktree_db_ids, |id, embedding| { |
| 611 | | − let similarity = dot(&embedding.0, &phrase_embedding); |
| 612 | | − let ix = match results.binary_search_by(|(_, s)| { |
| 613 | | − similarity.partial_cmp(&s).unwrap_or(Ordering::Equal) |
| 614 | | − }) { |
| 615 | | − Ok(ix) => ix, |
| 616 | | − Err(ix) => ix, |
| 617 | | − }; |
| 618 | | − results.insert(ix, (id, similarity)); |
| 619 | | − results.truncate(limit); |
| 620 | | − })?; |
| 621 | | − |
| 622 | | − let ids = results.into_iter().map(|(id, _)| id).collect::<Vec<_>>(); |
| 623 | | − database.get_documents_by_ids(&ids) |
| 634 | + database.top_k_search(&worktree_db_ids, &phrase_embedding, limit) |
| 624 | 635 | }) |
| 625 | 636 | .await?; |
| 626 | 637 | |
| 638 | + dbg!(&documents); |
| 639 | + |
| 627 | 640 | this.read_with(&cx, |this, _| { |
| 628 | 641 | let project_state = if let Some(state) = this.projects.get(&project.downgrade()) { |
| 629 | 642 | state |
| @@ -634,17 +647,7 @@ impl VectorStore { · 4 unmodified lines |
| 634 | 647 | Ok(documents |
| 635 | 648 | .into_iter() |
| 636 | 649 | .filter_map(|(worktree_db_id, file_path, offset, name)| { |
| 637 | | − let worktree_id = |
| 638 | | − project_state |
| 639 | | − .worktree_db_ids |
| 640 | | − .iter() |
| 641 | | − .find_map(|(id, db_id)| { |
| 642 | | − if *db_id == worktree_db_id { |
| 643 | | − Some(*id) |
| 644 | | − } else { |
| 645 | | − None |
| 646 | | − } |
| 647 | | − })?; |
| 650 | + let worktree_id = project_state.worktree_id_for_db_id(worktree_db_id)?; |
| 648 | 651 | Some(SearchResult { |
| 649 | 652 | worktree_id, |
| 650 | 653 | name, |
| @@ -660,51 +663,36 @@ impl VectorStore { · 9 unmodified lines |
| 660 | 663 | fn project_entries_changed( |
| 661 | 664 | &mut self, |
| 662 | 665 | project: ModelHandle<Project>, |
| 663 | | − changes: &[(Arc<Path>, ProjectEntryId, PathChange)], |
| 666 | + changes: Arc<[(Arc<Path>, ProjectEntryId, PathChange)]>, |
| 664 | 667 | cx: &mut ModelContext<'_, VectorStore>, |
| 665 | 668 | worktree_id: &WorktreeId, |
| 666 | 669 | ) -> Option<()> { |
| 667 | | − let project_state = self.projects.get_mut(&project.downgrade())?; |
| 668 | | − let worktree_db_ids = project_state.worktree_db_ids.clone(); |
| 669 | | − let worktree = project.read(cx).worktree_for_id(worktree_id.clone(), cx)?; |
| 670 | | − |
| 671 | | − // Get Database |
| 672 | | − let (file_mtimes, worktree_db_id) = { |
| 673 | | − if let Ok(db) = VectorDatabase::new(self.database_url.to_string_lossy().into()) { |
| 674 | | − let worktree_db_id = { |
| 675 | | − let mut found_db_id = None; |
| 676 | | − for (w_id, db_id) in worktree_db_ids.into_iter() { |
| 677 | | − if &w_id == &worktree.read(cx).id() { |
| 678 | | − found_db_id = Some(db_id) |
| 679 | | − } |
| 680 | | − } |
| 681 | | − found_db_id |
| 682 | | − }?; |
| 683 | | − |
| 684 | | − let file_mtimes = db.get_file_mtimes(worktree_db_id).log_err()?; |
| 670 | + let worktree = project |
| 671 | + .read(cx) |
| 672 | + .worktree_for_id(worktree_id.clone(), cx)? |
| 673 | + .read(cx) |
| 674 | + .snapshot(); |
| 685 | 675 | |
| 686 | | − Some((file_mtimes, worktree_db_id)) |
| 687 | | − } else { |
| 688 | | − return None; |
| 689 | | − } |
| 690 | | − }?; |
| 676 | + let worktree_db_id = self |
| 677 | + .projects |
| 678 | + .get(&project.downgrade())? |
| 679 | + .db_id_for_worktree_id(worktree.id())?; |
| 680 | + let file_mtimes = self.get_file_mtimes(worktree_db_id); |
| 691 | 681 | |
| 692 | | − // Iterate Through Changes |
| 693 | 682 | let language_registry = self.language_registry.clone(); |
| 694 | | − let parsing_files_tx = self.parsing_files_tx.clone(); |
| 695 | 683 | |
| 696 | | − smol::block_on(async move { |
| 684 | + cx.spawn(|this, mut cx| async move { |
| 685 | + let file_mtimes = file_mtimes.await.log_err()?; |
| 686 | + |
| 697 | 687 | for change in changes.into_iter() { |
| 698 | 688 | let change_path = change.0.clone(); |
| 699 | | − let absolute_path = worktree.read(cx).absolutize(&change_path); |
| 689 | + let absolute_path = worktree.absolutize(&change_path); |
| 700 | 690 | // Skip if git ignored or symlink |
| 701 | | − if let Some(entry) = worktree.read(cx).entry_for_id(change.1) { |
| 702 | | − if entry.is_ignored || entry.is_symlink { |
| 691 | + if let Some(entry) = worktree.entry_for_id(change.1) { |
| 692 | + if entry.is_ignored || entry.is_symlink || entry.is_external { |
| 703 | 693 | continue; |
| 704 | | − } else { |
| 705 | | − log::info!("Testing for Reindexing: {:?}", &change_path); |
| 706 | 694 | } |
| 707 | | − }; |
| 695 | + } |
| 708 | 696 | |
| 709 | 697 | if let Ok(language) = language_registry |
| 710 | 698 | .language_for_file(&change_path.to_path_buf(), None) |
| @@ -718,27 +706,18 @@ impl VectorStore { · 7 unmodified lines |
| 718 | 706 | continue; |
| 719 | 707 | } |
| 720 | 708 | |
| 721 | | − if let Some(modified_time) = { |
| 722 | | − let metadata = change_path.metadata(); |
| 723 | | − if metadata.is_err() { |
| 724 | | − None |
| 725 | | − } else { |
| 726 | | − let mtime = metadata.unwrap().modified(); |
| 727 | | − if mtime.is_err() { |
| 728 | | − None |
| 729 | | − } else { |
| 730 | | − Some(mtime.unwrap()) |
| 731 | | − } |
| 732 | | − } |
| 733 | | − } { |
| 734 | | − let existing_time = file_mtimes.get(&change_path.to_path_buf()); |
| 735 | | − let already_stored = existing_time |
| 736 | | − .map_or(false, |existing_time| &modified_time != existing_time); |
| 709 | + let modified_time = change_path.metadata().log_err()?.modified().log_err()?; |
| 737 | 710 | |
| 738 | | − let reindex_time = |
| 739 | | − modified_time + Duration::from_secs(REINDEXING_DELAY_SECONDS); |
| 711 | + let existing_time = file_mtimes.get(&change_path.to_path_buf()); |
| 712 | + let already_stored = existing_time |
| 713 | + .map_or(false, |existing_time| &modified_time != existing_time); |
| 740 | 714 | |
| 741 | | − if !already_stored { |
| 715 | + if !already_stored { |
| 716 | + this.update(&mut cx, |this, _| { |
| 717 | + let reindex_time = |
| 718 | + modified_time + Duration::from_secs(REINDEXING_DELAY_SECONDS); |
| 719 | + |
| 720 | + let project_state = this.projects.get_mut(&project.downgrade())?; |
| 742 | 721 | project_state.update_pending_files( |
| 743 | 722 | PendingFile { |
| 744 | 723 | relative_path: change_path.to_path_buf(), |
| @@ -751,13 +730,18 @@ impl VectorStore { · 6 unmodified lines |
| 751 | 730 | ); |
| 752 | 731 | |
| 753 | 732 | for file in project_state.get_outstanding_files() { |
| 754 | | − parsing_files_tx.try_send(file).unwrap(); |
| 733 | + this.parsing_files_tx.try_send(file).unwrap(); |
| 755 | 734 | } |
| 756 | | − } |
| 735 | + Some(()) |
| 736 | + }); |
| 757 | 737 | } |
| 758 | 738 | } |
| 759 | 739 | } |
| 760 | | − }); |
| 740 | + |
| 741 | + Some(()) |
| 742 | + }) |
| 743 | + .detach(); |
| 744 | + |
| 761 | 745 | Some(()) |
| 762 | 746 | } |
| 763 | 747 | } |
| @@ -765,29 +749,3 @@ impl VectorStore { · 1 unmodified line |
| 765 | 749 | impl Entity for VectorStore { |
| 766 | 750 | type Event = (); |
| 767 | 751 | } |
| 768 | | − |
| 769 | | −fn dot(vec_a: &[f32], vec_b: &[f32]) -> f32 { |
| 770 | | − let len = vec_a.len(); |
| 771 | | − assert_eq!(len, vec_b.len()); |
| 772 | | − |
| 773 | | − let mut result = 0.0; |
| 774 | | − unsafe { |
| 775 | | − matrixmultiply::sgemm( |
| 776 | | − 1, |
| 777 | | − len, |
| 778 | | − 1, |
| 779 | | − 1.0, |
| 780 | | − vec_a.as_ptr(), |
| 781 | | − len as isize, |
| 782 | | − 1, |
| 783 | | − vec_b.as_ptr(), |
| 784 | | − 1, |
| 785 | | − len as isize, |
| 786 | | − 0.0, |
| 787 | | − &mut result as *mut f32, |
| 788 | | − 1, |
| 789 | | − 1, |
| 790 | | − ); |
| 791 | | − } |
| 792 | | − result |
| 793 | | −} |