History

Reduced redundant database connections on each worktree change.

KCaverly committed 3 years ago

Commit 307d8d91 parent 82079dd2 files · +182 −178Browse files

Unified · Split

2 files
  1. crates/vector_store/src/db.rs +62 −16
  2. crates/vector_store/src/vector_store.rs +120 −162
crates/vector_store/src/db.rs+62−16
@@ -1,4 +1,5 @@
11 use std::{
2+ cmp::Ordering,
23 collections::HashMap,
34 path::{Path, PathBuf},
45 rc::Rc,
@@ -14,16 +15,6 @@ use rusqlite::{ · 9 unmodified lines
1415 types::{FromSql, FromSqlResult, ValueRef},
1516 };
1617
17−// Note this is not an appropriate document
18−#[derive(Debug)]
19−pub struct DocumentRecord {
20− pub id: usize,
21− pub file_id: usize,
22− pub offset: usize,
23− pub name: String,
24− pub embedding: Embedding,
25−}
26−
2718 #[derive(Debug)]
2819 pub struct FileRecord {
2920 pub id: usize,
@@ -32,7 +23,7 @@ pub struct FileRecord { · 2 unmodified lines
3223 }
3324
3425 #[derive(Debug)]
35−pub struct Embedding(pub Vec<f32>);
26+struct Embedding(pub Vec<f32>);
3627
3728 impl FromSql for Embedding {
3829 fn column_result(value: ValueRef) -> FromSqlResult<Self> {
@@ -205,10 +196,35 @@ impl VectorDatabase { · 166 unmodified lines
205196 Ok(result)
206197 }
207198
208− pub fn for_each_document(
199+ pub fn top_k_search(
200+ &self,
201+ worktree_ids: &[i64],
202+ query_embedding: &Vec<f32>,
203+ limit: usize,
204+ ) -> Result<Vec<(i64, PathBuf, usize, String)>> {
205+ let mut results = Vec::<(i64, f32)>::with_capacity(limit + 1);
206+ self.for_each_document(&worktree_ids, |id, embedding| {
207+ eprintln!("document {id} {embedding:?}");
208+
209+ let similarity = dot(&embedding, &query_embedding);
210+ let ix = match results
211+ .binary_search_by(|(_, s)| similarity.partial_cmp(&s).unwrap_or(Ordering::Equal))
212+ {
213+ Ok(ix) => ix,
214+ Err(ix) => ix,
215+ };
216+ results.insert(ix, (id, similarity));
217+ results.truncate(limit);
218+ })?;
219+
220+ let ids = results.into_iter().map(|(id, _)| id).collect::<Vec<_>>();
221+ self.get_documents_by_ids(&ids)
222+ }
223+
224+ fn for_each_document(
209225 &self,
210226 worktree_ids: &[i64],
211− mut f: impl FnMut(i64, Embedding),
227+ mut f: impl FnMut(i64, Vec<f32>),
212228 ) -> Result<()> {
213229 let mut query_statement = self.db.prepare(
214230 "
@@ -221,16 +237,20 @@ impl VectorDatabase { · 6 unmodified lines
221237 files.worktree_id IN rarray(?)
222238 ",
223239 )?;
240+
224241 query_statement
225242 .query_map(params![ids_to_sql(worktree_ids)], |row| {
226− Ok((row.get(0)?, row.get(1)?))
243+ Ok((row.get(0)?, row.get::<_, Embedding>(1)?))
227244 })?
228245 .filter_map(|row| row.ok())
229− .for_each(|row| f(row.0, row.1));
246+ .for_each(|(id, embedding)| {
247+ dbg!("id");
248+ f(id, embedding.0)
249+ });
230250 Ok(())
231251 }
232252
233− pub fn get_documents_by_ids(&self, ids: &[i64]) -> Result<Vec<(i64, PathBuf, usize, String)>> {
253+ fn get_documents_by_ids(&self, ids: &[i64]) -> Result<Vec<(i64, PathBuf, usize, String)>> {
234254 let mut statement = self.db.prepare(
235255 "
236256 SELECT
@@ -279,3 +299,29 @@ fn ids_to_sql(ids: &[i64]) -> Rc<Vec<rusqlite::types::Value>> { · 42 unmodified lines
279299 .collect::<Vec<_>>(),
280300 )
281301 }
302+
303+pub(crate) fn dot(vec_a: &[f32], vec_b: &[f32]) -> f32 {
304+ let len = vec_a.len();
305+ assert_eq!(len, vec_b.len());
306+
307+ let mut result = 0.0;
308+ unsafe {
309+ matrixmultiply::sgemm(
310+ 1,
311+ len,
312+ 1,
313+ 1.0,
314+ vec_a.as_ptr(),
315+ len as isize,
316+ 1,
317+ vec_b.as_ptr(),
318+ 1,
319+ len as isize,
320+ 0.0,
321+ &mut result as *mut f32,
322+ 1,
323+ 1,
324+ );
325+ }
326+ result
327+}
crates/vector_store/src/vector_store.rs+120−162
@@ -20,7 +20,6 @@ use parsing::{CodeContextRetriever, ParsedFile}; · 19 unmodified lines
2020 use project::{Fs, PathChange, Project, ProjectEntryId, WorktreeId};
2121 use smol::channel;
2222 use std::{
23− cmp::Ordering,
2423 collections::HashMap,
2524 path::{Path, PathBuf},
2625 sync::Arc,
@@ -112,10 +111,10 @@ pub struct VectorStore { · 85 unmodified lines
112111 database_url: Arc<PathBuf>,
113112 embedding_provider: Arc<dyn EmbeddingProvider>,
114113 language_registry: Arc<LanguageRegistry>,
115− db_update_tx: channel::Sender<DbWrite>,
114+ db_update_tx: channel::Sender<DbOperation>,
116115 parsing_files_tx: channel::Sender<PendingFile>,
117116 _db_update_task: Task<()>,
118− _embed_batch_task: Vec<Task<()>>,
117+ _embed_batch_task: Task<()>,
119118 _batch_files_task: Task<()>,
120119 _parsing_files_tasks: Vec<Task<()>>,
121120 projects: HashMap<WeakModelHandle<Project>, ProjectState>,
@@ -128,6 +127,30 @@ struct ProjectState { · 6 unmodified lines
128127 }
129128
130129 impl ProjectState {
130+ fn db_id_for_worktree_id(&self, id: WorktreeId) -> Option<i64> {
131+ self.worktree_db_ids
132+ .iter()
133+ .find_map(|(worktree_id, db_id)| {
134+ if *worktree_id == id {
135+ Some(*db_id)
136+ } else {
137+ None
138+ }
139+ })
140+ }
141+
142+ fn worktree_id_for_db_id(&self, id: i64) -> Option<WorktreeId> {
143+ self.worktree_db_ids
144+ .iter()
145+ .find_map(|(worktree_id, db_id)| {
146+ if *db_id == id {
147+ Some(*worktree_id)
148+ } else {
149+ None
150+ }
151+ })
152+ }
153+
131154 fn update_pending_files(&mut self, pending_file: PendingFile, indexing_time: SystemTime) {
132155 // If Pending File Already Exists, Replace it with the new one
133156 // but keep the old indexing time
@@ -185,7 +208,7 @@ pub struct SearchResult { · 51 unmodified lines
185208 pub file_path: PathBuf,
186209 }
187210
188−enum DbWrite {
211+enum DbOperation {
189212 InsertFile {
190213 worktree_id: i64,
191214 indexed_file: ParsedFile,
@@ -198,6 +221,10 @@ enum DbWrite { · 6 unmodified lines
198221 path: PathBuf,
199222 sender: oneshot::Sender<Result<i64>>,
200223 },
224+ FileMTimes {
225+ worktree_id: i64,
226+ sender: oneshot::Sender<Result<HashMap<PathBuf, SystemTime>>>,
227+ },
201228 }
202229
203230 enum EmbeddingJob {
@@ -243,20 +270,27 @@ impl VectorStore { · 39 unmodified lines
243270 let _db_update_task = cx.background().spawn(async move {
244271 while let Ok(job) = db_update_rx.recv().await {
245272 match job {
246− DbWrite::InsertFile {
273+ DbOperation::InsertFile {
247274 worktree_id,
248275 indexed_file,
249276 } => {
250277 log::info!("Inserting Data for {:?}", &indexed_file.path);
251278 db.insert_file(worktree_id, indexed_file).log_err();
252279 }
253− DbWrite::Delete { worktree_id, path } => {
280+ DbOperation::Delete { worktree_id, path } => {
254281 db.delete_file(worktree_id, path).log_err();
255282 }
256− DbWrite::FindOrCreateWorktree { path, sender } => {
283+ DbOperation::FindOrCreateWorktree { path, sender } => {
257284 let id = db.find_or_create_worktree(&path);
258285 sender.send(id).ok();
259286 }
287+ DbOperation::FileMTimes {
288+ worktree_id: worktree_db_id,
289+ sender,
290+ } => {
291+ let file_mtimes = db.get_file_mtimes(worktree_db_id);
292+ sender.send(file_mtimes).ok();
293+ }
260294 }
261295 }
262296 });
@@ -264,24 +298,18 @@ impl VectorStore { · 1 unmodified line
264298 // embed_tx/rx: Embed Batch and Send to Database
265299 let (embed_batch_tx, embed_batch_rx) =
266300 channel::unbounded::<Vec<(i64, ParsedFile, Vec<String>)>>();
267− let mut _embed_batch_task = Vec::new();
268− for _ in 0..1 {
269− //cx.background().num_cpus() {
301+ let _embed_batch_task = cx.background().spawn({
270302 let db_update_tx = db_update_tx.clone();
271− let embed_batch_rx = embed_batch_rx.clone();
272303 let embedding_provider = embedding_provider.clone();
273− _embed_batch_task.push(cx.background().spawn(async move {
274− while let Ok(embeddings_queue) = embed_batch_rx.recv().await {
304+ async move {
305+ while let Ok(mut embeddings_queue) = embed_batch_rx.recv().await {
275306 // Construct Batch
276− let mut embeddings_queue = embeddings_queue.clone();
277307 let mut document_spans = vec![];
278− for (_, _, document_span) in embeddings_queue.clone().into_iter() {
279− document_spans.extend(document_span);
308+ for (_, _, document_span) in embeddings_queue.iter() {
309+ document_spans.extend(document_span.iter().map(|s| s.as_str()));
280310 }
281311
282− if let Ok(embeddings) = embedding_provider
283− .embed_batch(document_spans.iter().map(|x| &**x).collect())
284− .await
312+ if let Ok(embeddings) = embedding_provider.embed_batch(document_spans).await
285313 {
286314 let mut i = 0;
287315 let mut j = 0;
@@ -306,7 +334,7 @@ impl VectorStore { · 18 unmodified lines
306334 }
307335
308336 db_update_tx
309− .send(DbWrite::InsertFile {
337+ .send(DbOperation::InsertFile {
310338 worktree_id,
311339 indexed_file,
312340 })
@@ -315,8 +343,9 @@ impl VectorStore { · 2 unmodified lines
315343 }
316344 }
317345 }
318− }))
319− }
346+ }
347+ });
348+
320349 // batch_tx/rx: Batch Files to Send for Embeddings
321350 let (batch_files_tx, batch_files_rx) = channel::unbounded::<EmbeddingJob>();
322351 let _batch_files_task = cx.background().spawn(async move {
@@ -398,7 +427,21 @@ impl VectorStore { · 75 unmodified lines
398427 fn find_or_create_worktree(&self, path: PathBuf) -> impl Future<Output = Result<i64>> {
399428 let (tx, rx) = oneshot::channel();
400429 self.db_update_tx
401− .try_send(DbWrite::FindOrCreateWorktree { path, sender: tx })
430+ .try_send(DbOperation::FindOrCreateWorktree { path, sender: tx })
431+ .unwrap();
432+ async move { rx.await? }
433+ }
434+
435+ fn get_file_mtimes(
436+ &self,
437+ worktree_id: i64,
438+ ) -> impl Future<Output = Result<HashMap<PathBuf, SystemTime>>> {
439+ let (tx, rx) = oneshot::channel();
440+ self.db_update_tx
441+ .try_send(DbOperation::FileMTimes {
442+ worktree_id,
443+ sender: tx,
444+ })
402445 .unwrap();
403446 async move { rx.await? }
404447 }
@@ -450,26 +493,17 @@ impl VectorStore { · 45 unmodified lines
450493 .collect::<Vec<_>>()
451494 });
452495
453− // Here we query the worktree ids, and yet we dont have them elsewhere
454− // We likely want to clean up these datastructures
455− let (mut worktree_file_times, db_ids_by_worktree_id) = cx
456− .background()
457− .spawn({
458− let worktrees = worktrees.clone();
459− async move {
460− let db = VectorDatabase::new(database_url.to_string_lossy().into())?;
461− let mut db_ids_by_worktree_id = HashMap::new();
462− let mut file_times: HashMap<WorktreeId, HashMap<PathBuf, SystemTime>> =
463− HashMap::new();
464− for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) {
465− let db_id = db_id?;
466− db_ids_by_worktree_id.insert(worktree.id(), db_id);
467− file_times.insert(worktree.id(), db.get_file_mtimes(db_id)?);
468− }
469− anyhow::Ok((file_times, db_ids_by_worktree_id))
470− }
471− })
472− .await?;
496+ let mut worktree_file_times = HashMap::new();
497+ let mut db_ids_by_worktree_id = HashMap::new();
498+ for (worktree, db_id) in worktrees.iter().zip(worktree_db_ids) {
499+ let db_id = db_id?;
500+ db_ids_by_worktree_id.insert(worktree.id(), db_id);
501+ worktree_file_times.insert(
502+ worktree.id(),
503+ this.read_with(&cx, |this, _| this.get_file_mtimes(db_id))
504+ .await?,
505+ );
506+ }
473507
474508 cx.background()
475509 .spawn({
@@ -520,7 +554,7 @@ impl VectorStore { · 44 unmodified lines
520554 }
521555 for file in file_mtimes.keys() {
522556 db_update_tx
523− .try_send(DbWrite::Delete {
557+ .try_send(DbOperation::Delete {
524558 worktree_id: db_ids_by_worktree_id[&worktree.id()],
525559 path: file.to_owned(),
526560 })
@@ -542,7 +576,7 @@ impl VectorStore { · 15 unmodified lines
542576 // greater than the previous embedded time by the REINDEXING_DELAY variable, we will send the file off to be indexed.
543577 let _subscription = cx.subscribe(&project, |this, project, event, cx| {
544578 if let project::Event::WorktreeUpdatedEntries(worktree_id, changes) = event {
545− this.project_entries_changed(project, changes, cx, worktree_id);
579+ this.project_entries_changed(project, changes.clone(), cx, worktree_id);
546580 }
547581 });
548582
@@ -578,16 +612,7 @@ impl VectorStore { · 29 unmodified lines
578612 .worktrees(cx)
579613 .filter_map(|worktree| {
580614 let worktree_id = worktree.read(cx).id();
581− project_state
582− .worktree_db_ids
583− .iter()
584− .find_map(|(id, db_id)| {
585− if *id == worktree_id {
586− Some(*db_id)
587− } else {
588− None
589− }
590− })
615+ project_state.db_id_for_worktree_id(worktree_id)
591616 })
592617 .collect::<Vec<_>>();
593618
@@ -606,24 +631,12 @@ impl VectorStore { · 12 unmodified lines
606631 .next()
607632 .unwrap();
608633
609− let mut results = Vec::<(i64, f32)>::with_capacity(limit + 1);
610− database.for_each_document(&worktree_db_ids, |id, embedding| {
611− let similarity = dot(&embedding.0, &phrase_embedding);
612− let ix = match results.binary_search_by(|(_, s)| {
613− similarity.partial_cmp(&s).unwrap_or(Ordering::Equal)
614− }) {
615− Ok(ix) => ix,
616− Err(ix) => ix,
617− };
618− results.insert(ix, (id, similarity));
619− results.truncate(limit);
620− })?;
621−
622− let ids = results.into_iter().map(|(id, _)| id).collect::<Vec<_>>();
623− database.get_documents_by_ids(&ids)
634+ database.top_k_search(&worktree_db_ids, &phrase_embedding, limit)
624635 })
625636 .await?;
626637
638+ dbg!(&documents);
639+
627640 this.read_with(&cx, |this, _| {
628641 let project_state = if let Some(state) = this.projects.get(&project.downgrade()) {
629642 state
@@ -634,17 +647,7 @@ impl VectorStore { · 4 unmodified lines
634647 Ok(documents
635648 .into_iter()
636649 .filter_map(|(worktree_db_id, file_path, offset, name)| {
637− let worktree_id =
638− project_state
639− .worktree_db_ids
640− .iter()
641− .find_map(|(id, db_id)| {
642− if *db_id == worktree_db_id {
643− Some(*id)
644− } else {
645− None
646− }
647− })?;
650+ let worktree_id = project_state.worktree_id_for_db_id(worktree_db_id)?;
648651 Some(SearchResult {
649652 worktree_id,
650653 name,
@@ -660,51 +663,36 @@ impl VectorStore { · 9 unmodified lines
660663 fn project_entries_changed(
661664 &mut self,
662665 project: ModelHandle<Project>,
663− changes: &[(Arc<Path>, ProjectEntryId, PathChange)],
666+ changes: Arc<[(Arc<Path>, ProjectEntryId, PathChange)]>,
664667 cx: &mut ModelContext<'_, VectorStore>,
665668 worktree_id: &WorktreeId,
666669 ) -> Option<()> {
667− let project_state = self.projects.get_mut(&project.downgrade())?;
668− let worktree_db_ids = project_state.worktree_db_ids.clone();
669− let worktree = project.read(cx).worktree_for_id(worktree_id.clone(), cx)?;
670−
671− // Get Database
672− let (file_mtimes, worktree_db_id) = {
673− if let Ok(db) = VectorDatabase::new(self.database_url.to_string_lossy().into()) {
674− let worktree_db_id = {
675− let mut found_db_id = None;
676− for (w_id, db_id) in worktree_db_ids.into_iter() {
677− if &w_id == &worktree.read(cx).id() {
678− found_db_id = Some(db_id)
679− }
680− }
681− found_db_id
682− }?;
683−
684− let file_mtimes = db.get_file_mtimes(worktree_db_id).log_err()?;
670+ let worktree = project
671+ .read(cx)
672+ .worktree_for_id(worktree_id.clone(), cx)?
673+ .read(cx)
674+ .snapshot();
685675
686− Some((file_mtimes, worktree_db_id))
687− } else {
688− return None;
689− }
690− }?;
676+ let worktree_db_id = self
677+ .projects
678+ .get(&project.downgrade())?
679+ .db_id_for_worktree_id(worktree.id())?;
680+ let file_mtimes = self.get_file_mtimes(worktree_db_id);
691681
692− // Iterate Through Changes
693682 let language_registry = self.language_registry.clone();
694− let parsing_files_tx = self.parsing_files_tx.clone();
695683
696− smol::block_on(async move {
684+ cx.spawn(|this, mut cx| async move {
685+ let file_mtimes = file_mtimes.await.log_err()?;
686+
697687 for change in changes.into_iter() {
698688 let change_path = change.0.clone();
699− let absolute_path = worktree.read(cx).absolutize(&change_path);
689+ let absolute_path = worktree.absolutize(&change_path);
700690 // Skip if git ignored or symlink
701− if let Some(entry) = worktree.read(cx).entry_for_id(change.1) {
702− if entry.is_ignored || entry.is_symlink {
691+ if let Some(entry) = worktree.entry_for_id(change.1) {
692+ if entry.is_ignored || entry.is_symlink || entry.is_external {
703693 continue;
704− } else {
705− log::info!("Testing for Reindexing: {:?}", &change_path);
706694 }
707− };
695+ }
708696
709697 if let Ok(language) = language_registry
710698 .language_for_file(&change_path.to_path_buf(), None)
@@ -718,27 +706,18 @@ impl VectorStore { · 7 unmodified lines
718706 continue;
719707 }
720708
721− if let Some(modified_time) = {
722− let metadata = change_path.metadata();
723− if metadata.is_err() {
724− None
725− } else {
726− let mtime = metadata.unwrap().modified();
727− if mtime.is_err() {
728− None
729− } else {
730− Some(mtime.unwrap())
731− }
732− }
733− } {
734− let existing_time = file_mtimes.get(&change_path.to_path_buf());
735− let already_stored = existing_time
736− .map_or(false, |existing_time| &modified_time != existing_time);
709+ let modified_time = change_path.metadata().log_err()?.modified().log_err()?;
737710
738− let reindex_time =
739− modified_time + Duration::from_secs(REINDEXING_DELAY_SECONDS);
711+ let existing_time = file_mtimes.get(&change_path.to_path_buf());
712+ let already_stored = existing_time
713+ .map_or(false, |existing_time| &modified_time != existing_time);
740714
741− if !already_stored {
715+ if !already_stored {
716+ this.update(&mut cx, |this, _| {
717+ let reindex_time =
718+ modified_time + Duration::from_secs(REINDEXING_DELAY_SECONDS);
719+
720+ let project_state = this.projects.get_mut(&project.downgrade())?;
742721 project_state.update_pending_files(
743722 PendingFile {
744723 relative_path: change_path.to_path_buf(),
@@ -751,13 +730,18 @@ impl VectorStore { · 6 unmodified lines
751730 );
752731
753732 for file in project_state.get_outstanding_files() {
754− parsing_files_tx.try_send(file).unwrap();
733+ this.parsing_files_tx.try_send(file).unwrap();
755734 }
756− }
735+ Some(())
736+ });
757737 }
758738 }
759739 }
760− });
740+
741+ Some(())
742+ })
743+ .detach();
744+
761745 Some(())
762746 }
763747 }
@@ -765,29 +749,3 @@ impl VectorStore { · 1 unmodified line
765749 impl Entity for VectorStore {
766750 type Event = ();
767751 }
768−
769−fn dot(vec_a: &[f32], vec_b: &[f32]) -> f32 {
770− let len = vec_a.len();
771− assert_eq!(len, vec_b.len());
772−
773− let mut result = 0.0;
774− unsafe {
775− matrixmultiply::sgemm(
776− 1,
777− len,
778− 1,
779− 1.0,
780− vec_a.as_ptr(),
781− len as isize,
782− 1,
783− vec_b.as_ptr(),
784− 1,
785− len as isize,
786− 0.0,
787− &mut result as *mut f32,
788− 1,
789− 1,
790− );
791− }
792− result
793−}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t