History

index: add function to build changed-path index at certain operation

Yuya Nishihara committed 1 year ago

Commit 8999e081 parent db0e4363 files · +215 −8Browse files

Unified · Split

3 files
  1. lib/src/default_index/changed_path.rs +16 −0
  2. lib/src/default_index/store.rs +77 −7
  3. lib/tests/test_index.rs +122 −1
lib/src/default_index/changed_path.rs+16−0
@@ -452,6 +452,22 @@ impl CompositeChangedPathIndex { · 451 unmodified lines
452452 &self.readonly_segments
453453 }
454454
455+ /// Appends segments from the `other` index. This and the other index should
456+ /// be contiguous.
457+ pub(super) fn append_segments(&mut self, other: &Self) {
458+ assert!(self.mutable_segment.is_none());
459+ let GlobalCommitPosition(self_start_pos) =
460+ self.start_commit_pos.expect("should have start pos");
461+ let Some(GlobalCommitPosition(other_start_pos)) = other.start_commit_pos else {
462+ return;
463+ };
464+ assert_eq!(self_start_pos + self.num_commits, other_start_pos);
465+ self.readonly_segments
466+ .extend_from_slice(&other.readonly_segments);
467+ self.mutable_segment = other.mutable_segment.clone();
468+ self.num_commits += other.num_commits;
469+ }
470+
455471 /// Maps `global_pos` to segment and segment-local position.
456472 fn find_segment(
457473 &self,
lib/src/default_index/store.rs+77−7
@@ -34,6 +34,8 @@ use thiserror::Error; · 33 unmodified lines
3434
3535 use super::changed_path::ChangedPathIndexSegmentId;
3636 use super::changed_path::CompositeChangedPathIndex;
37+use super::changed_path::collect_changed_paths;
38+use super::composite::AsCompositeIndex as _;
3739 use super::composite::CommitIndexSegmentId;
3840 use super::entry::GlobalCommitPosition;
3941 use super::mutable::DefaultMutableIndex;
@@ -390,11 +392,17 @@ impl DefaultIndexStore { · 350 unmodified lines
390392 Ok(index)
391393 }
392394
393− // TODO: for testing; replace with "build" function
394− pub fn enable_changed_path_index_at_operation(
395+ /// Builds changed-path index for the specified operation.
396+ ///
397+ /// At most `max_commits` number of commits will be scanned from the latest
398+ /// unindexed commit.
399+ #[tracing::instrument(skip(self, store))]
400+ pub async fn build_changed_path_index_at_operation(
395401 &self,
396402 op_id: &OperationId,
397403 store: &Arc<Store>,
404+ max_commits: u32,
405+ // TODO: add progress callback?
398406 ) -> Result<DefaultReadonlyIndex, DefaultIndexStoreError> {
399407 // Create directories in case the store was initialized by jj < 0.33.
400408 self.ensure_base_dirs()
@@ -404,13 +412,75 @@ impl DefaultIndexStore { · 3 unmodified lines
404412 change_id: store.change_id_length(),
405413 };
406414 let index = self.load_index_at_operation(op_id, field_lengths)?;
407− if index.changed_paths().start_commit_pos().is_some() {
408− return Ok(index);
415+ let old_changed_paths = index.changed_paths();
416+
417+ // Distribute max_commits to contiguous pre/post ranges:
418+ // ..|pre|old_changed_paths|post|
419+ // (where pre.len() + post.len() <= max_commits)
420+ let pre_start;
421+ let pre_end;
422+ let post_start;
423+ let post_end;
424+ if let Some(GlobalCommitPosition(pos)) = old_changed_paths.start_commit_pos() {
425+ post_start = pos + old_changed_paths.num_commits();
426+ assert!(post_start <= index.num_commits());
427+ post_end = u32::saturating_add(post_start, max_commits).min(index.num_commits());
428+ pre_start = u32::saturating_sub(pos, max_commits - (post_end - post_start));
429+ pre_end = pos;
430+ } else {
431+ pre_start = u32::saturating_sub(index.num_commits(), max_commits);
432+ pre_end = index.num_commits();
433+ post_start = pre_end;
434+ post_end = pre_end;
409435 }
410− let changed_paths =
411− CompositeChangedPathIndex::empty(GlobalCommitPosition(index.num_commits()));
436+
437+ let to_index_err = |source| DefaultIndexStoreError::IndexCommits {
438+ op_id: op_id.clone(),
439+ source,
440+ };
441+ let index_commit = async |changed_paths: &mut CompositeChangedPathIndex,
442+ pos: GlobalCommitPosition| {
443+ assert_eq!(changed_paths.next_mutable_commit_pos(), Some(pos));
444+ let commit_id = index.as_composite().commits().entry_by_pos(pos).commit_id();
445+ let commit = store.get_commit_async(&commit_id).await?;
446+ let paths = collect_changed_paths(&index, &commit).await?;
447+ changed_paths.add_changed_paths(paths);
448+ Ok(())
449+ };
450+
451+ // Index pre range
452+ let mut new_changed_paths =
453+ CompositeChangedPathIndex::empty(GlobalCommitPosition(pre_start));
454+ new_changed_paths.make_mutable();
455+ tracing::info!(?pre_start, ?pre_end, "indexing changed paths in commits");
456+ for pos in (pre_start..pre_end).map(GlobalCommitPosition) {
457+ index_commit(&mut new_changed_paths, pos)
458+ .await
459+ .map_err(to_index_err)?;
460+ }
461+ new_changed_paths
462+ .save_in(&self.changed_path_segments_dir())
463+ .map_err(DefaultIndexStoreError::SaveIndex)?;
464+
465+ // Copy previously-indexed segments
466+ new_changed_paths.append_segments(old_changed_paths);
467+
468+ // Index post range, which is usually empty
469+ new_changed_paths.make_mutable();
470+ tracing::info!(?post_start, ?post_end, "indexing changed paths in commits");
471+ for pos in (post_start..post_end).map(GlobalCommitPosition) {
472+ index_commit(&mut new_changed_paths, pos)
473+ .await
474+ .map_err(to_index_err)?;
475+ }
476+ new_changed_paths.maybe_squash_with_ancestors();
477+ new_changed_paths
478+ .save_in(&self.changed_path_segments_dir())
479+ .map_err(DefaultIndexStoreError::SaveIndex)?;
480+
481+ // Update the operation link to point to the new segments
412482 let commits = index.readonly_commits().clone();
413− let index = DefaultReadonlyIndex::from_segment(commits, changed_paths);
483+ let index = DefaultReadonlyIndex::from_segment(commits, new_changed_paths);
414484 self.associate_index_with_operation(&index, op_id)
415485 .map_err(|source| DefaultIndexStoreError::AssociateIndex {
416486 op_id: op_id.to_owned(),
lib/tests/test_index.rs+122−1
@@ -65,7 +65,8 @@ fn enable_changed_path_index(repo: &ReadonlyRepo) -> Arc<ReadonlyRepo> { · 64 unmodified lines
6565 let default_index_store: &DefaultIndexStore =
6666 repo.index_store().as_any().downcast_ref().unwrap();
6767 default_index_store
68− .enable_changed_path_index_at_operation(repo.op_id(), repo.store())
68+ .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 0)
69+ .block_on()
6970 .unwrap();
7071 repo.reload_at(repo.operation()).unwrap()
7172 }
@@ -850,6 +851,126 @@ fn test_changed_path_segments() { · 778 unmodified lines
850851 assert_eq!(stats.changed_path_levels[0].num_paths, 2);
851852 }
852853
854+#[test]
855+fn test_build_changed_path_segments() {
856+ let test_repo = TestRepo::init();
857+ let repo = test_repo.repo;
858+ let root_commit_id = repo.store().root_commit_id();
859+ let default_index_store: &DefaultIndexStore =
860+ repo.index_store().as_any().downcast_ref().unwrap();
861+
862+ let mut tx = repo.start_transaction();
863+ for i in 1..10 {
864+ let tree = create_tree(&repo, &[(repo_path(&i.to_string()), "")]);
865+ tx.repo_mut()
866+ .new_commit(vec![root_commit_id.clone()], tree.id())
867+ .write()
868+ .unwrap();
869+ }
870+ let repo = tx.commit("test").unwrap();
871+
872+ // Index the last 4 commits
873+ default_index_store
874+ .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 4)
875+ .block_on()
876+ .unwrap();
877+ let repo = repo.reload_at(repo.operation()).unwrap();
878+ let stats = as_readonly_index(&repo).stats();
879+ assert_eq!(stats.changed_path_commits_range, Some(6..10));
880+ assert_eq!(stats.changed_path_levels.len(), 1);
881+ assert_eq!(stats.changed_path_levels[0].num_commits, 4);
882+ assert_eq!(stats.changed_path_levels[0].num_changed_paths, 4);
883+ assert_eq!(stats.changed_path_levels[0].num_paths, 4);
884+
885+ // Index remainders
886+ default_index_store
887+ .build_changed_path_index_at_operation(repo.op_id(), repo.store(), u32::MAX)
888+ .block_on()
889+ .unwrap();
890+ let repo = repo.reload_at(repo.operation()).unwrap();
891+ let stats = as_readonly_index(&repo).stats();
892+ assert_eq!(stats.changed_path_commits_range, Some(0..10));
893+ assert_eq!(stats.changed_path_levels.len(), 2);
894+ assert_eq!(stats.changed_path_levels[0].num_commits, 6);
895+ assert_eq!(stats.changed_path_levels[0].num_changed_paths, 5);
896+ assert_eq!(stats.changed_path_levels[0].num_paths, 5);
897+ assert_eq!(stats.changed_path_levels[1].num_commits, 4);
898+ assert_eq!(stats.changed_path_levels[1].num_changed_paths, 4);
899+ assert_eq!(stats.changed_path_levels[1].num_paths, 4);
900+}
901+
902+#[test]
903+fn test_build_changed_path_segments_partially_enabled() {
904+ let test_repo = TestRepo::init();
905+ let repo = test_repo.repo;
906+ let root_commit_id = repo.store().root_commit_id();
907+ let default_index_store: &DefaultIndexStore =
908+ repo.index_store().as_any().downcast_ref().unwrap();
909+
910+ // Partially enable index by merging two operations
911+ let tx1 = {
912+ let mut tx = repo.start_transaction();
913+ for i in 1..5 {
914+ let tree = create_tree(&repo, &[(repo_path(&i.to_string()), "")]);
915+ tx.repo_mut()
916+ .new_commit(vec![root_commit_id.clone()], tree.id())
917+ .write()
918+ .unwrap();
919+ }
920+ let repo = tx.commit("test").unwrap();
921+ let repo = enable_changed_path_index(&repo);
922+ let mut tx = repo.start_transaction();
923+ let tree = create_tree(&repo, &[(repo_path("5"), "")]);
924+ tx.repo_mut()
925+ .new_commit(vec![root_commit_id.clone()], tree.id())
926+ .write()
927+ .unwrap();
928+ tx
929+ };
930+ let mut tx2 = repo.start_transaction();
931+ for i in 6..10 {
932+ let tree = create_tree(&repo, &[(repo_path(&i.to_string()), "")]);
933+ tx2.repo_mut()
934+ .new_commit(vec![root_commit_id.clone()], tree.id())
935+ .write()
936+ .unwrap();
937+ }
938+ let repo = commit_transactions(vec![tx1, tx2]);
939+ let stats = as_readonly_index(&repo).stats();
940+ assert_eq!(stats.num_commits, 10);
941+ assert_eq!(stats.changed_path_commits_range, Some(5..6));
942+ assert_eq!(stats.changed_path_levels.len(), 1);
943+ assert_eq!(stats.changed_path_levels[0].num_commits, 1);
944+ assert_eq!(stats.changed_path_levels[0].num_changed_paths, 1);
945+ assert_eq!(stats.changed_path_levels[0].num_paths, 1);
946+
947+ // Index later commits from the mid point
948+ default_index_store
949+ .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 2)
950+ .block_on()
951+ .unwrap();
952+ let repo = repo.reload_at(repo.operation()).unwrap();
953+ let stats = as_readonly_index(&repo).stats();
954+ assert_eq!(stats.changed_path_commits_range, Some(5..8));
955+ assert_eq!(stats.changed_path_levels.len(), 1);
956+ assert_eq!(stats.changed_path_levels[0].num_commits, 3);
957+ assert_eq!(stats.changed_path_levels[0].num_changed_paths, 3);
958+ assert_eq!(stats.changed_path_levels[0].num_paths, 3);
959+
960+ // Index later and earlier commits from the mid point
961+ default_index_store
962+ .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 3)
963+ .block_on()
964+ .unwrap();
965+ let repo = repo.reload_at(repo.operation()).unwrap();
966+ let stats = as_readonly_index(&repo).stats();
967+ assert_eq!(stats.changed_path_commits_range, Some(4..10));
968+ assert_eq!(stats.changed_path_levels.len(), 1);
969+ assert_eq!(stats.changed_path_levels[0].num_commits, 6);
970+ assert_eq!(stats.changed_path_levels[0].num_changed_paths, 6);
971+ assert_eq!(stats.changed_path_levels[0].num_paths, 6);
972+}
973+
853974 #[test]
854975 fn test_merge_changed_path_segments_both_enabled() {
855976 let test_repo = TestRepo::init();