History

index: add function to build changed-path index at certain operation

Yuya Nishihara committed 1 year ago

Commit 8999e081 parent db0e4363 files · +215 −8Browse files

Unified · Split

3 files
  1. lib/src/default_index/changed_path.rs +16 −0
  2. lib/src/default_index/store.rs +77 −7
  3. lib/tests/test_index.rs +122 −1
lib/src/default_index/changed_path.rs+16−0
@@ -452,6 +452,22 @@ impl CompositeChangedPathIndex { · 451 unmodified lines
452 &self.readonly_segments452 &self.readonly_segments
453 }453 }
454 454
455 /// Appends segments from the `other` index. This and the other index should
456 /// be contiguous.
457 pub(super) fn append_segments(&mut self, other: &Self) {
458 assert!(self.mutable_segment.is_none());
459 let GlobalCommitPosition(self_start_pos) =
460 self.start_commit_pos.expect("should have start pos");
461 let Some(GlobalCommitPosition(other_start_pos)) = other.start_commit_pos else {
462 return;
463 };
464 assert_eq!(self_start_pos + self.num_commits, other_start_pos);
465 self.readonly_segments
466 .extend_from_slice(&other.readonly_segments);
467 self.mutable_segment = other.mutable_segment.clone();
468 self.num_commits += other.num_commits;
469 }
470
455 /// Maps `global_pos` to segment and segment-local position.471 /// Maps `global_pos` to segment and segment-local position.
456 fn find_segment(472 fn find_segment(
457 &self,473 &self,
lib/src/default_index/store.rs+77−7
@@ -34,6 +34,8 @@ use thiserror::Error; · 33 unmodified lines
34 34
35use super::changed_path::ChangedPathIndexSegmentId;35use super::changed_path::ChangedPathIndexSegmentId;
36use super::changed_path::CompositeChangedPathIndex;36use super::changed_path::CompositeChangedPathIndex;
37use super::changed_path::collect_changed_paths;
38use super::composite::AsCompositeIndex as _;
37use super::composite::CommitIndexSegmentId;39use super::composite::CommitIndexSegmentId;
38use super::entry::GlobalCommitPosition;40use super::entry::GlobalCommitPosition;
39use super::mutable::DefaultMutableIndex;41use super::mutable::DefaultMutableIndex;
@@ -390,11 +392,17 @@ impl DefaultIndexStore { · 350 unmodified lines
390 Ok(index)392 Ok(index)
391 }393 }
392 394
393 // TODO: for testing; replace with "build" function395 /// Builds changed-path index for the specified operation.
394 pub fn enable_changed_path_index_at_operation(396 ///
397 /// At most `max_commits` number of commits will be scanned from the latest
398 /// unindexed commit.
399 #[tracing::instrument(skip(self, store))]
400 pub async fn build_changed_path_index_at_operation(
395 &self,401 &self,
396 op_id: &OperationId,402 op_id: &OperationId,
397 store: &Arc<Store>,403 store: &Arc<Store>,
404 max_commits: u32,
405 // TODO: add progress callback?
398 ) -> Result<DefaultReadonlyIndex, DefaultIndexStoreError> {406 ) -> Result<DefaultReadonlyIndex, DefaultIndexStoreError> {
399 // Create directories in case the store was initialized by jj < 0.33.407 // Create directories in case the store was initialized by jj < 0.33.
400 self.ensure_base_dirs()408 self.ensure_base_dirs()
@@ -404,13 +412,75 @@ impl DefaultIndexStore { · 3 unmodified lines
404 change_id: store.change_id_length(),412 change_id: store.change_id_length(),
405 };413 };
406 let index = self.load_index_at_operation(op_id, field_lengths)?;414 let index = self.load_index_at_operation(op_id, field_lengths)?;
407 if index.changed_paths().start_commit_pos().is_some() {415 let old_changed_paths = index.changed_paths();
408 return Ok(index);416
417 // Distribute max_commits to contiguous pre/post ranges:
418 // ..|pre|old_changed_paths|post|
419 // (where pre.len() + post.len() <= max_commits)
420 let pre_start;
421 let pre_end;
422 let post_start;
423 let post_end;
424 if let Some(GlobalCommitPosition(pos)) = old_changed_paths.start_commit_pos() {
425 post_start = pos + old_changed_paths.num_commits();
426 assert!(post_start <= index.num_commits());
427 post_end = u32::saturating_add(post_start, max_commits).min(index.num_commits());
428 pre_start = u32::saturating_sub(pos, max_commits - (post_end - post_start));
429 pre_end = pos;
430 } else {
431 pre_start = u32::saturating_sub(index.num_commits(), max_commits);
432 pre_end = index.num_commits();
433 post_start = pre_end;
434 post_end = pre_end;
409 }435 }
410 let changed_paths =436
411 CompositeChangedPathIndex::empty(GlobalCommitPosition(index.num_commits()));437 let to_index_err = |source| DefaultIndexStoreError::IndexCommits {
438 op_id: op_id.clone(),
439 source,
440 };
441 let index_commit = async |changed_paths: &mut CompositeChangedPathIndex,
442 pos: GlobalCommitPosition| {
443 assert_eq!(changed_paths.next_mutable_commit_pos(), Some(pos));
444 let commit_id = index.as_composite().commits().entry_by_pos(pos).commit_id();
445 let commit = store.get_commit_async(&commit_id).await?;
446 let paths = collect_changed_paths(&index, &commit).await?;
447 changed_paths.add_changed_paths(paths);
448 Ok(())
449 };
450
451 // Index pre range
452 let mut new_changed_paths =
453 CompositeChangedPathIndex::empty(GlobalCommitPosition(pre_start));
454 new_changed_paths.make_mutable();
455 tracing::info!(?pre_start, ?pre_end, "indexing changed paths in commits");
456 for pos in (pre_start..pre_end).map(GlobalCommitPosition) {
457 index_commit(&mut new_changed_paths, pos)
458 .await
459 .map_err(to_index_err)?;
460 }
461 new_changed_paths
462 .save_in(&self.changed_path_segments_dir())
463 .map_err(DefaultIndexStoreError::SaveIndex)?;
464
465 // Copy previously-indexed segments
466 new_changed_paths.append_segments(old_changed_paths);
467
468 // Index post range, which is usually empty
469 new_changed_paths.make_mutable();
470 tracing::info!(?post_start, ?post_end, "indexing changed paths in commits");
471 for pos in (post_start..post_end).map(GlobalCommitPosition) {
472 index_commit(&mut new_changed_paths, pos)
473 .await
474 .map_err(to_index_err)?;
475 }
476 new_changed_paths.maybe_squash_with_ancestors();
477 new_changed_paths
478 .save_in(&self.changed_path_segments_dir())
479 .map_err(DefaultIndexStoreError::SaveIndex)?;
480
481 // Update the operation link to point to the new segments
412 let commits = index.readonly_commits().clone();482 let commits = index.readonly_commits().clone();
413 let index = DefaultReadonlyIndex::from_segment(commits, changed_paths);483 let index = DefaultReadonlyIndex::from_segment(commits, new_changed_paths);
414 self.associate_index_with_operation(&index, op_id)484 self.associate_index_with_operation(&index, op_id)
415 .map_err(|source| DefaultIndexStoreError::AssociateIndex {485 .map_err(|source| DefaultIndexStoreError::AssociateIndex {
416 op_id: op_id.to_owned(),486 op_id: op_id.to_owned(),
lib/tests/test_index.rs+122−1
@@ -65,7 +65,8 @@ fn enable_changed_path_index(repo: &ReadonlyRepo) -> Arc<ReadonlyRepo> { · 64 unmodified lines
65 let default_index_store: &DefaultIndexStore =65 let default_index_store: &DefaultIndexStore =
66 repo.index_store().as_any().downcast_ref().unwrap();66 repo.index_store().as_any().downcast_ref().unwrap();
67 default_index_store67 default_index_store
68 .enable_changed_path_index_at_operation(repo.op_id(), repo.store())68 .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 0)
69 .block_on()
69 .unwrap();70 .unwrap();
70 repo.reload_at(repo.operation()).unwrap()71 repo.reload_at(repo.operation()).unwrap()
71}72}
@@ -850,6 +851,126 @@ fn test_changed_path_segments() { · 778 unmodified lines
850 assert_eq!(stats.changed_path_levels[0].num_paths, 2);851 assert_eq!(stats.changed_path_levels[0].num_paths, 2);
851}852}
852 853
854#[test]
855fn test_build_changed_path_segments() {
856 let test_repo = TestRepo::init();
857 let repo = test_repo.repo;
858 let root_commit_id = repo.store().root_commit_id();
859 let default_index_store: &DefaultIndexStore =
860 repo.index_store().as_any().downcast_ref().unwrap();
861
862 let mut tx = repo.start_transaction();
863 for i in 1..10 {
864 let tree = create_tree(&repo, &[(repo_path(&i.to_string()), "")]);
865 tx.repo_mut()
866 .new_commit(vec![root_commit_id.clone()], tree.id())
867 .write()
868 .unwrap();
869 }
870 let repo = tx.commit("test").unwrap();
871
872 // Index the last 4 commits
873 default_index_store
874 .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 4)
875 .block_on()
876 .unwrap();
877 let repo = repo.reload_at(repo.operation()).unwrap();
878 let stats = as_readonly_index(&repo).stats();
879 assert_eq!(stats.changed_path_commits_range, Some(6..10));
880 assert_eq!(stats.changed_path_levels.len(), 1);
881 assert_eq!(stats.changed_path_levels[0].num_commits, 4);
882 assert_eq!(stats.changed_path_levels[0].num_changed_paths, 4);
883 assert_eq!(stats.changed_path_levels[0].num_paths, 4);
884
885 // Index remainders
886 default_index_store
887 .build_changed_path_index_at_operation(repo.op_id(), repo.store(), u32::MAX)
888 .block_on()
889 .unwrap();
890 let repo = repo.reload_at(repo.operation()).unwrap();
891 let stats = as_readonly_index(&repo).stats();
892 assert_eq!(stats.changed_path_commits_range, Some(0..10));
893 assert_eq!(stats.changed_path_levels.len(), 2);
894 assert_eq!(stats.changed_path_levels[0].num_commits, 6);
895 assert_eq!(stats.changed_path_levels[0].num_changed_paths, 5);
896 assert_eq!(stats.changed_path_levels[0].num_paths, 5);
897 assert_eq!(stats.changed_path_levels[1].num_commits, 4);
898 assert_eq!(stats.changed_path_levels[1].num_changed_paths, 4);
899 assert_eq!(stats.changed_path_levels[1].num_paths, 4);
900}
901
902#[test]
903fn test_build_changed_path_segments_partially_enabled() {
904 let test_repo = TestRepo::init();
905 let repo = test_repo.repo;
906 let root_commit_id = repo.store().root_commit_id();
907 let default_index_store: &DefaultIndexStore =
908 repo.index_store().as_any().downcast_ref().unwrap();
909
910 // Partially enable index by merging two operations
911 let tx1 = {
912 let mut tx = repo.start_transaction();
913 for i in 1..5 {
914 let tree = create_tree(&repo, &[(repo_path(&i.to_string()), "")]);
915 tx.repo_mut()
916 .new_commit(vec![root_commit_id.clone()], tree.id())
917 .write()
918 .unwrap();
919 }
920 let repo = tx.commit("test").unwrap();
921 let repo = enable_changed_path_index(&repo);
922 let mut tx = repo.start_transaction();
923 let tree = create_tree(&repo, &[(repo_path("5"), "")]);
924 tx.repo_mut()
925 .new_commit(vec![root_commit_id.clone()], tree.id())
926 .write()
927 .unwrap();
928 tx
929 };
930 let mut tx2 = repo.start_transaction();
931 for i in 6..10 {
932 let tree = create_tree(&repo, &[(repo_path(&i.to_string()), "")]);
933 tx2.repo_mut()
934 .new_commit(vec![root_commit_id.clone()], tree.id())
935 .write()
936 .unwrap();
937 }
938 let repo = commit_transactions(vec![tx1, tx2]);
939 let stats = as_readonly_index(&repo).stats();
940 assert_eq!(stats.num_commits, 10);
941 assert_eq!(stats.changed_path_commits_range, Some(5..6));
942 assert_eq!(stats.changed_path_levels.len(), 1);
943 assert_eq!(stats.changed_path_levels[0].num_commits, 1);
944 assert_eq!(stats.changed_path_levels[0].num_changed_paths, 1);
945 assert_eq!(stats.changed_path_levels[0].num_paths, 1);
946
947 // Index later commits from the mid point
948 default_index_store
949 .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 2)
950 .block_on()
951 .unwrap();
952 let repo = repo.reload_at(repo.operation()).unwrap();
953 let stats = as_readonly_index(&repo).stats();
954 assert_eq!(stats.changed_path_commits_range, Some(5..8));
955 assert_eq!(stats.changed_path_levels.len(), 1);
956 assert_eq!(stats.changed_path_levels[0].num_commits, 3);
957 assert_eq!(stats.changed_path_levels[0].num_changed_paths, 3);
958 assert_eq!(stats.changed_path_levels[0].num_paths, 3);
959
960 // Index later and earlier commits from the mid point
961 default_index_store
962 .build_changed_path_index_at_operation(repo.op_id(), repo.store(), 3)
963 .block_on()
964 .unwrap();
965 let repo = repo.reload_at(repo.operation()).unwrap();
966 let stats = as_readonly_index(&repo).stats();
967 assert_eq!(stats.changed_path_commits_range, Some(4..10));
968 assert_eq!(stats.changed_path_levels.len(), 1);
969 assert_eq!(stats.changed_path_levels[0].num_commits, 6);
970 assert_eq!(stats.changed_path_levels[0].num_changed_paths, 6);
971 assert_eq!(stats.changed_path_levels[0].num_paths, 6);
972}
973
853#[test]974#[test]
854fn test_merge_changed_path_segments_both_enabled() {975fn test_merge_changed_path_segments_both_enabled() {
855 let test_repo = TestRepo::init();976 let test_repo = TestRepo::init();