Improve traversal performance when hidden tips are used.

Now it will abort early if there are only hidden tips to be traversed, which cannot change the result anymore.

Sebastian Thiel committed 1 year ago

Commit 219655f1 parent 1b08fd92 files · +60 −25Browse files

Unified · Split

2 files

gix-traverse/src/commit/simple.rs +59 −24
gix-traverse/tests/traverse/commit/simple.rs +1 −1

@@ -78,13 +78,13 @@ pub enum Error { · 77 unmodified lines
78	use Result as Either;	78	use Result as Either;
79		79
80	type QueueKey<T> = Either<T, Reverse<T>>;	80	type QueueKey<T> = Either<T, Reverse<T>>;
81	type CommitDateQueue = gix_revwalk::PriorityQueue<QueueKey<SecondsSinceUnixEpoch>, ObjectId>;	81	type CommitDateQueue = gix_revwalk::PriorityQueue<QueueKey<SecondsSinceUnixEpoch>, (ObjectId, CommitState)>;
82	type Candidates = VecDeque<crate::commit::Info>;	82	type Candidates = VecDeque<crate::commit::Info>;
83		83
84	/// The state used and potentially shared by multiple graph traversals.	84	/// The state used and potentially shared by multiple graph traversals.
85	#[derive(Clone)]	85	#[derive(Clone)]
86	pub(super) struct State {	86	pub(super) struct State {
87	next: VecDeque<ObjectId>,	87	next: VecDeque<(ObjectId, CommitState)>,
88	queue: CommitDateQueue,	88	queue: CommitDateQueue,
89	buf: Vec<u8>,	89	buf: Vec<u8>,
90	seen: gix_revwalk::graph::IdMap<CommitState>,	90	seen: gix_revwalk::graph::IdMap<CommitState>,
@@ -184,9 +184,10 @@ mod init { · 93 unmodified lines
184	}	184	}
185	Sorting::ByCommitTime(order) \| Sorting::ByCommitTimeCutoff { order, .. } => {	185	Sorting::ByCommitTime(order) \| Sorting::ByCommitTimeCutoff { order, .. } => {
186	let state = &mut self.state;	186	let state = &mut self.state;
187	for commit_id in state.next.drain(..) {	187	for (commit_id, commit_state) in state.next.drain(..) {
188	add_to_queue(	188	add_to_queue(
189	commit_id,	189	commit_id,
		190	commit_state,
190	order,	191	order,
191	sorting.cutoff_time(),	192	sorting.cutoff_time(),
192	&mut state.queue,	193	&mut state.queue,
@@ -226,11 +227,12 @@ mod init { · 33 unmodified lines
226	// Assure we start traversing hidden variants of a commit first, give them a head-start.	227	// Assure we start traversing hidden variants of a commit first, give them a head-start.
227	match self.sorting {	228	match self.sorting {
228	Sorting::BreadthFirst => {	229	Sorting::BreadthFirst => {
229	state.next.push_front(id_to_ignore);	230	state.next.push_front((id_to_ignore, CommitState::Hidden));
230	}	231	}
231	Sorting::ByCommitTime(order) \| Sorting::ByCommitTimeCutoff { order, .. } => {	232	Sorting::ByCommitTime(order) \| Sorting::ByCommitTimeCutoff { order, .. } => {
232	add_to_queue(	233	add_to_queue(
233	id_to_ignore,	234	id_to_ignore,
		235	CommitState::Hidden,
234	order,	236	order,
235	self.sorting.cutoff_time(),	237	self.sorting.cutoff_time(),
236	&mut state.queue,	238	&mut state.queue,
@@ -273,6 +275,7 @@ mod init { · 36 unmodified lines
273		275
274	fn add_to_queue(	276	fn add_to_queue(
275	commit_id: ObjectId,	277	commit_id: ObjectId,
		278	commit_state: CommitState,
276	order: CommitTimeOrder,	279	order: CommitTimeOrder,
277	cutoff_time: Option<SecondsSinceUnixEpoch>,	280	cutoff_time: Option<SecondsSinceUnixEpoch>,
278	queue: &mut CommitDateQueue,	281	queue: &mut CommitDateQueue,
@@ -284,11 +287,11 @@ mod init { · 5 unmodified lines
284	let key = to_queue_key(time, order);	287	let key = to_queue_key(time, order);
285	match (cutoff_time, order) {	288	match (cutoff_time, order) {
286	(Some(cutoff_time), _) if time >= cutoff_time => {	289	(Some(cutoff_time), _) if time >= cutoff_time => {
287	queue.insert(key, commit_id);	290	queue.insert(key, (commit_id, commit_state));
288	}	291	}
289	(Some(_), _) => {}	292	(Some(_), _) => {}
290	(None, _) => {	293	(None, _) => {
291	queue.insert(key, commit_id);	294	queue.insert(key, (commit_id, commit_state));
292	}	295	}
293	}	296	}
294	Ok(())	297	Ok(())
@@ -339,10 +342,11 @@ mod init { · 44 unmodified lines
339	state.clear();	342	state.clear();
340	state.next.reserve(tips.size_hint().0);	343	state.next.reserve(tips.size_hint().0);
341	for tip in tips.map(Into::into) {	344	for tip in tips.map(Into::into) {
342	let seen = state.seen.insert(tip, CommitState::Interesting);	345	let commit_state = CommitState::Interesting;
		346	let seen = state.seen.insert(tip, commit_state);
343	// We know there can only be duplicate interesting ones.	347	// We know there can only be duplicate interesting ones.
344	if seen.is_none() && predicate(&tip) {	348	if seen.is_none() && predicate(&tip) {
345	state.next.push_back(tip);	349	state.next.push_back((tip, commit_state));
346	}	350	}
347	}	351	}
348	}	352	}
@@ -418,16 +422,23 @@ mod init { · 69 unmodified lines
418	cutoff: Option<SecondsSinceUnixEpoch>,	422	cutoff: Option<SecondsSinceUnixEpoch>,
419	) -> Option<Result<Info, Error>> {	423	) -> Option<Result<Info, Error>> {
420	let state = &mut self.state;	424	let state = &mut self.state;
		425	let next = &mut state.queue;
421		426
422	'skip_hidden: loop {	427	'skip_hidden: loop {
423	let (commit_time, oid) = match state.queue.pop()? {	428	let (commit_time, (oid, _queued_commit_state)) = match next.pop()? {
424	(Newest(t) \| Oldest(Reverse(t)), o) => (t, o),	429	(Newest(t) \| Oldest(Reverse(t)), o) => (t, o),
425	};	430	};
426	let mut parents: ParentIds = Default::default();	431	let mut parents: ParentIds = Default::default();
427	// TODO(perf): can avoid this lookup by storing state on `queue` respectively.	432
428	// ALSO: need to look ahead for early aborts, i.e. if there is only hidden left to traverse.	433	// Always use the state that is actually stored, as we may change the type as we go.
429	// Maybe this can be counted?
430	let commit_state = *state.seen.get(&oid).expect("every commit we traverse has state added");	434	let commit_state = *state.seen.get(&oid).expect("every commit we traverse has state added");
		435	if can_deplete_candidates_early(
		436	next.iter_unordered().map(\|t\| t.1),
		437	commit_state,
		438	state.candidates.as_ref(),
		439	) {
		440	return None;
		441	}
431	match super::super::find(self.cache.as_ref(), &self.objects, &oid, &mut state.buf) {	442	match super::super::find(self.cache.as_ref(), &self.objects, &oid, &mut state.buf) {
432	Ok(Either::CachedCommit(commit)) => {	443	Ok(Either::CachedCommit(commit)) => {
433	if !collect_parents(&mut state.parent_ids, self.cache.as_ref(), commit.iter_parents()) {	444	if !collect_parents(&mut state.parent_ids, self.cache.as_ref(), commit.iter_parents()) {
@@ -443,7 +454,7 @@ mod init { · 9 unmodified lines
443	&mut state.candidates,	454	&mut state.candidates,
444	commit_state,	455	commit_state,
445	&mut self.predicate,	456	&mut self.predicate,
446	&mut state.queue,	457	next,
447	order,	458	order,
448	cutoff,	459	cutoff,
449	\|\| parent_commit_time,	460	\|\| parent_commit_time,
@@ -462,7 +473,7 @@ mod init { · 12 unmodified lines
462	&mut state.candidates,	473	&mut state.candidates,
463	commit_state,	474	commit_state,
464	&mut self.predicate,	475	&mut self.predicate,
465	&mut state.queue,	476	next,
466	order,	477	order,
467	cutoff,	478	cutoff,
468	\|\| {	479	\|\| {
@@ -505,6 +516,28 @@ mod init { · 36 unmodified lines
505	}	516	}
506	}	517	}
507		518
		519	/// Returns `true` if we have only hidden cursors queued for traversal, assuming that we don't see interesting ones ever again.
		520	///
		521	/// `unqueued_commit_state` is the state of the commit that is currently being processed.
		522	fn can_deplete_candidates_early(
		523	mut queued_states: impl Iterator<Item = CommitState>,
		524	unqueued_commit_state: CommitState,
		525	candidates: Option<&Candidates>,
		526	) -> bool {
		527	if candidates.is_none() {
		528	return false;
		529	}
		530	if unqueued_commit_state.is_interesting() {
		531	return false;
		532	}
		533
		534	let mut is_empty = true;
		535	queued_states.all(\|state\| {
		536	is_empty = false;
		537	state.is_hidden()
		538	}) && !is_empty
		539	}
		540
508	/// Utilities	541	/// Utilities
509	impl<Find, Predicate> Simple<Find, Predicate>	542	impl<Find, Predicate> Simple<Find, Predicate>
510	where	543	where
@@ -513,14 +546,16 @@ mod init { · 2 unmodified lines
513	{	546	{
514	fn next_by_topology(&mut self) -> Option<Result<Info, Error>> {	547	fn next_by_topology(&mut self) -> Option<Result<Info, Error>> {
515	let state = &mut self.state;	548	let state = &mut self.state;
		549	let next = &mut state.next;
516	'skip_hidden: loop {	550	'skip_hidden: loop {
517	let oid = state.next.pop_front()?;	551	let (oid, _queued_commit_state) = next.pop_front()?;
518	let mut parents: ParentIds = Default::default();	552	let mut parents: ParentIds = Default::default();
519	// TODO(perf): can avoid this lookup by storing state on `next` respectively.
520	// ALSO: need to look ahead for early aborts, i.e. if there is only hidden left to traverse.
521	// Maybe this can be counted?
522	let commit_state = *state.seen.get(&oid).expect("every commit we traverse has state added");
523		553
		554	// Always use the state that is actually stored, as we may change the type as we go.
		555	let commit_state = *state.seen.get(&oid).expect("every commit we traverse has state added");
		556	if can_deplete_candidates_early(next.iter().map(\|t\| t.1), commit_state, state.candidates.as_ref()) {
		557	return None;
		558	}
524	match super::super::find(self.cache.as_ref(), &self.objects, &oid, &mut state.buf) {	559	match super::super::find(self.cache.as_ref(), &self.objects, &oid, &mut state.buf) {
525	Ok(Either::CachedCommit(commit)) => {	560	Ok(Either::CachedCommit(commit)) => {
526	if !collect_parents(&mut state.parent_ids, self.cache.as_ref(), commit.iter_parents()) {	561	if !collect_parents(&mut state.parent_ids, self.cache.as_ref(), commit.iter_parents()) {
@@ -537,7 +572,7 @@ mod init { · 10 unmodified lines
537	&mut state.candidates,	572	&mut state.candidates,
538	commit_state,	573	commit_state,
539	&mut self.predicate,	574	&mut self.predicate,
540	&mut state.next,	575	next,
541	);	576	);
542	if commit_state.is_interesting() && matches!(self.parents, Parents::First) {	577	if commit_state.is_interesting() && matches!(self.parents, Parents::First) {
543	break;	578	break;
@@ -556,7 +591,7 @@ mod init { · 12 unmodified lines
556	&mut state.candidates,	591	&mut state.candidates,
557	commit_state,	592	commit_state,
558	&mut self.predicate,	593	&mut self.predicate,
559	&mut state.next,	594	next,
560	);	595	);
561	if commit_state.is_interesting() && matches!(self.parents, Parents::First) {	596	if commit_state.is_interesting() && matches!(self.parents, Parents::First) {
562	break;	597	break;
@@ -608,7 +643,7 @@ mod init { · 45 unmodified lines
608	candidates: &mut Option<Candidates>,	643	candidates: &mut Option<Candidates>,
609	commit_state: CommitState,	644	commit_state: CommitState,
610	predicate: &mut impl FnMut(&oid) -> bool,	645	predicate: &mut impl FnMut(&oid) -> bool,
611	next: &mut VecDeque<ObjectId>,	646	next: &mut VecDeque<(ObjectId, CommitState)>,
612	) {	647	) {
613	let enqueue = match seen.entry(parent_id) {	648	let enqueue = match seen.entry(parent_id) {
614	Entry::Occupied(mut e) => {	649	Entry::Occupied(mut e) => {
@@ -627,7 +662,7 @@ mod init { · 12 unmodified lines
627	}	662	}
628	};	663	};
629	if enqueue {	664	if enqueue {
630	next.push_back(parent_id);	665	next.push_back((parent_id, commit_state));
631	}	666	}
632	}	667	}
633		668
@@ -665,7 +700,7 @@ mod init { · 31 unmodified lines
665	let key = to_queue_key(parent_commit_time, order);	700	let key = to_queue_key(parent_commit_time, order);
666	match cutoff {	701	match cutoff {
667	Some(cutoff_older_than) if parent_commit_time < cutoff_older_than => {}	702	Some(cutoff_older_than) if parent_commit_time < cutoff_older_than => {}
668	Some(_) \| None => queue.insert(key, parent_id),	703	Some(_) \| None => queue.insert(key, (parent_id, commit_state)),
669	}	704	}
670	}	705	}
671	}	706	}

gix-traverse/tests/traverse/commit/simple.rs+1−1

     fn all_sortings() -> impl Iterator<Item = Sorting> {
         [
+            Sorting::BreadthFirst,
             Sorting::ByCommitTime(CommitTimeOrder::NewestFirst),
             Sorting::ByCommitTime(CommitTimeOrder::OldestFirst),
-            Sorting::BreadthFirst,
         ]
         .into_iter()
     }

Improve traversal performance when hidden tips are used.

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code