[pack] also put counts in order for stable packs

Sebastian Thiel committed 5 years ago

Commit f2991601 parent e7ecdc16 files · +35 −28Browse files

Unified · Split

6 files

git-pack/src/data/output/count/from_objects_iter.rs +14 −10
git-pack/src/data/output/entry/from_counts_iter.rs +5 −5
git-pack/src/data/output/in_order.rs +8 −6
git-pack/src/data/output/mod.rs +1 −1
git-pack/tests/pack/data/output.rs +2 −2
gitoxide-core/src/pack/create.rs +5 −4

git-pack/src/data/output/count/from_objects_iter.rs+14−10

@@ -1,4 +1,7 @@
1		−use crate::{data::output, find, FindExt};
	1	+use crate::{
	2	+ data::{output, output::ChunkId},
	3	+ find, FindExt,
	4	+};
2	5	use dashmap::DashSet;
3	6	use git_features::{parallel, progress::Progress};
4	7	use git_hash::{oid, ObjectId};
@@ -29,7 +32,7 @@ pub fn from_objects_iter<Find, Iter, Oid, Cache>( · 24 unmodified lines
29	32	input_object_expansion,
30	33	chunk_size,
31	34	}: Options,
32		−) -> impl Iterator<Item = Result<Vec<output::Count>, Error<find::existing::Error<Find::Error>>>>
	35	+) -> impl Iterator<Item = Result<(ChunkId, Vec<output::Count>), Error<find::existing::Error<Find::Error>>>>
33	36	+ parallel::reduce::Finalize<Reduce = reduce::Statistics<Error<find::existing::Error<Find::Error>>>>
34	37	where
35	38	Find: crate::Find + Clone + Send + Sync + 'static,
@@ -48,7 +51,8 @@ where · 12 unmodified lines
48	51	let chunks = util::Chunks {
49	52	iter: objects,
50	53	size: chunk_size,
51		− };
	54	+ }
	55	+ .enumerate();
52	56	let seen_objs = Arc::new(dashmap::DashSet::<ObjectId>::new());
53	57	let progress = Arc::new(parking_lot::Mutex::new(progress));
54	58
@@ -72,7 +76,7 @@ where · 17 unmodified lines
72	76	},
73	77	{
74	78	let seen_objs = Arc::clone(&seen_objs);
75		− move \|oids: Vec<Oid>, (buf1, buf2, cache, progress)\| {
	79	+ move \|(chunk_id, oids): (ChunkId, Vec<Oid>), (buf1, buf2, cache, progress)\| {
76	80	use ObjectExpansion::*;
77	81	let mut out = Vec::new();
78	82	let mut tree_traversal_state = git_traverse::tree::breadthfirst::State::default();
@@ -243,7 +247,7 @@ where · 164 unmodified lines
243	247	AsIs => push_obj_count_unique(&mut out, seen_objs, id, &obj, progress, stats, false),
244	248	}
245	249	}
246		− Ok((out, outcome))
	250	+ Ok((chunk_id, out, outcome))
247	251	}
248	252	},
249	253	reduce::Statistics::default(),
@@ -525,7 +529,7 @@ pub use types::{Error, ObjectExpansion, Options, Outcome}; · 275 unmodified lines
525	529
526	530	mod reduce {
527	531	use super::Outcome;
528		− use crate::data::output;
	532	+ use crate::data::{output, output::ChunkId};
529	533	use git_features::parallel;
530	534	use std::marker::PhantomData;
531	535
@@ -544,16 +548,16 @@ mod reduce { · 12 unmodified lines
544	548	}
545	549
546	550	impl<Error> parallel::Reduce for Statistics<Error> {
547		− type Input = Result<(Vec<output::Count>, Outcome), Error>;
548		− type FeedProduce = Vec<output::Count>;
	551	+ type Input = Result<(ChunkId, Vec<output::Count>, Outcome), Error>;
	552	+ type FeedProduce = (ChunkId, Vec<output::Count>);
549	553	type Output = Outcome;
550	554	type Error = Error;
551	555
552	556	fn feed(&mut self, item: Self::Input) -> Result<Self::FeedProduce, Self::Error> {
553		− item.map(\|(counts, mut stats)\| {
	557	+ item.map(\|(chunk_id, counts, mut stats)\| {
554	558	stats.total_objects = counts.len();
555	559	self.total.aggregate(stats);
556		− counts
	560	+ (chunk_id, counts)
557	561	})
558	562	}
559	563

git-pack/src/data/output/entry/from_counts_iter.rs+5−5

@@ -1,4 +1,7 @@
1		−use crate::{data::output, find, FindExt};
	1	+use crate::{
	2	+ data::{output, output::ChunkId},
	3	+ find, FindExt,
	4	+};
2	5	use git_features::{parallel, progress::Progress};
3	6	use std::sync::Arc;
4	7
@@ -185,9 +188,6 @@ mod reduce { · 180 unmodified lines
185	188	mod types {
186	189	use crate::data::output::entry;
187	190
188		− /// A counter for chunks to be able to put them back into original order later.
189		− pub type ChunkId = usize;
190		−
191	191	/// Information gathered during the run of [`from_counts_iter()`][super::from_counts_iter()].
192	192	#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
193	193	#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
@@ -248,4 +248,4 @@ mod types { · 54 unmodified lines
248	248	NewEntry(#[from] entry::Error),
249	249	}
250	250	}
251		−pub use types::{ChunkId, Error, Options, Outcome};
	251	+pub use types::{Error, Options, Outcome};

git-pack/src/data/output/in_order.rs+8−6

@@ -1,5 +1,7 @@
1		−use std::cmp::Ordering;
2		−use std::collections::BTreeMap;
	1	+use std::{cmp::Ordering, collections::BTreeMap};
	2	+
	3	+/// A counter for chunks to be able to put them back into original order later.
	4	+pub type ChunkId = usize;
3	5
4	6	/// An iterator which olds iterated items with a sequential ID starting at 0 long enough to dispense them in order.
5	7	///
@@ -8,14 +10,14 @@ use std::collections::BTreeMap; · 2 unmodified lines
8	10	pub struct InOrderIter<T, I> {
9	11	/// The iterator yielding the out-of-order elements we are to yield in order.
10	12	pub inner: I,
11		− store: BTreeMap<usize, T>,
12		− next_chunk: usize,
	13	+ store: BTreeMap<ChunkId, T>,
	14	+ next_chunk: ChunkId,
13	15	is_done: bool,
14	16	}
15	17
16	18	impl<T, E, I> From<I> for InOrderIter<T, I>
17	19	where
18		− I: Iterator<Item = Result<(usize, T), E>>,
	20	+ I: Iterator<Item = Result<(ChunkId, T), E>>,
19	21	{
20	22	fn from(iter: I) -> Self {
21	23	InOrderIter {
@@ -29,7 +31,7 @@ where · 7 unmodified lines
29	31
30	32	impl<T, E, I> Iterator for InOrderIter<T, I>
31	33	where
32		− I: Iterator<Item = Result<(usize, T), E>>,
	34	+ I: Iterator<Item = Result<(ChunkId, T), E>>,
33	35	{
34	36	type Item = Result<T, E>;
35	37

git-pack/src/data/output/mod.rs+1−1

@@ -12,4 +12,4 @@ pub use entry::Entry; · 11 unmodified lines
12	12	pub mod bytes;
13	13
14	14	mod in_order;
15		−pub use in_order::InOrderIter;
	15	+pub use in_order::{ChunkId, InOrderIter};

git-pack/tests/pack/data/output.rs+2−2

@@ -50,6 +50,7 @@ mod count_and_entries { · 49 unmodified lines
50	50	data::output::{db, DbKind},
51	51	hex_to_id,
52	52	};
	53	+ use git_odb::data::output::InOrderIter;
53	54
54	55	#[test]
55	56	fn traversals() -> crate::Result {
@@ -153,8 +154,7 @@ mod count_and_entries { · 97 unmodified lines
153	154	..Default::default()
154	155	},
155	156	);
156		− let counts: Vec<_> = counts_iter
157		− .by_ref()
	157	+ let counts: Vec<_> = InOrderIter::from(counts_iter.by_ref())
158	158	.collect::<Result<Vec<_>, _>>()?
159	159	.into_iter()
160	160	.flatten()

gitoxide-core/src/pack/create.rs+5−4

@@ -1,5 +1,6 @@
1	1	use crate::OutputFormat;
2	2	use anyhow::bail;
	3	+use git_repository::odb::data::output::InOrderIter;
3	4	use git_repository::{
4	5	hash,
5	6	hash::ObjectId,
@@ -117,7 +118,7 @@ where · 111 unmodified lines
117	118	let counts = {
118	119	let mut progress = progress.add_child("counting");
119	120	progress.init(None, progress::count("objects"));
120		− let mut counts_iter = pack::data::output::count::from_objects_iter(
	121	+ let mut in_order_counts_iter = InOrderIter::from(pack::data::output::count::from_objects_iter(
121	122	Arc::clone(&db),
122	123	pack::cache::lru::StaticLinkedList::<64>::default,
123	124	input,
@@ -127,9 +128,9 @@ where · 3 unmodified lines
127	128	chunk_size,
128	129	input_object_expansion: expansion.into(),
129	130	},
130		− );
	131	+ ));
131	132	let mut counts = Vec::new();
132		− for c in counts_iter.by_ref() {
	133	+ for c in in_order_counts_iter.by_ref() {
133	134	if interrupt::is_triggered() {
134	135	bail!("Cancelled by user")
135	136	}
@@ -137,7 +138,7 @@ where · 1 unmodified line
137	138	progress.inc_by(c.len());
138	139	counts.extend(c.into_iter());
139	140	}
140		− stats.counts = counts_iter.finalize()?;
	141	+ stats.counts = in_order_counts_iter.inner.finalize()?;
141	142	progress.show_throughput(start);
142	143	counts.shrink_to_fit();
143	144	counts