[pack] also put counts in order for stable packs

Sebastian Thiel committed 5 years ago

Commit f2991601 parent e7ecdc16 files · +35 −28Browse files

Unified · Split

6 files

git-pack/src/data/output/count/from_objects_iter.rs +14 −10
git-pack/src/data/output/entry/from_counts_iter.rs +5 −5
git-pack/src/data/output/in_order.rs +8 −6
git-pack/src/data/output/mod.rs +1 −1
git-pack/tests/pack/data/output.rs +2 −2
gitoxide-core/src/pack/create.rs +5 −4

git-pack/src/data/output/count/from_objects_iter.rs+14−10

-use crate::{data::output, find, FindExt};
+use crate::{
+    data::{output, output::ChunkId},
+    find, FindExt,
+};
 use dashmap::DashSet;
 use git_features::{parallel, progress::Progress};
 use git_hash::{oid, ObjectId};
         input_object_expansion,
         chunk_size,
     }: Options,
-) -> impl Iterator<Item = Result<Vec<output::Count>, Error<find::existing::Error<Find::Error>>>>
+) -> impl Iterator<Item = Result<(ChunkId, Vec<output::Count>), Error<find::existing::Error<Find::Error>>>>
        + parallel::reduce::Finalize<Reduce = reduce::Statistics<Error<find::existing::Error<Find::Error>>>>
 where
     Find: crate::Find + Clone + Send + Sync + 'static,
     let chunks = util::Chunks {
         iter: objects,
         size: chunk_size,
-    };
+    }
+    .enumerate();
     let seen_objs = Arc::new(dashmap::DashSet::<ObjectId>::new());
     let progress = Arc::new(parking_lot::Mutex::new(progress));
         },
         {
             let seen_objs = Arc::clone(&seen_objs);
-            move |oids: Vec<Oid>, (buf1, buf2, cache, progress)| {
+            move |(chunk_id, oids): (ChunkId, Vec<Oid>), (buf1, buf2, cache, progress)| {
                 use ObjectExpansion::*;
                 let mut out = Vec::new();
                 let mut tree_traversal_state = git_traverse::tree::breadthfirst::State::default();
                         AsIs => push_obj_count_unique(&mut out, seen_objs, id, &obj, progress, stats, false),
                     }
                 }
-                Ok((out, outcome))
+                Ok((chunk_id, out, outcome))
             }
         },
         reduce::Statistics::default(),
 mod reduce {
     use super::Outcome;
-    use crate::data::output;
+    use crate::data::{output, output::ChunkId};
     use git_features::parallel;
     use std::marker::PhantomData;
     }
     impl<Error> parallel::Reduce for Statistics<Error> {
-        type Input = Result<(Vec<output::Count>, Outcome), Error>;
+        type Input = Result<(ChunkId, Vec<output::Count>, Outcome), Error>;
-        type FeedProduce = Vec<output::Count>;
+        type FeedProduce = (ChunkId, Vec<output::Count>);
         type Output = Outcome;
         type Error = Error;
         fn feed(&mut self, item: Self::Input) -> Result<Self::FeedProduce, Self::Error> {
-            item.map(|(counts, mut stats)| {
+            item.map(|(chunk_id, counts, mut stats)| {
                 stats.total_objects = counts.len();
                 self.total.aggregate(stats);
-                counts
+                (chunk_id, counts)
             })
         }

git-pack/src/data/output/entry/from_counts_iter.rs+5−5

-use crate::{data::output, find, FindExt};
+use crate::{
+    data::{output, output::ChunkId},
+    find, FindExt,
+};
 use git_features::{parallel, progress::Progress};
 use std::sync::Arc;
 mod types {
     use crate::data::output::entry;
-    /// A counter for chunks to be able to put them back into original order later.
-    pub type ChunkId = usize;
     /// Information gathered during the run of [`from_counts_iter()`][super::from_counts_iter()].
     #[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
     #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
         NewEntry(#[from] entry::Error),
     }
 }
-pub use types::{ChunkId, Error, Options, Outcome};
+pub use types::{Error, Options, Outcome};

git-pack/src/data/output/in_order.rs+8−6

@@ -1,5 +1,7 @@
1	use std::cmp::Ordering;	1	use std::{cmp::Ordering, collections::BTreeMap};
2	use std::collections::BTreeMap;	2
		3	/// A counter for chunks to be able to put them back into original order later.
		4	pub type ChunkId = usize;
3		5
4	/// An iterator which olds iterated items with a sequential ID starting at 0 long enough to dispense them in order.	6	/// An iterator which olds iterated items with a sequential ID starting at 0 long enough to dispense them in order.
5	///	7	///
@@ -8,14 +10,14 @@ use std::collections::BTreeMap; · 2 unmodified lines
8	pub struct InOrderIter<T, I> {	10	pub struct InOrderIter<T, I> {
9	/// The iterator yielding the out-of-order elements we are to yield in order.	11	/// The iterator yielding the out-of-order elements we are to yield in order.
10	pub inner: I,	12	pub inner: I,
11	store: BTreeMap<usize, T>,	13	store: BTreeMap<ChunkId, T>,
12	next_chunk: usize,	14	next_chunk: ChunkId,
13	is_done: bool,	15	is_done: bool,
14	}	16	}
15		17
16	impl<T, E, I> From<I> for InOrderIter<T, I>	18	impl<T, E, I> From<I> for InOrderIter<T, I>
17	where	19	where
18	I: Iterator<Item = Result<(usize, T), E>>,	20	I: Iterator<Item = Result<(ChunkId, T), E>>,
19	{	21	{
20	fn from(iter: I) -> Self {	22	fn from(iter: I) -> Self {
21	InOrderIter {	23	InOrderIter {
@@ -29,7 +31,7 @@ where · 7 unmodified lines
29		31
30	impl<T, E, I> Iterator for InOrderIter<T, I>	32	impl<T, E, I> Iterator for InOrderIter<T, I>
31	where	33	where
32	I: Iterator<Item = Result<(usize, T), E>>,	34	I: Iterator<Item = Result<(ChunkId, T), E>>,
33	{	35	{
34	type Item = Result<T, E>;	36	type Item = Result<T, E>;
35		37

git-pack/src/data/output/mod.rs+1−1

@@ -12,4 +12,4 @@ pub use entry::Entry; · 11 unmodified lines
12	pub mod bytes;	12	pub mod bytes;
13		13
14	mod in_order;	14	mod in_order;
15	pub use in_order::InOrderIter;	15	pub use in_order::{ChunkId, InOrderIter};

git-pack/tests/pack/data/output.rs+2−2

         data::output::{db, DbKind},
         hex_to_id,
     };
+    use git_odb::data::output::InOrderIter;
     #[test]
     fn traversals() -> crate::Result {
                     ..Default::default()
                 },
             );
-            let counts: Vec<_> = counts_iter
+            let counts: Vec<_> = InOrderIter::from(counts_iter.by_ref())
-                .by_ref()
                 .collect::<Result<Vec<_>, _>>()?
                 .into_iter()
                 .flatten()

gitoxide-core/src/pack/create.rs+5−4

 use crate::OutputFormat;
 use anyhow::bail;
+use git_repository::odb::data::output::InOrderIter;
 use git_repository::{
     hash,
     hash::ObjectId,
     let counts = {
         let mut progress = progress.add_child("counting");
         progress.init(None, progress::count("objects"));
-        let mut counts_iter = pack::data::output::count::from_objects_iter(
+        let mut in_order_counts_iter = InOrderIter::from(pack::data::output::count::from_objects_iter(
             Arc::clone(&db),
             pack::cache::lru::StaticLinkedList::<64>::default,
             input,
                 chunk_size,
                 input_object_expansion: expansion.into(),
             },
-        );
+        ));
         let mut counts = Vec::new();
-        for c in counts_iter.by_ref() {
+        for c in in_order_counts_iter.by_ref() {
             if interrupt::is_triggered() {
                 bail!("Cancelled by user")
             }
             progress.inc_by(c.len());
             counts.extend(c.into_iter());
         }
-        stats.counts = counts_iter.finalize()?;
+        stats.counts = in_order_counts_iter.inner.finalize()?;
         progress.show_throughput(start);
         counts.shrink_to_fit();
         counts