[pack-gen] very close to a basic impl of count + entries-gen…

…but now the type system plays tricks on me and it's kind of in the way.

Sebastian Thiel committed 5 years ago

Commit c9274291 parent 932b4396 files · +135 −517Browse files

Unified · Split

6 files

git-odb/src/pack/data/output/count_objects.rs +21 −40
git-odb/src/pack/data/output/entry.rs +36 −4
git-odb/src/pack/data/output/mod.rs +4 −17
git-odb/src/pack/data/output/objects_to_entries.rs +36 −437
git-odb/tests/odb/pack/data/output.rs +21 −3
gitoxide-core/src/pack/create.rs +17 −16

git-odb/src/pack/data/output/count_objects.rs+21−40

 use crate::{pack, pack::data::output, FindExt};
 use dashmap::DashSet;
-use git_features::{hash, parallel, progress::Progress};
+use git_features::{parallel, progress::Progress};
 use git_hash::{oid, ObjectId};
 use git_object::immutable;
                             let mut id = id.to_owned();
                             loop {
-                                push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
+                                push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj);
                                 match obj.kind {
                                     Tree | Blob => break,
                                     Tag => {
                                             let obj = db
                                                 .find_existing(tree_id, buf1, cache)
                                                 .map_err(|_| Error::NotFound { oid: tree_id })?;
-                                            push_obj_count_unique(&mut out, seen_objs, &db, version, &tree_id, &obj)?;
+                                            push_obj_count_unique(&mut out, seen_objs, &db, version, &tree_id, &obj);
                                             immutable::TreeIter::from_bytes(obj.data)
                                         };
                                                         version,
                                                         &commit_id,
                                                         &parent_commit_obj,
-                                                    )?;
+                                                    );
                                                     immutable::CommitIter::from_bytes(parent_commit_obj.data)
                                                         .tree_id()
                                                         .expect("every commit has a tree")
                                                         version,
                                                         &parent_tree_id,
                                                         &parent_tree_obj,
-                                                    )?;
+                                                    );
                                                     immutable::TreeIter::from_bytes(parent_tree_obj.data)
                                                 };
                                             let obj = db
                                                 .find(id, buf2, cache)?
                                                 .ok_or(Error::NotFound { oid: id.to_owned() })?;
-                                            out.push(obj_to_count(&db, version, id, &obj)?);
+                                            out.push(obj_to_count(&db, version, id, &obj));
                                         }
                                         break;
                                     }
                             let mut id: ObjectId = id.into();
                             let mut obj = obj;
                             loop {
-                                push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
+                                push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj);
                                 match obj.kind {
                                     Tree => {
                                         traverse_delegate.clear();
                                             let obj = db
                                                 .find(id, buf1, cache)?
                                                 .ok_or(Error::NotFound { oid: id.to_owned() })?;
-                                            out.push(obj_to_count(&db, version, id, &obj)?);
+                                            out.push(obj_to_count(&db, version, id, &obj));
                                         }
                                         break;
                                     }
                                 }
                             }
                         }
-                        AsIs => push_obj_count_unique(&mut out, seen_objs, &db, version, id, &obj)?,
+                        AsIs => push_obj_count_unique(&mut out, seen_objs, &db, version, id, &obj),
                     }
                 }
                 Ok(out)
     }
 }
-fn push_obj_count_unique<Find>(
+fn push_obj_count_unique(
     out: &mut Vec<output::Count>,
     all_seen: &DashSet<ObjectId>,
-    db: &Find,
+    db: &impl crate::Find,
     version: pack::data::Version,
     id: &oid,
     obj: &crate::data::Object<'_>,
-) -> Result<(), Error<Find::Error>>
+) {
-where
-    Find: crate::Find,
-{
     let inserted = all_seen.insert(id.to_owned());
     if inserted {
-        out.push(obj_to_count(db, version, id, obj)?);
+        out.push(obj_to_count(db, version, id, obj));
     }
-    Ok(())
 }
-fn obj_to_count<Find>(
+fn obj_to_count(
-    db: &Find,
+    db: &impl crate::Find,
     version: pack::data::Version,
     id: &oid,
     obj: &crate::data::Object<'_>,
-) -> Result<output::Count, Error<Find::Error>>
+) -> output::Count {
-where
+    match db.pack_entry(&obj) {
-    Find: crate::Find,
-{
-    Ok(match db.pack_entry(&obj) {
         Some(entry) if entry.version == version => {
             let pack_entry = pack::data::Entry::from_bytes(entry.data, 0);
-            if let Some(expected) = entry.crc32 {
-                let actual = hash::crc32(entry.data);
-                if actual != expected {
-                    return Err(Error::PackToPackCopyCrc32Mismatch { actual, expected });
-                }
-            }
             if pack_entry.header.is_base() {
                 output::Count {
                     id: id.to_owned(),
                     object_kind: pack_entry.header.to_kind().expect("non-delta"),
-                    entry_kind: output::entry::Kind::Base,
                     decompressed_size: obj.data.len(),
                     entry_pack_location: obj.pack_location.clone(),
                 }
             } else {
-                output::Count::from_data(id, &obj).map_err(Error::NewEntry)?
+                output::Count::from_data(id, &obj)
             }
         }
-        _ => output::Count::from_data(id, &obj).map_err(Error::NewEntry)?,
+        _ => output::Count::from_data(id, &obj),
-    })
+    }
 }
 mod util {
 }
 mod types {
-    use crate::pack::data::output::count;
     use git_hash::ObjectId;
     /// The way input objects are handled
         #[error(transparent)]
         CommitDecode(git_object::immutable::object::decode::Error),
         #[error(transparent)]
-        Locate(#[from] LocateErr),
+        Find(#[from] LocateErr),
         #[error(transparent)]
         TreeTraverse(git_traverse::tree::breadthfirst::Error),
         #[error(transparent)]
         TreeChanges(git_diff::tree::changes::Error),
         #[error("Object id {oid} wasn't found in object database")]
         NotFound { oid: ObjectId },
-        #[error("Entry expected to have hash {expected}, but it had {actual}")]
-        PackToPackCopyCrc32Mismatch { actual: u32, expected: u32 },
-        #[error(transparent)]
-        NewEntry(count::Error),
     }
 }
 pub use types::{Error, ObjectExpansion, Options};

git-odb/src/pack/data/output/entry.rs+36−4

@@ -1,7 +1,8 @@
1	use crate::{	1	use crate::{
2	data,	2	data, find,
3	pack::{self, data::output},	3	pack::{self, data::output},
4	};	4	};
		5	use git_features::hash;
5	use git_hash::ObjectId;	6	use git_hash::ObjectId;
6	use std::io::Write;	7	use std::io::Write;
7		8
@@ -52,14 +53,45 @@ pub enum Kind { · 44 unmodified lines
52	pub enum Error {	53	pub enum Error {
53	#[error("{0}")]	54	#[error("{0}")]
54	ZlibDeflate(#[from] std::io::Error),	55	ZlibDeflate(#[from] std::io::Error),
		56	#[error("Entry expected to have hash {expected}, but it had {actual}")]
		57	PackToPackCopyCrc32Mismatch { actual: u32, expected: u32 },
55	}	58	}
56		59
57	impl output::Entry {	60	impl output::Entry {
		61	pub fn from_pack_entry(
		62	entry: find::PackEntry<'_>,
		63	count: &output::Count,
		64	version: pack::data::Version,
		65	) -> Option<Result<Self, Error>> {
		66	if entry.version != version {
		67	return None;
		68	};
		69
		70	let pack_entry = pack::data::Entry::from_bytes(entry.data, 0);
		71	if let Some(expected) = entry.crc32 {
		72	let actual = hash::crc32(entry.data);
		73	if actual != expected {
		74	return Some(Err(Error::PackToPackCopyCrc32Mismatch { actual, expected }));
		75	}
		76	}
		77	if pack_entry.header.is_base() {
		78	Some(Ok(output::Entry {
		79	id: count.id.to_owned(),
		80	object_kind: count.object_kind,
		81	kind: output::entry::Kind::Base,
		82	decompressed_size: count.decompressed_size,
		83	compressed_data: entry.data[pack_entry.data_offset as usize..].to_owned(),
		84	}))
		85	} else {
		86	None
		87	}
		88	}
		89
58	/// Create a new instance from the given `oid` and its corresponding git `obj`ect data.	90	/// Create a new instance from the given `oid` and its corresponding git `obj`ect data.
59	pub fn from_data(oid: impl Into<ObjectId>, obj: &data::Object<'_>) -> Result<Self, Error> {	91	pub fn from_data(count: &output::Count, obj: &data::Object<'_>) -> Result<Self, Error> {
60	Ok(output::Entry {	92	Ok(output::Entry {
61	id: oid.into(),	93	id: count.id.to_owned(),
62	object_kind: obj.kind,	94	object_kind: count.object_kind,
63	kind: Kind::Base,	95	kind: Kind::Base,
64	decompressed_size: obj.data.len(),	96	decompressed_size: obj.data.len(),
65	compressed_data: {	97	compressed_data: {

git-odb/src/pack/data/output/mod.rs+4−17

 mod count {
-    use crate::{data, pack, pack::data::output::entry};
+    use crate::{data, pack};
     use git_hash::ObjectId;
     /// An item representing a future Entry in the leanest way possible.
         pub object_kind: git_object::Kind,
         /// The size in bytes needed once `data` gets decompressed
         pub decompressed_size: usize,
-        /// The kind of entry represented by `data`. It's used alongside with it to complete the pack entry
-        /// at rest or in transit.
-        pub entry_kind: entry::Kind,
         /// A way to locate a pack entry in the object database, only available if the object is in a pack.
         pub(crate) entry_pack_location: Option<pack::bundle::Location>,
     }
-    /// The error returned by [`output::Entry::from_data()`].
-    #[allow(missing_docs)]
-    #[derive(Debug, thiserror::Error)]
-    pub enum Error {
-        #[error("{0}")]
-        ZlibDeflate(#[from] std::io::Error),
-    }
     impl Count {
         /// Create a new instance from the given `oid` and its corresponding git `obj`ect data.
-        pub fn from_data(oid: impl Into<ObjectId>, obj: &data::Object<'_>) -> Result<Self, Error> {
+        pub fn from_data(oid: impl Into<ObjectId>, obj: &data::Object<'_>) -> Self {
-            Ok(Count {
+            Count {
                 id: oid.into(),
                 object_kind: obj.kind,
-                entry_kind: entry::Kind::Base,
                 decompressed_size: obj.data.len(),
                 entry_pack_location: obj.pack_location.clone(),
-            })
+            }
         }
     }
 }

git-odb/src/pack/data/output/objects_to_entries.rs+36−437

@@ -1,16 +1,15 @@
1	use crate::{pack, pack::data::output, FindExt};	1	use crate::{pack, pack::data::output, FindExt};
2	use git_features::{hash, parallel, progress::Progress};	2	use git_features::{parallel, progress::Progress};
3	use git_hash::{oid, ObjectId};
4		3
5	/// Write all `objects` into `out` without attempting to apply any delta compression.	4	/// Given a known list of object `counts`, calculate entries ready to be put into a data pack.
6	/// TODO: Update this
7	/// This allows objects to be written rather immediately.
8	/// Objects are held in memory and compressed using DEFLATE, with those in-flight chunks of compressed
9	/// objects being sent to the current thread for writing. No buffering of these objects is performed,
10	/// allowing for natural back-pressure in case of slow writers.
11	///	5	///
12	/// * `objects`	6	/// This allows objects to be written quite soon without having to wait for the entire pack to be built in memory.
13	/// * A list of objects to add to the pack. Duplication checks are performed so no object is ever added to a pack twice.	7	/// A chunk of objects is held in memory and compressed using DEFLATE, and serve the output of this iterator.
		8	/// That way slow writers will naturally apply back pressure, and communicate to the implementation that more time can be
		9	/// spent compressing objects.
		10	///
		11	/// * `counts`
		12	/// * A list of previously counted objects to add to the pack. Duplication checks are not performed, no object is expected to be duplicated.
14	/// * `progress`	13	/// * `progress`
15	/// * a way to obtain progress information	14	/// * a way to obtain progress information
16	/// * `options`	15	/// * `options`
@@ -32,15 +31,14 @@ use git_hash::{oid, ObjectId}; · 15 unmodified lines
32	/// so with minimal overhead (especially compared to `gixp index-from-pack`)~~ Probably works now by chaining Iterators	31	/// so with minimal overhead (especially compared to `gixp index-from-pack`)~~ Probably works now by chaining Iterators
33	/// or keeping enough state to write a pack and then generate an index with recorded data.	32	/// or keeping enough state to write a pack and then generate an index with recorded data.
34	///	33	///
35	pub fn objects_to_entries_iter<Find, Iter, Oid, Cache>(	34	pub fn objects_to_entries_iter<Find, Cache>(
		35	counts: impl AsRef<[output::Count]>,
36	db: Find,	36	db: Find,
37	make_cache: impl Fn() -> Cache + Send + Clone + Sync + 'static,	37	make_cache: impl Fn() -> Cache + Send + Clone + Sync + 'static,
38	objects: Iter,
39	_progress: impl Progress,	38	_progress: impl Progress,
40	Options {	39	Options {
41	version,	40	version,
42	thread_limit,	41	thread_limit,
43	input_object_expansion,
44	chunk_size,	42	chunk_size,
45	}: Options,	43	}: Options,
46	) -> impl Iterator<Item = Result<Vec<output::Entry>, Error<Find::Error>>>	44	) -> impl Iterator<Item = Result<Vec<output::Entry>, Error<Find::Error>>>
@@ -48,26 +46,16 @@ pub fn objects_to_entries_iter<Find, Iter, Oid, Cache>( · 1 unmodified line
48	where	46	where
49	Find: crate::Find + Clone + Send + Sync + 'static,	47	Find: crate::Find + Clone + Send + Sync + 'static,
50	<Find as crate::Find>::Error: Send,	48	<Find as crate::Find>::Error: Send,
51	Iter: Iterator<Item = Oid> + Send + 'static,
52	Oid: AsRef<oid> + Send + 'static,
53	Cache: pack::cache::DecodeEntry,	49	Cache: pack::cache::DecodeEntry,
54	{	50	{
55	assert!(	51	assert!(
56	matches!(version, pack::data::Version::V2),	52	matches!(version, pack::data::Version::V2),
57	"currently we can only write version 2"	53	"currently we can only write version 2"
58	);	54	);
59	let lower_bound = objects.size_hint().0;	55	let counts = counts.as_ref();
60	let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit(	56	let (chunk_size, thread_limit, _) =
61	chunk_size,	57	parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None);
62	if lower_bound == 0 { None } else { Some(lower_bound) },	58	let chunks = counts.chunks(chunk_size);
63	thread_limit,
64	None,
65	);
66	let chunks = util::Chunks {
67	iter: objects,
68	size: chunk_size,
69	};
70	let seen_objs = std::sync::Arc::new(dashmap::DashSet::<ObjectId>::new());
71		59
72	parallel::reduce::Stepwise::new(	60	parallel::reduce::Stepwise::new(
73	chunks,	61	chunks,
@@ -75,408 +63,34 @@ where · 1 unmodified line
75	move \|_n\| {	63	move \|_n\| {
76	(	64	(
77	Vec::new(), // object data buffer	65	Vec::new(), // object data buffer
78	Vec::new(), // object data buffer 2 to hold two objects at a time
79	make_cache(), // cache to speed up pack operations	66	make_cache(), // cache to speed up pack operations
80	)	67	)
81	},	68	},
82	{	69	move \|counts: &[output::Count], (buf, cache)\| {
83	let seen_objs = std::sync::Arc::clone(&seen_objs);	70	let mut out = Vec::new();
84	move \|oids: Vec<Oid>, (buf1, buf2, cache)\| {	71	for count in counts.into_iter() {
85	use ObjectExpansion::*;	72	out.push(match count.entry_pack_location {
86	let mut out = Vec::new();	73	Some(ref location) => match output::Entry::from_pack_entry(location, count, version) {
87	let mut tree_traversal_state = git_traverse::tree::breadthfirst::State::default();	74	Some(entry) => entry,
88	let mut tree_diff_state = git_diff::tree::State::default();	75	None => {
89	let mut parent_commit_ids = Vec::new();	76	let obj = db.find_existing(count.id, buf, cache)?;
90	let seen_objs = seen_objs.as_ref();	77	output::Entry::from_data(count, &obj)
91	let mut traverse_delegate = tree::traverse::AllUnseen::new(seen_objs);
92	let mut changes_delegate = tree::changes::AllNew::new(seen_objs);
93
94	for id in oids.into_iter() {
95	let id = id.as_ref();
96	let obj = db
97	.find(id, buf1, cache)?
98	.ok_or_else(\|\| Error::NotFound { oid: id.to_owned() })?;
99	match input_object_expansion {
100	TreeAdditionsComparedToAncestor => {
101	use git_object::Kind::*;
102	let mut obj = obj;
103	let mut id = id.to_owned();
104
105	loop {
106	push_obj_entry_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
107	match obj.kind {
108	Tree \| Blob => break,
109	Tag => {
110	id = immutable::TagIter::from_bytes(obj.data)
111	.target_id()
112	.expect("every tag has a target");
113	obj = db
114	.find_existing(id, buf1, cache)
115	.map_err(\|_\| Error::NotFound { oid: id.to_owned() })?;
116	continue;
117	}
118	Commit => {
119	let current_tree_iter = {
120	let mut commit_iter = immutable::CommitIter::from_bytes(obj.data);
121	let tree_id = commit_iter.tree_id().expect("every commit has a tree");
122	parent_commit_ids.clear();
123	for token in commit_iter {
124	match token {
125	Ok(immutable::commit::iter::Token::Parent { id }) => {
126	parent_commit_ids.push(id)
127	}
128	Ok(_) => break,
129	Err(err) => return Err(Error::CommitDecode(err)),
130	}
131	}
132	let obj = db
133	.find_existing(tree_id, buf1, cache)
134	.map_err(\|_\| Error::NotFound { oid: tree_id })?;
135	push_obj_entry_unique(&mut out, seen_objs, &db, version, &tree_id, &obj)?;
136	immutable::TreeIter::from_bytes(obj.data)
137	};
138
139	let objects = if parent_commit_ids.is_empty() {
140	traverse_delegate.clear();
141	git_traverse::tree::breadthfirst(
142	current_tree_iter,
143	&mut tree_traversal_state,
144	\|oid, buf\| db.find_existing_tree_iter(oid, buf, cache).ok(),
145	&mut traverse_delegate,
146	)
147	.map_err(Error::TreeTraverse)?;
148	&traverse_delegate.objects
149	} else {
150	changes_delegate.clear();
151	for commit_id in &parent_commit_ids {
152	let parent_tree_id = {
153	let parent_commit_obj = db
154	.find_existing(commit_id, buf2, cache)
155	.map_err(\|_\| Error::NotFound {
156	oid: commit_id.to_owned(),
157	})?;
158
159	push_obj_entry_unique(
160	&mut out,
161	seen_objs,
162	&db,
163	version,
164	&commit_id,
165	&parent_commit_obj,
166	)?;
167	immutable::CommitIter::from_bytes(parent_commit_obj.data)
168	.tree_id()
169	.expect("every commit has a tree")
170	};
171	let parent_tree = {
172	let parent_tree_obj = db
173	.find_existing(parent_tree_id, buf2, cache)
174	.map_err(\|_\| Error::NotFound { oid: parent_tree_id })?;
175	push_obj_entry_unique(
176	&mut out,
177	seen_objs,
178	&db,
179	version,
180	&parent_tree_id,
181	&parent_tree_obj,
182	)?;
183	immutable::TreeIter::from_bytes(parent_tree_obj.data)
184	};
185
186	git_diff::tree::Changes::from(Some(parent_tree))
187	.needed_to_obtain(
188	current_tree_iter.clone(),
189	&mut tree_diff_state,
190	\|oid, buf\| db.find_existing_tree_iter(oid, buf, cache).ok(),
191	&mut changes_delegate,
192	)
193	.map_err(Error::TreeChanges)?;
194	}
195	&changes_delegate.objects
196	};
197	for id in objects.iter() {
198	let obj = db
199	.find(id, buf2, cache)?
200	.ok_or(Error::NotFound { oid: id.to_owned() })?;
201	out.push(obj_to_entry(&db, version, id, &obj)?);
202	}
203	break;
204	}
205	}
206	}
207	}
208	TreeContents => {
209	use git_object::Kind::*;
210	let mut id: ObjectId = id.into();
211	let mut obj = obj;
212	loop {
213	push_obj_entry_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
214	match obj.kind {
215	Tree => {
216	traverse_delegate.clear();
217	git_traverse::tree::breadthfirst(
218	git_object::immutable::TreeIter::from_bytes(obj.data),
219	&mut tree_traversal_state,
220	\|oid, buf\| db.find_existing_tree_iter(oid, buf, cache).ok(),
221	&mut traverse_delegate,
222	)
223	.map_err(Error::TreeTraverse)?;
224	for id in traverse_delegate.objects.iter() {
225	let obj = db
226	.find(id, buf1, cache)?
227	.ok_or(Error::NotFound { oid: id.to_owned() })?;
228	out.push(obj_to_entry(&db, version, id, &obj)?);
229	}
230	break;
231	}
232	Commit => {
233	id = immutable::CommitIter::from_bytes(obj.data)
234	.tree_id()
235	.expect("every commit has a tree");
236	obj = db
237	.find_existing(id, buf1, cache)
238	.map_err(\|_\| Error::NotFound { oid: id.to_owned() })?;
239	continue;
240	}
241	Blob => break,
242	Tag => {
243	id = immutable::TagIter::from_bytes(obj.data)
244	.target_id()
245	.expect("every tag has a target");
246	obj = db
247	.find_existing(id, buf1, cache)
248	.map_err(\|_\| Error::NotFound { oid: id.to_owned() })?;
249	continue;
250	}
251	}
252	}
253	}	78	}
254	AsIs => push_obj_entry_unique(&mut out, seen_objs, &db, version, id, &obj)?,	79	},
		80	None => {
		81	let obj = db.find_existing(count.id, buf, cache)?;
		82	output::Entry::from_data(count, &obj)
255	}	83	}
256	}	84	}?);
257	Ok(out)
258	}	85	}
		86	Ok(out)
259	},	87	},
260	parallel::reduce::IdentityWithResult::default(),	88	parallel::reduce::IdentityWithResult::default(),
261	)	89	)
262	}	90	}
263		91
264	mod tree {
265	pub mod changes {
266	use dashmap::DashSet;
267	use git_diff::tree::visit::{Action, Change};
268	use git_diff::tree::Visit;
269	use git_hash::bstr::BStr;
270	use git_hash::ObjectId;
271	use std::collections::HashSet;
272
273	pub struct AllNew<'a> {
274	pub objects: HashSet<ObjectId>,
275	all_seen: &'a DashSet<ObjectId>,
276	}
277
278	impl<'a> AllNew<'a> {
279	pub fn new(all_seen: &'a DashSet<ObjectId>) -> Self {
280	AllNew {
281	objects: Default::default(),
282	all_seen,
283	}
284	}
285	pub fn clear(&mut self) {
286	self.objects.clear();
287	}
288	}
289
290	impl<'a> Visit for AllNew<'a> {
291	fn pop_front_tracked_path_and_set_current(&mut self) {}
292
293	fn push_back_tracked_path_component(&mut self, _component: &BStr) {}
294
295	fn push_path_component(&mut self, _component: &BStr) {}
296
297	fn pop_path_component(&mut self) {}
298
299	fn visit(&mut self, change: Change) -> Action {
300	match change {
301	Change::Addition { oid, .. } \| Change::Modification { oid, .. } => {
302	let inserted = self.all_seen.insert(oid);
303	if inserted {
304	self.objects.insert(oid);
305	}
306	}
307	Change::Deletion { .. } => {}
308	};
309	Action::Continue
310	}
311	}
312	}
313	pub mod traverse {
314	use dashmap::DashSet;
315	use git_hash::{bstr::BStr, ObjectId};
316	use git_object::immutable::tree::Entry;
317	use git_traverse::tree::visit::{Action, Visit};
318	use std::collections::HashSet;
319
320	pub struct AllUnseen<'a> {
321	pub objects: HashSet<ObjectId>,
322	all_seen: &'a DashSet<ObjectId>,
323	}
324
325	impl<'a> AllUnseen<'a> {
326	pub fn new(all_seen: &'a DashSet<ObjectId>) -> Self {
327	AllUnseen {
328	objects: Default::default(),
329	all_seen,
330	}
331	}
332	pub fn clear(&mut self) {
333	self.objects.clear();
334	}
335	}
336
337	impl<'a> Visit for AllUnseen<'a> {
338	fn pop_front_tracked_path_and_set_current(&mut self) {}
339
340	fn push_back_tracked_path_component(&mut self, _component: &BStr) {}
341
342	fn push_path_component(&mut self, _component: &BStr) {}
343
344	fn pop_path_component(&mut self) {}
345
346	fn visit_tree(&mut self, entry: &Entry<'_>) -> Action {
347	let inserted = self.all_seen.insert(entry.oid.to_owned());
348	if inserted {
349	self.objects.insert(entry.oid.to_owned());
350	Action::Continue
351	} else {
352	Action::Skip
353	}
354	}
355
356	fn visit_nontree(&mut self, entry: &Entry<'_>) -> Action {
357	let inserted = self.all_seen.insert(entry.oid.to_owned());
358	if inserted {
359	self.objects.insert(entry.oid.to_owned());
360	}
361	Action::Continue
362	}
363	}
364	}
365	}
366
367	fn push_obj_entry_unique<Find>(
368	out: &mut Vec<output::Entry>,
369	all_seen: &DashSet<ObjectId>,
370	db: &Find,
371	version: pack::data::Version,
372	id: &oid,
373	obj: &crate::data::Object<'_>,
374	) -> Result<(), Error<Find::Error>>
375	where
376	Find: crate::Find,
377	{
378	let inserted = all_seen.insert(id.to_owned());
379	if inserted {
380	out.push(obj_to_entry(db, version, id, obj)?);
381	}
382	Ok(())
383	}
384
385	fn obj_to_entry<Find>(
386	db: &Find,
387	version: pack::data::Version,
388	id: &oid,
389	obj: &crate::data::Object<'_>,
390	) -> Result<output::Entry, Error<Find::Error>>
391	where
392	Find: crate::Find,
393	{
394	Ok(match db.pack_entry(&obj) {
395	Some(entry) if entry.version == version => {
396	let pack_entry = pack::data::Entry::from_bytes(entry.data, 0);
397	if let Some(expected) = entry.crc32 {
398	let actual = hash::crc32(entry.data);
399	if actual != expected {
400	return Err(Error::PackToPackCopyCrc32Mismatch { actual, expected });
401	}
402	}
403	if pack_entry.header.is_base() {
404	output::Entry {
405	id: id.to_owned(),
406	object_kind: pack_entry.header.to_kind().expect("non-delta"),
407	kind: output::entry::Kind::Base,
408	decompressed_size: obj.data.len(),
409	compressed_data: entry.data[pack_entry.data_offset as usize..].to_owned(),
410	}
411	} else {
412	output::Entry::from_data(id, &obj).map_err(Error::NewEntry)?
413	}
414	}
415	_ => output::Entry::from_data(id, &obj).map_err(Error::NewEntry)?,
416	})
417	}
418
419	mod util {
420	pub struct Chunks<I> {
421	pub size: usize,
422	pub iter: I,
423	}
424
425	impl<I, Item> Iterator for Chunks<I>
426	where
427	I: Iterator<Item = Item>,
428	{
429	type Item = Vec<Item>;
430
431	fn next(&mut self) -> Option<Self::Item> {
432	let mut res = Vec::with_capacity(self.size);
433	let mut items_left = self.size;
434	while let Some(item) = self.iter.next() {
435	res.push(item);
436	items_left -= 1;
437	if items_left == 0 {
438	break;
439	}
440	}
441	if res.is_empty() {
442	None
443	} else {
444	Some(res)
445	}
446	}
447	}
448	}
449
450	mod types {	92	mod types {
451	use crate::pack::data::output::entry;	93	use crate::{find, pack::data::output::entry};
452	use git_hash::ObjectId;
453
454	/// The way input objects are handled
455	#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
456	#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
457	pub enum ObjectExpansion {
458	/// Don't do anything with the input objects except for transforming them into pack entries
459	AsIs,
460	/// If the input object is a Commit then turn it into a pack entry. Additionally obtain its tree, turn it into a pack entry
461	/// along with all of its contents, that is nested trees, and any other objects reachable from it.
462	/// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs].
463	///
464	/// This mode is useful if all reachable objects should be added, as in cloning a repository.
465	TreeContents,
466	/// If the input is a commit, obtain its ancestors and turn them into pack entries. Obtain the ancestor trees along with the commits
467	/// tree and turn them into pack entries. Finally obtain the added/changed objects when comparing the ancestor trees with the
468	/// current tree and turn them into entries as well.
469	/// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs].
470	///
471	/// This mode is useful to build a pack containing only new objects compared to a previous state.
472	TreeAdditionsComparedToAncestor,
473	}
474
475	impl Default for ObjectExpansion {
476	fn default() -> Self {
477	ObjectExpansion::AsIs
478	}
479	}
480		94
481	/// Configuration options for the pack generation functions provied in [this module][crate::pack::data::output].	95	/// Configuration options for the pack generation functions provied in [this module][crate::pack::data::output].
482	#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]	96	#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
@@ -489,8 +103,6 @@ mod types { · 6 unmodified lines
489	pub chunk_size: usize,	103	pub chunk_size: usize,
490	/// The pack data version to produce	104	/// The pack data version to produce
491	pub version: crate::pack::data::Version,	105	pub version: crate::pack::data::Version,
492	/// The way input objects are handled
493	pub input_object_expansion: ObjectExpansion,
494	}	106	}
495		107
496	impl Default for Options {	108	impl Default for Options {
@@ -499,7 +111,6 @@ mod types { · 2 unmodified lines
499	thread_limit: None,	111	thread_limit: None,
500	chunk_size: 10,	112	chunk_size: 10,
501	version: Default::default(),	113	version: Default::default(),
502	input_object_expansion: Default::default(),
503	}	114	}
504	}	115	}
505	}	116	}
@@ -507,26 +118,14 @@ mod types { · 1 unmodified line
507	/// The error returned by the pack generation function [`to_entry_iter()`][crate::pack::data::output::objects_to_entries_iter()].	118	/// The error returned by the pack generation function [`to_entry_iter()`][crate::pack::data::output::objects_to_entries_iter()].
508	#[derive(Debug, thiserror::Error)]	119	#[derive(Debug, thiserror::Error)]
509	#[allow(missing_docs)]	120	#[allow(missing_docs)]
510	pub enum Error<LocateErr>	121	pub enum Error<FindErr>
511	where	122	where
512	LocateErr: std::error::Error + 'static,	123	FindErr: std::error::Error + 'static,
513	{	124	{
514	#[error(transparent)]	125	#[error(transparent)]
515	CommitDecode(git_object::immutable::object::decode::Error),	126	FindExisting(find::existing_object::Error<FindErr>),
516	#[error(transparent)]
517	Locate(#[from] LocateErr),
518	#[error(transparent)]
519	TreeTraverse(git_traverse::tree::breadthfirst::Error),
520	#[error(transparent)]
521	TreeChanges(git_diff::tree::changes::Error),
522	#[error("Object id {oid} wasn't found in object database")]
523	NotFound { oid: ObjectId },
524	#[error("Entry expected to have hash {expected}, but it had {actual}")]
525	PackToPackCopyCrc32Mismatch { actual: u32, expected: u32 },
526	#[error(transparent)]	127	#[error(transparent)]
527	NewEntry(entry::Error),	128	NewEntry(#[from] entry::Error),
528	}	129	}
529	}	130	}
530	use dashmap::DashSet;	131	pub use types::{Error, Options};
531	use git_object::immutable;
532	pub use types::{Error, ObjectExpansion, Options};

git-odb/tests/odb/pack/data/output.rs+21−3

     linked::Db::at(path).map_err(Into::into).map(Into::into)
 }
-mod count {
+mod count_and_entries {
     use crate::odb::{
         hex_to_id,
         pack::data::output::{db, DbKind},
                 move |oid, buf| db.find_existing_commit_iter(oid, buf, &mut pack::cache::Never).ok()
             })
             .map(Result::unwrap);
-            let entries: Vec<_> = output::count_objects_iter(
+            let counts: Vec<_> = output::count_objects_iter(
                 db.clone(),
                 || pack::cache::Never,
                 commits.chain(std::iter::once(hex_to_id("e3fb53cbb4c346d48732a24f09cf445e49bc63d6"))),
             .into_iter()
             .flatten()
             .collect();
+            let actual_count = counts.iter().fold(Count::default(), |mut c, e| {
+                c.add(e.object_kind);
+                c
+            });
+            assert_eq!(actual_count, expected_count);
+            assert_eq!(counts.len(), expected_count.total());
+            let entries: Vec<_> = output::objects_to_entries_iter(
+                counts,
+                db.clone(),
+                || pack::cache::Never,
+                progress::Discard,
+                Default::default(),
+            )
+            .collect::<Result<Vec<_>, _>>()?
+            .into_iter()
+            .flatten()
+            .collect();
             let actual_count = entries.iter().fold(Count::default(), |mut c, e| {
                 c.add(e.object_kind);
                 c
             });
             assert_eq!(actual_count, expected_count);
-            assert_eq!(entries.len(), expected_count.total());
+            assert_eq!(counts.len(), expected_count.total());
         }
         Ok(())
     }

gitoxide-core/src/pack/create.rs+17−16

@@ -37,9 +37,9 @@ impl FromStr for ObjectExpansion { · 36 unmodified lines
37	}	37	}
38	}	38	}
39		39
40	impl From<ObjectExpansion> for pack::data::output::objects_to_entries::ObjectExpansion {	40	impl From<ObjectExpansion> for pack::data::output::count_objects::ObjectExpansion {
41	fn from(v: ObjectExpansion) -> Self {	41	fn from(v: ObjectExpansion) -> Self {
42	use pack::data::output::objects_to_entries::ObjectExpansion::*;	42	use pack::data::output::count_objects::ObjectExpansion::*;
43	match v {	43	match v {
44	ObjectExpansion::None => AsIs,	44	ObjectExpansion::None => AsIs,
45	ObjectExpansion::TreeTraversal => TreeContents,	45	ObjectExpansion::TreeTraversal => TreeContents,
@@ -86,30 +86,31 @@ pub fn create( · 40 unmodified lines
86	.and_then(\|hex_id\| git_hash::ObjectId::from_hex(hex_id.as_bytes()).ok())	86	.and_then(\|hex_id\| git_hash::ObjectId::from_hex(hex_id.as_bytes()).ok())
87	})),	87	})),
88	};	88	};
89	let entries = pack::data::output::objects_to_entries_iter(	89	let entries = pack::data::output::count_objects_iter(
90	Arc::clone(&db),	90	Arc::clone(&db),
91	pack::cache::lru::StaticLinkedList::<64>::default,	91	pack::cache::lru::StaticLinkedList::<64>::default,
92	input,	92	input,
93	git_features::progress::Discard,	93	git_features::progress::Discard,
94	pack::data::output::objects_to_entries::Options {	94	pack::data::output::count_objects::Options {
95	thread_limit: ctx.thread_limit,	95	thread_limit: ctx.thread_limit,
96	chunk_size: 200,	96	chunk_size: 200,
97	version: Default::default(),	97	version: Default::default(),
98	input_object_expansion: ctx.expansion.into(),	98	input_object_expansion: ctx.expansion.into(),
99	},	99	},
100	);	100	);
101	let mut output_iter = pack::data::output::EntriesToBytesIter::new(	101	todo!("entries from counts");
102	entries,	102	// let mut output_iter = pack::data::output::EntriesToBytesIter::new(
103	out,	103	// entries,
104	0,	104	// out,
105	pack::data::Version::default(),	105	// 0,
106	git_hash::Kind::default(),	106	// pack::data::Version::default(),
107	);	107	// git_hash::Kind::default(),
108	while let Some(io_res) = output_iter.next() {	108	// );
109	let _written = io_res?;	109	// while let Some(io_res) = output_iter.next() {
110	}	110	// let _written = io_res?;
111	output_iter.into_write().flush()?;	111	// }
112	Ok(())	112	// output_iter.into_write().flush()?;
		113	// Ok(())
113	}	114	}
114		115
115	fn find_db(repository: impl AsRef<Path>) -> anyhow::Result<linked::Db> {	116	fn find_db(repository: impl AsRef<Path>) -> anyhow::Result<linked::Db> {