History

[pack-gen] very close to a basic impl of count + entries-gen…

…but now the type system plays tricks on me and it's kind of in the way.

Sebastian Thiel committed 5 years ago

Commit c9274291 parent 932b4396 files · +135 −517Browse files

Unified · Split

6 files
  1. git-odb/src/pack/data/output/count_objects.rs +21 −40
  2. git-odb/src/pack/data/output/entry.rs +36 −4
  3. git-odb/src/pack/data/output/mod.rs +4 −17
  4. git-odb/src/pack/data/output/objects_to_entries.rs +36 −437
  5. git-odb/tests/odb/pack/data/output.rs +21 −3
  6. gitoxide-core/src/pack/create.rs +17 −16
git-odb/src/pack/data/output/count_objects.rs+21−40
@@ -1,6 +1,6 @@
11 use crate::{pack, pack::data::output, FindExt};
22 use dashmap::DashSet;
3−use git_features::{hash, parallel, progress::Progress};
3+use git_features::{parallel, progress::Progress};
44 use git_hash::{oid, ObjectId};
55 use git_object::immutable;
66
@@ -84,7 +84,7 @@ where · 77 unmodified lines
8484 let mut id = id.to_owned();
8585
8686 loop {
87− push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
87+ push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj);
8888 match obj.kind {
8989 Tree | Blob => break,
9090 Tag => {
@@ -113,7 +113,7 @@ where · 22 unmodified lines
113113 let obj = db
114114 .find_existing(tree_id, buf1, cache)
115115 .map_err(|_| Error::NotFound { oid: tree_id })?;
116− push_obj_count_unique(&mut out, seen_objs, &db, version, &tree_id, &obj)?;
116+ push_obj_count_unique(&mut out, seen_objs, &db, version, &tree_id, &obj);
117117 immutable::TreeIter::from_bytes(obj.data)
118118 };
119119
@@ -144,7 +144,7 @@ where · 24 unmodified lines
144144 version,
145145 &commit_id,
146146 &parent_commit_obj,
147− )?;
147+ );
148148 immutable::CommitIter::from_bytes(parent_commit_obj.data)
149149 .tree_id()
150150 .expect("every commit has a tree")
@@ -160,7 +160,7 @@ where · 9 unmodified lines
160160 version,
161161 &parent_tree_id,
162162 &parent_tree_obj,
163− )?;
163+ );
164164 immutable::TreeIter::from_bytes(parent_tree_obj.data)
165165 };
166166
@@ -179,7 +179,7 @@ where · 12 unmodified lines
179179 let obj = db
180180 .find(id, buf2, cache)?
181181 .ok_or(Error::NotFound { oid: id.to_owned() })?;
182− out.push(obj_to_count(&db, version, id, &obj)?);
182+ out.push(obj_to_count(&db, version, id, &obj));
183183 }
184184 break;
185185 }
@@ -191,7 +191,7 @@ where · 5 unmodified lines
191191 let mut id: ObjectId = id.into();
192192 let mut obj = obj;
193193 loop {
194− push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
194+ push_obj_count_unique(&mut out, seen_objs, &db, version, &id, &obj);
195195 match obj.kind {
196196 Tree => {
197197 traverse_delegate.clear();
@@ -206,7 +206,7 @@ where · 8 unmodified lines
206206 let obj = db
207207 .find(id, buf1, cache)?
208208 .ok_or(Error::NotFound { oid: id.to_owned() })?;
209− out.push(obj_to_count(&db, version, id, &obj)?);
209+ out.push(obj_to_count(&db, version, id, &obj));
210210 }
211211 break;
212212 }
@@ -232,7 +232,7 @@ where · 19 unmodified lines
232232 }
233233 }
234234 }
235− AsIs => push_obj_count_unique(&mut out, seen_objs, &db, version, id, &obj)?,
235+ AsIs => push_obj_count_unique(&mut out, seen_objs, &db, version, id, &obj),
236236 }
237237 }
238238 Ok(out)
@@ -346,56 +346,42 @@ mod tree { · 107 unmodified lines
346346 }
347347 }
348348
349−fn push_obj_count_unique<Find>(
349+fn push_obj_count_unique(
350350 out: &mut Vec<output::Count>,
351351 all_seen: &DashSet<ObjectId>,
352− db: &Find,
352+ db: &impl crate::Find,
353353 version: pack::data::Version,
354354 id: &oid,
355355 obj: &crate::data::Object<'_>,
356−) -> Result<(), Error<Find::Error>>
357−where
358− Find: crate::Find,
359−{
356+) {
360357 let inserted = all_seen.insert(id.to_owned());
361358 if inserted {
362− out.push(obj_to_count(db, version, id, obj)?);
359+ out.push(obj_to_count(db, version, id, obj));
363360 }
364− Ok(())
365361 }
366362
367−fn obj_to_count<Find>(
368− db: &Find,
363+fn obj_to_count(
364+ db: &impl crate::Find,
369365 version: pack::data::Version,
370366 id: &oid,
371367 obj: &crate::data::Object<'_>,
372−) -> Result<output::Count, Error<Find::Error>>
373−where
374− Find: crate::Find,
375−{
376− Ok(match db.pack_entry(&obj) {
368+) -> output::Count {
369+ match db.pack_entry(&obj) {
377370 Some(entry) if entry.version == version => {
378371 let pack_entry = pack::data::Entry::from_bytes(entry.data, 0);
379− if let Some(expected) = entry.crc32 {
380− let actual = hash::crc32(entry.data);
381− if actual != expected {
382− return Err(Error::PackToPackCopyCrc32Mismatch { actual, expected });
383− }
384− }
385372 if pack_entry.header.is_base() {
386373 output::Count {
387374 id: id.to_owned(),
388375 object_kind: pack_entry.header.to_kind().expect("non-delta"),
389− entry_kind: output::entry::Kind::Base,
390376 decompressed_size: obj.data.len(),
391377 entry_pack_location: obj.pack_location.clone(),
392378 }
393379 } else {
394− output::Count::from_data(id, &obj).map_err(Error::NewEntry)?
380+ output::Count::from_data(id, &obj)
395381 }
396382 }
397− _ => output::Count::from_data(id, &obj).map_err(Error::NewEntry)?,
398− })
383+ _ => output::Count::from_data(id, &obj),
384+ }
399385 }
400386
401387 mod util {
@@ -430,7 +416,6 @@ mod util { · 28 unmodified lines
430416 }
431417
432418 mod types {
433− use crate::pack::data::output::count;
434419 use git_hash::ObjectId;
435420
436421 /// The way input objects are handled
@@ -496,17 +481,13 @@ mod types { · 59 unmodified lines
496481 #[error(transparent)]
497482 CommitDecode(git_object::immutable::object::decode::Error),
498483 #[error(transparent)]
499− Locate(#[from] LocateErr),
484+ Find(#[from] LocateErr),
500485 #[error(transparent)]
501486 TreeTraverse(git_traverse::tree::breadthfirst::Error),
502487 #[error(transparent)]
503488 TreeChanges(git_diff::tree::changes::Error),
504489 #[error("Object id {oid} wasn't found in object database")]
505490 NotFound { oid: ObjectId },
506− #[error("Entry expected to have hash {expected}, but it had {actual}")]
507− PackToPackCopyCrc32Mismatch { actual: u32, expected: u32 },
508− #[error(transparent)]
509− NewEntry(count::Error),
510491 }
511492 }
512493 pub use types::{Error, ObjectExpansion, Options};
git-odb/src/pack/data/output/entry.rs+36−4
@@ -1,7 +1,8 @@
11 use crate::{
2− data,
2+ data, find,
33 pack::{self, data::output},
44 };
5+use git_features::hash;
56 use git_hash::ObjectId;
67 use std::io::Write;
78
@@ -52,14 +53,45 @@ pub enum Kind { · 44 unmodified lines
5253 pub enum Error {
5354 #[error("{0}")]
5455 ZlibDeflate(#[from] std::io::Error),
56+ #[error("Entry expected to have hash {expected}, but it had {actual}")]
57+ PackToPackCopyCrc32Mismatch { actual: u32, expected: u32 },
5558 }
5659
5760 impl output::Entry {
61+ pub fn from_pack_entry(
62+ entry: find::PackEntry<'_>,
63+ count: &output::Count,
64+ version: pack::data::Version,
65+ ) -> Option<Result<Self, Error>> {
66+ if entry.version != version {
67+ return None;
68+ };
69+
70+ let pack_entry = pack::data::Entry::from_bytes(entry.data, 0);
71+ if let Some(expected) = entry.crc32 {
72+ let actual = hash::crc32(entry.data);
73+ if actual != expected {
74+ return Some(Err(Error::PackToPackCopyCrc32Mismatch { actual, expected }));
75+ }
76+ }
77+ if pack_entry.header.is_base() {
78+ Some(Ok(output::Entry {
79+ id: count.id.to_owned(),
80+ object_kind: count.object_kind,
81+ kind: output::entry::Kind::Base,
82+ decompressed_size: count.decompressed_size,
83+ compressed_data: entry.data[pack_entry.data_offset as usize..].to_owned(),
84+ }))
85+ } else {
86+ None
87+ }
88+ }
89+
5890 /// Create a new instance from the given `oid` and its corresponding git `obj`ect data.
59− pub fn from_data(oid: impl Into<ObjectId>, obj: &data::Object<'_>) -> Result<Self, Error> {
91+ pub fn from_data(count: &output::Count, obj: &data::Object<'_>) -> Result<Self, Error> {
6092 Ok(output::Entry {
61− id: oid.into(),
62− object_kind: obj.kind,
93+ id: count.id.to_owned(),
94+ object_kind: count.object_kind,
6395 kind: Kind::Base,
6496 decompressed_size: obj.data.len(),
6597 compressed_data: {
git-odb/src/pack/data/output/mod.rs+4−17
@@ -1,5 +1,5 @@
11 mod count {
2− use crate::{data, pack, pack::data::output::entry};
2+ use crate::{data, pack};
33 use git_hash::ObjectId;
44
55 /// An item representing a future Entry in the leanest way possible.
@@ -15,32 +15,19 @@ mod count { · 9 unmodified lines
1515 pub object_kind: git_object::Kind,
1616 /// The size in bytes needed once `data` gets decompressed
1717 pub decompressed_size: usize,
18−
19− /// The kind of entry represented by `data`. It's used alongside with it to complete the pack entry
20− /// at rest or in transit.
21− pub entry_kind: entry::Kind,
2218 /// A way to locate a pack entry in the object database, only available if the object is in a pack.
2319 pub(crate) entry_pack_location: Option<pack::bundle::Location>,
2420 }
2521
26− /// The error returned by [`output::Entry::from_data()`].
27− #[allow(missing_docs)]
28− #[derive(Debug, thiserror::Error)]
29− pub enum Error {
30− #[error("{0}")]
31− ZlibDeflate(#[from] std::io::Error),
32− }
33−
3422 impl Count {
3523 /// Create a new instance from the given `oid` and its corresponding git `obj`ect data.
36− pub fn from_data(oid: impl Into<ObjectId>, obj: &data::Object<'_>) -> Result<Self, Error> {
37− Ok(Count {
24+ pub fn from_data(oid: impl Into<ObjectId>, obj: &data::Object<'_>) -> Self {
25+ Count {
3826 id: oid.into(),
3927 object_kind: obj.kind,
40− entry_kind: entry::Kind::Base,
4128 decompressed_size: obj.data.len(),
4229 entry_pack_location: obj.pack_location.clone(),
43− })
30+ }
4431 }
4532 }
4633 }
git-odb/src/pack/data/output/objects_to_entries.rs+36−437
@@ -1,16 +1,15 @@
11 use crate::{pack, pack::data::output, FindExt};
2−use git_features::{hash, parallel, progress::Progress};
3−use git_hash::{oid, ObjectId};
2+use git_features::{parallel, progress::Progress};
43
5−/// Write all `objects` into `out` without attempting to apply any delta compression.
6−/// TODO: Update this
7−/// This allows objects to be written rather immediately.
8−/// Objects are held in memory and compressed using DEFLATE, with those in-flight chunks of compressed
9−/// objects being sent to the current thread for writing. No buffering of these objects is performed,
10−/// allowing for natural back-pressure in case of slow writers.
4+/// Given a known list of object `counts`, calculate entries ready to be put into a data pack.
115 ///
12−/// * `objects`
13−/// * A list of objects to add to the pack. Duplication checks are performed so no object is ever added to a pack twice.
6+/// This allows objects to be written quite soon without having to wait for the entire pack to be built in memory.
7+/// A chunk of objects is held in memory and compressed using DEFLATE, and serve the output of this iterator.
8+/// That way slow writers will naturally apply back pressure, and communicate to the implementation that more time can be
9+/// spent compressing objects.
10+///
11+/// * `counts`
12+/// * A list of previously counted objects to add to the pack. Duplication checks are not performed, no object is expected to be duplicated.
1413 /// * `progress`
1514 /// * a way to obtain progress information
1615 /// * `options`
@@ -32,15 +31,14 @@ use git_hash::{oid, ObjectId}; · 15 unmodified lines
3231 /// so with minimal overhead (especially compared to `gixp index-from-pack`)~~ Probably works now by chaining Iterators
3332 /// or keeping enough state to write a pack and then generate an index with recorded data.
3433 ///
35−pub fn objects_to_entries_iter<Find, Iter, Oid, Cache>(
34+pub fn objects_to_entries_iter<Find, Cache>(
35+ counts: impl AsRef<[output::Count]>,
3636 db: Find,
3737 make_cache: impl Fn() -> Cache + Send + Clone + Sync + 'static,
38− objects: Iter,
3938 _progress: impl Progress,
4039 Options {
4140 version,
4241 thread_limit,
43− input_object_expansion,
4442 chunk_size,
4543 }: Options,
4644 ) -> impl Iterator<Item = Result<Vec<output::Entry>, Error<Find::Error>>>
@@ -48,26 +46,16 @@ pub fn objects_to_entries_iter<Find, Iter, Oid, Cache>( · 1 unmodified line
4846 where
4947 Find: crate::Find + Clone + Send + Sync + 'static,
5048 <Find as crate::Find>::Error: Send,
51− Iter: Iterator<Item = Oid> + Send + 'static,
52− Oid: AsRef<oid> + Send + 'static,
5349 Cache: pack::cache::DecodeEntry,
5450 {
5551 assert!(
5652 matches!(version, pack::data::Version::V2),
5753 "currently we can only write version 2"
5854 );
59− let lower_bound = objects.size_hint().0;
60− let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit(
61− chunk_size,
62− if lower_bound == 0 { None } else { Some(lower_bound) },
63− thread_limit,
64− None,
65− );
66− let chunks = util::Chunks {
67− iter: objects,
68− size: chunk_size,
69− };
70− let seen_objs = std::sync::Arc::new(dashmap::DashSet::<ObjectId>::new());
55+ let counts = counts.as_ref();
56+ let (chunk_size, thread_limit, _) =
57+ parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None);
58+ let chunks = counts.chunks(chunk_size);
7159
7260 parallel::reduce::Stepwise::new(
7361 chunks,
@@ -75,408 +63,34 @@ where · 1 unmodified line
7563 move |_n| {
7664 (
7765 Vec::new(), // object data buffer
78− Vec::new(), // object data buffer 2 to hold two objects at a time
7966 make_cache(), // cache to speed up pack operations
8067 )
8168 },
82− {
83− let seen_objs = std::sync::Arc::clone(&seen_objs);
84− move |oids: Vec<Oid>, (buf1, buf2, cache)| {
85− use ObjectExpansion::*;
86− let mut out = Vec::new();
87− let mut tree_traversal_state = git_traverse::tree::breadthfirst::State::default();
88− let mut tree_diff_state = git_diff::tree::State::default();
89− let mut parent_commit_ids = Vec::new();
90− let seen_objs = seen_objs.as_ref();
91− let mut traverse_delegate = tree::traverse::AllUnseen::new(seen_objs);
92− let mut changes_delegate = tree::changes::AllNew::new(seen_objs);
93−
94− for id in oids.into_iter() {
95− let id = id.as_ref();
96− let obj = db
97− .find(id, buf1, cache)?
98− .ok_or_else(|| Error::NotFound { oid: id.to_owned() })?;
99− match input_object_expansion {
100− TreeAdditionsComparedToAncestor => {
101− use git_object::Kind::*;
102− let mut obj = obj;
103− let mut id = id.to_owned();
104−
105− loop {
106− push_obj_entry_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
107− match obj.kind {
108− Tree | Blob => break,
109− Tag => {
110− id = immutable::TagIter::from_bytes(obj.data)
111− .target_id()
112− .expect("every tag has a target");
113− obj = db
114− .find_existing(id, buf1, cache)
115− .map_err(|_| Error::NotFound { oid: id.to_owned() })?;
116− continue;
117− }
118− Commit => {
119− let current_tree_iter = {
120− let mut commit_iter = immutable::CommitIter::from_bytes(obj.data);
121− let tree_id = commit_iter.tree_id().expect("every commit has a tree");
122− parent_commit_ids.clear();
123− for token in commit_iter {
124− match token {
125− Ok(immutable::commit::iter::Token::Parent { id }) => {
126− parent_commit_ids.push(id)
127− }
128− Ok(_) => break,
129− Err(err) => return Err(Error::CommitDecode(err)),
130− }
131− }
132− let obj = db
133− .find_existing(tree_id, buf1, cache)
134− .map_err(|_| Error::NotFound { oid: tree_id })?;
135− push_obj_entry_unique(&mut out, seen_objs, &db, version, &tree_id, &obj)?;
136− immutable::TreeIter::from_bytes(obj.data)
137− };
138−
139− let objects = if parent_commit_ids.is_empty() {
140− traverse_delegate.clear();
141− git_traverse::tree::breadthfirst(
142− current_tree_iter,
143− &mut tree_traversal_state,
144− |oid, buf| db.find_existing_tree_iter(oid, buf, cache).ok(),
145− &mut traverse_delegate,
146− )
147− .map_err(Error::TreeTraverse)?;
148− &traverse_delegate.objects
149− } else {
150− changes_delegate.clear();
151− for commit_id in &parent_commit_ids {
152− let parent_tree_id = {
153− let parent_commit_obj = db
154− .find_existing(commit_id, buf2, cache)
155− .map_err(|_| Error::NotFound {
156− oid: commit_id.to_owned(),
157− })?;
158−
159− push_obj_entry_unique(
160− &mut out,
161− seen_objs,
162− &db,
163− version,
164− &commit_id,
165− &parent_commit_obj,
166− )?;
167− immutable::CommitIter::from_bytes(parent_commit_obj.data)
168− .tree_id()
169− .expect("every commit has a tree")
170− };
171− let parent_tree = {
172− let parent_tree_obj = db
173− .find_existing(parent_tree_id, buf2, cache)
174− .map_err(|_| Error::NotFound { oid: parent_tree_id })?;
175− push_obj_entry_unique(
176− &mut out,
177− seen_objs,
178− &db,
179− version,
180− &parent_tree_id,
181− &parent_tree_obj,
182− )?;
183− immutable::TreeIter::from_bytes(parent_tree_obj.data)
184− };
185−
186− git_diff::tree::Changes::from(Some(parent_tree))
187− .needed_to_obtain(
188− current_tree_iter.clone(),
189− &mut tree_diff_state,
190− |oid, buf| db.find_existing_tree_iter(oid, buf, cache).ok(),
191− &mut changes_delegate,
192− )
193− .map_err(Error::TreeChanges)?;
194− }
195− &changes_delegate.objects
196− };
197− for id in objects.iter() {
198− let obj = db
199− .find(id, buf2, cache)?
200− .ok_or(Error::NotFound { oid: id.to_owned() })?;
201− out.push(obj_to_entry(&db, version, id, &obj)?);
202− }
203− break;
204− }
205− }
206− }
207− }
208− TreeContents => {
209− use git_object::Kind::*;
210− let mut id: ObjectId = id.into();
211− let mut obj = obj;
212− loop {
213− push_obj_entry_unique(&mut out, seen_objs, &db, version, &id, &obj)?;
214− match obj.kind {
215− Tree => {
216− traverse_delegate.clear();
217− git_traverse::tree::breadthfirst(
218− git_object::immutable::TreeIter::from_bytes(obj.data),
219− &mut tree_traversal_state,
220− |oid, buf| db.find_existing_tree_iter(oid, buf, cache).ok(),
221− &mut traverse_delegate,
222− )
223− .map_err(Error::TreeTraverse)?;
224− for id in traverse_delegate.objects.iter() {
225− let obj = db
226− .find(id, buf1, cache)?
227− .ok_or(Error::NotFound { oid: id.to_owned() })?;
228− out.push(obj_to_entry(&db, version, id, &obj)?);
229− }
230− break;
231− }
232− Commit => {
233− id = immutable::CommitIter::from_bytes(obj.data)
234− .tree_id()
235− .expect("every commit has a tree");
236− obj = db
237− .find_existing(id, buf1, cache)
238− .map_err(|_| Error::NotFound { oid: id.to_owned() })?;
239− continue;
240− }
241− Blob => break,
242− Tag => {
243− id = immutable::TagIter::from_bytes(obj.data)
244− .target_id()
245− .expect("every tag has a target");
246− obj = db
247− .find_existing(id, buf1, cache)
248− .map_err(|_| Error::NotFound { oid: id.to_owned() })?;
249− continue;
250− }
251− }
252− }
69+ move |counts: &[output::Count], (buf, cache)| {
70+ let mut out = Vec::new();
71+ for count in counts.into_iter() {
72+ out.push(match count.entry_pack_location {
73+ Some(ref location) => match output::Entry::from_pack_entry(location, count, version) {
74+ Some(entry) => entry,
75+ None => {
76+ let obj = db.find_existing(count.id, buf, cache)?;
77+ output::Entry::from_data(count, &obj)
25378 }
254− AsIs => push_obj_entry_unique(&mut out, seen_objs, &db, version, id, &obj)?,
79+ },
80+ None => {
81+ let obj = db.find_existing(count.id, buf, cache)?;
82+ output::Entry::from_data(count, &obj)
25583 }
256− }
257− Ok(out)
84+ }?);
25885 }
86+ Ok(out)
25987 },
26088 parallel::reduce::IdentityWithResult::default(),
26189 )
26290 }
26391
264−mod tree {
265− pub mod changes {
266− use dashmap::DashSet;
267− use git_diff::tree::visit::{Action, Change};
268− use git_diff::tree::Visit;
269− use git_hash::bstr::BStr;
270− use git_hash::ObjectId;
271− use std::collections::HashSet;
272−
273− pub struct AllNew<'a> {
274− pub objects: HashSet<ObjectId>,
275− all_seen: &'a DashSet<ObjectId>,
276− }
277−
278− impl<'a> AllNew<'a> {
279− pub fn new(all_seen: &'a DashSet<ObjectId>) -> Self {
280− AllNew {
281− objects: Default::default(),
282− all_seen,
283− }
284− }
285− pub fn clear(&mut self) {
286− self.objects.clear();
287− }
288− }
289−
290− impl<'a> Visit for AllNew<'a> {
291− fn pop_front_tracked_path_and_set_current(&mut self) {}
292−
293− fn push_back_tracked_path_component(&mut self, _component: &BStr) {}
294−
295− fn push_path_component(&mut self, _component: &BStr) {}
296−
297− fn pop_path_component(&mut self) {}
298−
299− fn visit(&mut self, change: Change) -> Action {
300− match change {
301− Change::Addition { oid, .. } | Change::Modification { oid, .. } => {
302− let inserted = self.all_seen.insert(oid);
303− if inserted {
304− self.objects.insert(oid);
305− }
306− }
307− Change::Deletion { .. } => {}
308− };
309− Action::Continue
310− }
311− }
312− }
313− pub mod traverse {
314− use dashmap::DashSet;
315− use git_hash::{bstr::BStr, ObjectId};
316− use git_object::immutable::tree::Entry;
317− use git_traverse::tree::visit::{Action, Visit};
318− use std::collections::HashSet;
319−
320− pub struct AllUnseen<'a> {
321− pub objects: HashSet<ObjectId>,
322− all_seen: &'a DashSet<ObjectId>,
323− }
324−
325− impl<'a> AllUnseen<'a> {
326− pub fn new(all_seen: &'a DashSet<ObjectId>) -> Self {
327− AllUnseen {
328− objects: Default::default(),
329− all_seen,
330− }
331− }
332− pub fn clear(&mut self) {
333− self.objects.clear();
334− }
335− }
336−
337− impl<'a> Visit for AllUnseen<'a> {
338− fn pop_front_tracked_path_and_set_current(&mut self) {}
339−
340− fn push_back_tracked_path_component(&mut self, _component: &BStr) {}
341−
342− fn push_path_component(&mut self, _component: &BStr) {}
343−
344− fn pop_path_component(&mut self) {}
345−
346− fn visit_tree(&mut self, entry: &Entry<'_>) -> Action {
347− let inserted = self.all_seen.insert(entry.oid.to_owned());
348− if inserted {
349− self.objects.insert(entry.oid.to_owned());
350− Action::Continue
351− } else {
352− Action::Skip
353− }
354− }
355−
356− fn visit_nontree(&mut self, entry: &Entry<'_>) -> Action {
357− let inserted = self.all_seen.insert(entry.oid.to_owned());
358− if inserted {
359− self.objects.insert(entry.oid.to_owned());
360− }
361− Action::Continue
362− }
363− }
364− }
365−}
366−
367−fn push_obj_entry_unique<Find>(
368− out: &mut Vec<output::Entry>,
369− all_seen: &DashSet<ObjectId>,
370− db: &Find,
371− version: pack::data::Version,
372− id: &oid,
373− obj: &crate::data::Object<'_>,
374−) -> Result<(), Error<Find::Error>>
375−where
376− Find: crate::Find,
377−{
378− let inserted = all_seen.insert(id.to_owned());
379− if inserted {
380− out.push(obj_to_entry(db, version, id, obj)?);
381− }
382− Ok(())
383−}
384−
385−fn obj_to_entry<Find>(
386− db: &Find,
387− version: pack::data::Version,
388− id: &oid,
389− obj: &crate::data::Object<'_>,
390−) -> Result<output::Entry, Error<Find::Error>>
391−where
392− Find: crate::Find,
393−{
394− Ok(match db.pack_entry(&obj) {
395− Some(entry) if entry.version == version => {
396− let pack_entry = pack::data::Entry::from_bytes(entry.data, 0);
397− if let Some(expected) = entry.crc32 {
398− let actual = hash::crc32(entry.data);
399− if actual != expected {
400− return Err(Error::PackToPackCopyCrc32Mismatch { actual, expected });
401− }
402− }
403− if pack_entry.header.is_base() {
404− output::Entry {
405− id: id.to_owned(),
406− object_kind: pack_entry.header.to_kind().expect("non-delta"),
407− kind: output::entry::Kind::Base,
408− decompressed_size: obj.data.len(),
409− compressed_data: entry.data[pack_entry.data_offset as usize..].to_owned(),
410− }
411− } else {
412− output::Entry::from_data(id, &obj).map_err(Error::NewEntry)?
413− }
414− }
415− _ => output::Entry::from_data(id, &obj).map_err(Error::NewEntry)?,
416− })
417−}
418−
419−mod util {
420− pub struct Chunks<I> {
421− pub size: usize,
422− pub iter: I,
423− }
424−
425− impl<I, Item> Iterator for Chunks<I>
426− where
427− I: Iterator<Item = Item>,
428− {
429− type Item = Vec<Item>;
430−
431− fn next(&mut self) -> Option<Self::Item> {
432− let mut res = Vec::with_capacity(self.size);
433− let mut items_left = self.size;
434− while let Some(item) = self.iter.next() {
435− res.push(item);
436− items_left -= 1;
437− if items_left == 0 {
438− break;
439− }
440− }
441− if res.is_empty() {
442− None
443− } else {
444− Some(res)
445− }
446− }
447− }
448−}
449−
45092 mod types {
451− use crate::pack::data::output::entry;
452− use git_hash::ObjectId;
453−
454− /// The way input objects are handled
455− #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
456− #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
457− pub enum ObjectExpansion {
458− /// Don't do anything with the input objects except for transforming them into pack entries
459− AsIs,
460− /// If the input object is a Commit then turn it into a pack entry. Additionally obtain its tree, turn it into a pack entry
461− /// along with all of its contents, that is nested trees, and any other objects reachable from it.
462− /// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs].
463− ///
464− /// This mode is useful if all reachable objects should be added, as in cloning a repository.
465− TreeContents,
466− /// If the input is a commit, obtain its ancestors and turn them into pack entries. Obtain the ancestor trees along with the commits
467− /// tree and turn them into pack entries. Finally obtain the added/changed objects when comparing the ancestor trees with the
468− /// current tree and turn them into entries as well.
469− /// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs].
470− ///
471− /// This mode is useful to build a pack containing only new objects compared to a previous state.
472− TreeAdditionsComparedToAncestor,
473− }
474−
475− impl Default for ObjectExpansion {
476− fn default() -> Self {
477− ObjectExpansion::AsIs
478− }
479− }
93+ use crate::{find, pack::data::output::entry};
48094
48195 /// Configuration options for the pack generation functions provied in [this module][crate::pack::data::output].
48296 #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
@@ -489,8 +103,6 @@ mod types { · 6 unmodified lines
489103 pub chunk_size: usize,
490104 /// The pack data version to produce
491105 pub version: crate::pack::data::Version,
492− /// The way input objects are handled
493− pub input_object_expansion: ObjectExpansion,
494106 }
495107
496108 impl Default for Options {
@@ -499,7 +111,6 @@ mod types { · 2 unmodified lines
499111 thread_limit: None,
500112 chunk_size: 10,
501113 version: Default::default(),
502− input_object_expansion: Default::default(),
503114 }
504115 }
505116 }
@@ -507,26 +118,14 @@ mod types { · 1 unmodified line
507118 /// The error returned by the pack generation function [`to_entry_iter()`][crate::pack::data::output::objects_to_entries_iter()].
508119 #[derive(Debug, thiserror::Error)]
509120 #[allow(missing_docs)]
510− pub enum Error<LocateErr>
121+ pub enum Error<FindErr>
511122 where
512− LocateErr: std::error::Error + 'static,
123+ FindErr: std::error::Error + 'static,
513124 {
514125 #[error(transparent)]
515− CommitDecode(git_object::immutable::object::decode::Error),
516− #[error(transparent)]
517− Locate(#[from] LocateErr),
518− #[error(transparent)]
519− TreeTraverse(git_traverse::tree::breadthfirst::Error),
520− #[error(transparent)]
521− TreeChanges(git_diff::tree::changes::Error),
522− #[error("Object id {oid} wasn't found in object database")]
523− NotFound { oid: ObjectId },
524− #[error("Entry expected to have hash {expected}, but it had {actual}")]
525− PackToPackCopyCrc32Mismatch { actual: u32, expected: u32 },
126+ FindExisting(find::existing_object::Error<FindErr>),
526127 #[error(transparent)]
527− NewEntry(entry::Error),
128+ NewEntry(#[from] entry::Error),
528129 }
529130 }
530−use dashmap::DashSet;
531−use git_object::immutable;
532−pub use types::{Error, ObjectExpansion, Options};
131+pub use types::{Error, Options};
git-odb/tests/odb/pack/data/output.rs+21−3
@@ -36,7 +36,7 @@ fn db(kind: DbKind) -> crate::Result<Arc<linked::Db>> { · 35 unmodified lines
3636 linked::Db::at(path).map_err(Into::into).map(Into::into)
3737 }
3838
39−mod count {
39+mod count_and_entries {
4040 use crate::odb::{
4141 hex_to_id,
4242 pack::data::output::{db, DbKind},
@@ -101,7 +101,7 @@ mod count { · 58 unmodified lines
101101 move |oid, buf| db.find_existing_commit_iter(oid, buf, &mut pack::cache::Never).ok()
102102 })
103103 .map(Result::unwrap);
104− let entries: Vec<_> = output::count_objects_iter(
104+ let counts: Vec<_> = output::count_objects_iter(
105105 db.clone(),
106106 || pack::cache::Never,
107107 commits.chain(std::iter::once(hex_to_id("e3fb53cbb4c346d48732a24f09cf445e49bc63d6"))),
@@ -115,12 +115,30 @@ mod count { · 7 unmodified lines
115115 .into_iter()
116116 .flatten()
117117 .collect();
118+ let actual_count = counts.iter().fold(Count::default(), |mut c, e| {
119+ c.add(e.object_kind);
120+ c
121+ });
122+ assert_eq!(actual_count, expected_count);
123+ assert_eq!(counts.len(), expected_count.total());
124+
125+ let entries: Vec<_> = output::objects_to_entries_iter(
126+ counts,
127+ db.clone(),
128+ || pack::cache::Never,
129+ progress::Discard,
130+ Default::default(),
131+ )
132+ .collect::<Result<Vec<_>, _>>()?
133+ .into_iter()
134+ .flatten()
135+ .collect();
118136 let actual_count = entries.iter().fold(Count::default(), |mut c, e| {
119137 c.add(e.object_kind);
120138 c
121139 });
122140 assert_eq!(actual_count, expected_count);
123− assert_eq!(entries.len(), expected_count.total());
141+ assert_eq!(counts.len(), expected_count.total());
124142 }
125143 Ok(())
126144 }
gitoxide-core/src/pack/create.rs+17−16
@@ -37,9 +37,9 @@ impl FromStr for ObjectExpansion { · 36 unmodified lines
3737 }
3838 }
3939
40−impl From<ObjectExpansion> for pack::data::output::objects_to_entries::ObjectExpansion {
40+impl From<ObjectExpansion> for pack::data::output::count_objects::ObjectExpansion {
4141 fn from(v: ObjectExpansion) -> Self {
42− use pack::data::output::objects_to_entries::ObjectExpansion::*;
42+ use pack::data::output::count_objects::ObjectExpansion::*;
4343 match v {
4444 ObjectExpansion::None => AsIs,
4545 ObjectExpansion::TreeTraversal => TreeContents,
@@ -86,30 +86,31 @@ pub fn create( · 40 unmodified lines
8686 .and_then(|hex_id| git_hash::ObjectId::from_hex(hex_id.as_bytes()).ok())
8787 })),
8888 };
89− let entries = pack::data::output::objects_to_entries_iter(
89+ let entries = pack::data::output::count_objects_iter(
9090 Arc::clone(&db),
9191 pack::cache::lru::StaticLinkedList::<64>::default,
9292 input,
9393 git_features::progress::Discard,
94− pack::data::output::objects_to_entries::Options {
94+ pack::data::output::count_objects::Options {
9595 thread_limit: ctx.thread_limit,
9696 chunk_size: 200,
9797 version: Default::default(),
9898 input_object_expansion: ctx.expansion.into(),
9999 },
100100 );
101− let mut output_iter = pack::data::output::EntriesToBytesIter::new(
102− entries,
103− out,
104− 0,
105− pack::data::Version::default(),
106− git_hash::Kind::default(),
107− );
108− while let Some(io_res) = output_iter.next() {
109− let _written = io_res?;
110− }
111− output_iter.into_write().flush()?;
112− Ok(())
101+ todo!("entries from counts");
102+ // let mut output_iter = pack::data::output::EntriesToBytesIter::new(
103+ // entries,
104+ // out,
105+ // 0,
106+ // pack::data::Version::default(),
107+ // git_hash::Kind::default(),
108+ // );
109+ // while let Some(io_res) = output_iter.next() {
110+ // let _written = io_res?;
111+ // }
112+ // output_iter.into_write().flush()?;
113+ // Ok(())
113114 }
114115
115116 fn find_db(repository: impl AsRef<Path>) -> anyhow::Result<linked::Db> {