nearly there! Interesting that anyhow errors must be sync!

Sebastian Thiel committed 6 years ago

Commit eaee77e1 parent 1fc82529 files · +41 −674Browse files

Unified · Split

9 files

git-odb/src/pack/bundle.rs +3 −2
git-odb/src/pack/index/mod.rs +0 −1
git-odb/src/pack/index/verify.rs +24 −24
git-odb/src/pack/index/verify/indexed.rs +0 −163
git-odb/src/pack/index/verify/lookup.rs +0 −168
git-odb/src/pack/index/verify/mod.rs +0 −209
git-odb/src/pack/index/verify/reduce.rs +0 −95
git-odb/tests/pack/index.rs +6 −6
gitoxide-core/src/pack/verify.rs +8 −6

@@ -80,14 +80,15 @@ impl Bundle { · 79 unmodified lines
80	80	mode: pack::index::verify::Mode,
81	81	progress: Option<P>,
82	82	make_cache: impl Fn() -> C + Send + Sync,
83		− ) -> Result<(owned::Id, Option<pack::index::verify::Outcome>), pack::index::verify::Error>
	83	+ ) -> Result<(owned::Id, Option<pack::index::traverse::Outcome>), pack::index::traverse::Error>
84	84	where
85	85	P: Progress,
86	86	<P as Progress>::SubProgress: Send,
87	87	C: pack::cache::DecodeEntry,
	88	+ <<P as Progress>::SubProgress as Progress>::SubProgress: std::marker::Send,
88	89	{
89	90	self.index.verify_integrity(
90		− Some((&self.pack, mode, pack::index::verify::Algorithm::default())),
	91	+ Some((&self.pack, mode, pack::index::traverse::Algorithm::default())),
91	92	thread_limit,
92	93	progress,
93	94	make_cache,

git-odb/src/pack/index/mod.rs+0−1

@@ -116,4 +116,3 @@ pub use access::Entry; · 115 unmodified lines
116	116	pub mod traverse;
117	117	pub(crate) mod util;
118	118	pub mod verify;
119		−mod verify2;

git-odb/src/pack/index/verify.rsrenamed from git-odb/src/pack/index/verify2.rs+24−24

@@ -39,35 +39,35 @@ pub enum Mode { · 38 unmodified lines
39	39
40	40	/// Verify and validate the content of the index file
41	41	impl index::File {
42		− // pub fn index_checksum(&self) -> owned::Id {
43		− // owned::Id::from_20_bytes(&self.data[self.data.len() - SHA1_SIZE..])
44		− // }
45		− //
46		− // pub fn pack_checksum(&self) -> owned::Id {
47		− // let from = self.data.len() - SHA1_SIZE * 2;
48		− // owned::Id::from_20_bytes(&self.data[from..from + SHA1_SIZE])
49		− // }
50		− //
51		− // pub fn verify_checksum(&self, mut progress: impl Progress) -> Result<owned::Id, Error> {
52		− // let throughput = index::util::TimeThroughput::new(self.data.len());
53		− // let mut hasher = git_features::hash::Sha1::default();
54		− // hasher.update(&self.data[..self.data.len() - SHA1_SIZE]);
55		− // let actual = owned::Id::new_sha1(hasher.digest());
56		− // progress.done(throughput);
57		− //
58		− // let expected = self.index_checksum();
59		− // if actual == expected {
60		− // Ok(actual)
61		− // } else {
62		− // Err(Error::Mismatch { actual, expected })
63		− // }
64		− // }
	42	+ pub fn index_checksum(&self) -> owned::Id {
	43	+ owned::Id::from_20_bytes(&self.data[self.data.len() - SHA1_SIZE..])
	44	+ }
	45	+
	46	+ pub fn pack_checksum(&self) -> owned::Id {
	47	+ let from = self.data.len() - SHA1_SIZE * 2;
	48	+ owned::Id::from_20_bytes(&self.data[from..from + SHA1_SIZE])
	49	+ }
	50	+
	51	+ pub fn verify_checksum(&self, mut progress: impl Progress) -> Result<owned::Id, Error> {
	52	+ let throughput = index::util::TimeThroughput::new(self.data.len());
	53	+ let mut hasher = git_features::hash::Sha1::default();
	54	+ hasher.update(&self.data[..self.data.len() - SHA1_SIZE]);
	55	+ let actual = owned::Id::new_sha1(hasher.digest());
	56	+ progress.done(throughput);
	57	+
	58	+ let expected = self.index_checksum();
	59	+ if actual == expected {
	60	+ Ok(actual)
	61	+ } else {
	62	+ Err(Error::Mismatch { actual, expected })
	63	+ }
	64	+ }
65	65
66	66	/// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index.
67	67	/// It will be used to validate internal integrity of the pack before checking each objects integrity
68	68	/// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash.
69	69	/// redoing a lot of work across multiple objects.
70		− pub fn verify_integrity_2<P, C>(
	70	+ pub fn verify_integrity<P, C>(
71	71	&self,
72	72	pack: Option<(&pack::data::File, Mode, index::traverse::Algorithm)>,
73	73	thread_limit: Option<usize>,

git-odb/src/pack/index/verify/indexed.rs+0−163

@@ -1,163 +0,0 @@
1		−use super::{Error, Mode, Outcome};
2		−use crate::{
3		− pack,
4		− pack::index::access::PackOffset,
5		− pack::index::{self, util},
6		−};
7		−use git_features::{
8		− parallel::{self, in_parallel_if},
9		− progress::{self, Progress},
10		−};
11		−use git_object::Kind;
12		−use std::collections::BTreeMap;
13		−
14		−impl index::File {
15		− pub(crate) fn inner_verify_with_indexed_lookup<P>(
16		− &self,
17		− thread_limit: Option<usize>,
18		− mode: Mode,
19		− mut root: progress::DoOrDiscard<P>,
20		− pack: &pack::data::File,
21		− ) -> Result<Outcome, Error>
22		− where
23		− P: Progress,
24		− <P as Progress>::SubProgress: Send,
25		− {
26		− let sorted_entries =
27		− util::index_entries_sorted_by_offset_ascending(self, root.add_child("collecting sorted index"));
28		− let tree = pack::graph::DeltaTree::from_sorted_offsets(
29		− sorted_entries.iter().map(\|e\| e.pack_offset),
30		− pack.path(),
31		− root.add_child("indexing"),
32		− \|id\| self.lookup_index(id).map(\|idx\| self.pack_offset_at_index(idx)),
33		− )?;
34		− let if_there_are_enough_objects = \|\| self.num_objects > 10_000;
35		−
36		− let reduce_progress = std::sync::Mutex::new({
37		− let mut p = root.add_child("Checking");
38		− p.init(Some(self.num_objects()), Some("objects"));
39		− p
40		− });
41		−
42		− let state_per_thread = \|index\| {
43		− (
44		− Vec::<u8>::with_capacity(2048), // decode buffer
45		− Vec::<u8>::with_capacity(2048), // re-encode buffer
46		− Vec::<(pack::graph::Node, u32)>::new(),
47		− reduce_progress.lock().unwrap().add_child(format!("thread {}", index)), // per thread progress
48		− )
49		− };
50		− struct Chunks<I> {
51		− size: usize,
52		− iter: I,
53		− }
54		− impl<I> Iterator for Chunks<I>
55		− where
56		− I: Iterator<Item = pack::graph::Node>,
57		− {
58		− type Item = Vec<pack::graph::Node>;
59		−
60		− fn next(&mut self) -> Option<Self::Item> {
61		− let mut res = Vec::with_capacity(self.size);
62		− let mut items_left = self.size;
63		− while let Some(item) = self.iter.next() {
64		− res.push(item);
65		− items_left -= 1;
66		− if items_left == 0 {
67		− break;
68		− }
69		− }
70		− if res.is_empty() {
71		− None
72		− } else {
73		− Some(res)
74		− }
75		− }
76		− }
77		− let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit(1, None, thread_limit, None);
78		− in_parallel_if(
79		− if_there_are_enough_objects,
80		− Chunks {
81		− size: chunk_size,
82		− iter: tree.bases(),
83		− },
84		− thread_limit,
85		− state_per_thread,
86		− \|input: Vec<pack::graph::Node>,
87		− (buf, encode_buf, nodes, progress)\|
88		− -> Result<Vec<pack::data::decode::Outcome>, Error> {
89		− let mut stats = Vec::new();
90		− let mut header_buf = [0u8; 64];
91		− let mut children = Vec::new();
92		− progress.init(None, Some("entries"));
93		−
94		− struct CacheEntry {
95		− kind: Kind,
96		− data: Vec<u8>,
97		− compressed_size: usize,
98		− }
99		− struct SharedCache<'a>(&'a mut BTreeMap<PackOffset, CacheEntry>);
100		−
101		− impl<'a> pack::cache::DecodeEntry for SharedCache<'a> {
102		− fn put(&mut self, pack_offset: u64, data: &[u8], kind: Kind, compressed_size: usize) {
103		− self.0.entry(pack_offset).or_insert_with(\|\| CacheEntry {
104		− kind,
105		− data: data.to_owned(),
106		− compressed_size,
107		− });
108		− }
109		−
110		− fn get(&mut self, offset: u64, out: &mut Vec<u8>) -> Option<(Kind, usize)> {
111		− self.0.get_mut(&offset).map(
112		− \|CacheEntry {
113		− kind,
114		− data,
115		− compressed_size,
116		− }\| {
117		− out.resize(data.len(), 0);
118		− out.copy_from_slice(&data);
119		− (kind, compressed_size)
120		− },
121		− )
122		− }
123		− }
124		−
125		− for node in input {
126		− nodes.clear();
127		− nodes.push((node, 0));
128		− let mut cache = BTreeMap::new();
129		−
130		− while let Some((node, level)) = nodes.pop() {
131		− let pack_offset = node.pack_offset;
132		− let index_entry = sorted_entries
133		− .binary_search_by(\|e\| e.pack_offset.cmp(&pack_offset))
134		− .expect("tree created by our sorted entries");
135		− let index_entry_of_node = &sorted_entries[index_entry];
136		−
137		− tree.children(node, &mut children);
138		−
139		− let shared_cache = &mut SharedCache(&mut cache);
140		− let mut stat = self.process_entry(
141		− mode,
142		− pack,
143		− shared_cache,
144		− buf,
145		− encode_buf,
146		− progress,
147		− &mut header_buf,
148		− index_entry_of_node,
149		− )?;
150		− stat.num_deltas = level;
151		− stats.push(stat);
152		−
153		− progress.inc();
154		− nodes.extend(children.iter().cloned().map(\|cn\| (cn, level + 1)));
155		− }
156		− }
157		−
158		− Ok(stats)
159		− },
160		− index::verify::Reducer::from_progress(&reduce_progress, pack.data_len()),
161		− )
162		− }
163		−}

git-odb/src/pack/index/verify/lookup.rs+0−168

@@ -1,168 +0,0 @@
1		−use super::{Error, Mode, Outcome, Reducer};
2		−use crate::pack::{self, data::decode, index, index::util};
3		−use git_features::{
4		− parallel::{self, in_parallel_if},
5		− progress::{self, Progress},
6		−};
7		−use git_object::{borrowed, bstr::ByteSlice, owned};
8		−
9		−/// Verify and validate the content of the index file
10		−impl index::File {
11		− pub(crate) fn inner_verify_with_lookup<P, C>(
12		− &self,
13		− thread_limit: Option<usize>,
14		− mode: Mode,
15		− make_cache: impl Fn() -> C + Send + Sync,
16		− mut root: progress::DoOrDiscard<P>,
17		− pack: &pack::data::File,
18		− ) -> Result<Outcome, Error>
19		− where
20		− P: Progress,
21		− <P as Progress>::SubProgress: Send,
22		− C: pack::cache::DecodeEntry,
23		− {
24		− let index_entries =
25		− util::index_entries_sorted_by_offset_ascending(self, root.add_child("collecting sorted index"));
26		−
27		− let (chunk_size, thread_limit, available_cores) =
28		− parallel::optimize_chunk_size_and_thread_limit(1000, Some(index_entries.len()), thread_limit, None);
29		− let there_are_enough_entries_to_process = \|\| index_entries.len() > chunk_size * available_cores;
30		− let input_chunks = index_entries.chunks(chunk_size.max(chunk_size));
31		− let reduce_progress = std::sync::Mutex::new({
32		− let mut p = root.add_child("Checking");
33		− p.init(Some(self.num_objects()), Some("objects"));
34		− p
35		− });
36		− let state_per_thread = \|index\| {
37		− (
38		− make_cache(),
39		− Vec::with_capacity(2048), // decode buffer
40		− Vec::with_capacity(2048), // re-encode buffer
41		− reduce_progress.lock().unwrap().add_child(format!("thread {}", index)), // per thread progress
42		− )
43		− };
44		−
45		− in_parallel_if(
46		− there_are_enough_entries_to_process,
47		− input_chunks,
48		− thread_limit,
49		− state_per_thread,
50		− \|entries: &[index::Entry], (cache, buf, encode_buf, progress)\| -> Result<Vec<decode::Outcome>, Error> {
51		− progress.init(Some(entries.len() as u32), Some("entries"));
52		− let mut stats = Vec::with_capacity(entries.len());
53		− let mut header_buf = [0u8; 64];
54		− for index_entry in entries.iter() {
55		− stats.push(self.process_entry(
56		− mode,
57		− pack,
58		− cache,
59		− buf,
60		− encode_buf,
61		− progress,
62		− &mut header_buf,
63		− index_entry,
64		− )?);
65		− progress.inc();
66		− }
67		− Ok(stats)
68		− },
69		− Reducer::from_progress(&reduce_progress, pack.data_len()),
70		− )
71		− }
72		−
73		− #[allow(clippy::too_many_arguments)]
74		− pub(crate) fn process_entry<C>(
75		− &self,
76		− mode: Mode,
77		− pack: &pack::data::File,
78		− cache: &mut C,
79		− buf: &mut Vec<u8>,
80		− encode_buf: &mut Vec<u8>,
81		− progress: &mut impl Progress,
82		− header_buf: &mut [u8; 64],
83		− index_entry: &pack::index::Entry,
84		− ) -> Result<decode::Outcome, Error>
85		− where
86		− C: pack::cache::DecodeEntry,
87		− {
88		− let pack_entry = pack.entry(index_entry.pack_offset);
89		− let pack_entry_data_offset = pack_entry.data_offset;
90		− let entry_stats = pack
91		− .decode_entry(
92		− pack_entry,
93		− buf,
94		− \|id, _\| {
95		− self.lookup_index(id).map(\|index\| {
96		− pack::data::decode::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
97		− })
98		− },
99		− cache,
100		− )
101		− .map_err(\|e\| Error::PackDecode(e, index_entry.oid, index_entry.pack_offset))?;
102		− let object_kind = entry_stats.kind;
103		− let consumed_input = entry_stats.compressed_size;
104		−
105		− let header_size = crate::loose::object::header::encode(object_kind, buf.len() as u64, &mut header_buf[..])
106		− .expect("header buffer to be big enough");
107		− let mut hasher = git_features::hash::Sha1::default();
108		− hasher.update(&header_buf[..header_size]);
109		− hasher.update(buf.as_slice());
110		−
111		− let actual_oid = owned::Id::new_sha1(hasher.digest());
112		− if actual_oid != index_entry.oid {
113		− return Err(Error::PackObjectMismatch {
114		− actual: actual_oid,
115		− expected: index_entry.oid,
116		− offset: index_entry.pack_offset,
117		− kind: object_kind,
118		− });
119		− }
120		− if let Some(desired_crc32) = index_entry.crc32 {
121		− let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
122		− let actual_crc32 = pack.entry_crc32(index_entry.pack_offset, header_size + consumed_input);
123		− if actual_crc32 != desired_crc32 {
124		− return Err(Error::Crc32Mismatch {
125		− actual: actual_crc32,
126		− expected: desired_crc32,
127		− offset: index_entry.pack_offset,
128		− kind: object_kind,
129		− });
130		− }
131		− }
132		− if let Mode::Sha1CRC32Decode \| Mode::Sha1CRC32DecodeEncode = mode {
133		− use git_object::Kind::*;
134		− match object_kind {
135		− Tree \| Commit \| Tag => {
136		− let borrowed_object = borrowed::Object::from_bytes(object_kind, buf.as_slice())
137		− .map_err(\|err\| Error::ObjectDecode(err, object_kind, index_entry.oid))?;
138		− if let Mode::Sha1CRC32DecodeEncode = mode {
139		− let object = owned::Object::from(borrowed_object);
140		− encode_buf.clear();
141		− object.write_to(&mut *encode_buf)?;
142		− if encode_buf != buf {
143		− let mut should_return_error = true;
144		− if let git_object::Kind::Tree = object_kind {
145		− if buf.as_slice().as_bstr().find(b"100664").is_some()
146		− \|\| buf.as_slice().as_bstr().find(b"100640").is_some()
147		− {
148		− progress.info(format!("Tree object {} would be cleaned up during re-serialization, replacing mode '100664\|100640' with '100644'", index_entry.oid));
149		− should_return_error = false
150		− }
151		− }
152		− if should_return_error {
153		− return Err(Error::ObjectEncodeMismatch(
154		− object_kind,
155		− index_entry.oid,
156		− buf.clone().into(),
157		− encode_buf.clone().into(),
158		− ));
159		− }
160		− }
161		− }
162		− }
163		− Blob => {}
164		− };
165		− }
166		− Ok(entry_stats)
167		− }
168		−}

git-odb/src/pack/index/verify/mod.rs+0−209

@@ -1,209 +0,0 @@
1		−use crate::{pack, pack::data::decode, pack::index};
2		−use git_features::{
3		− parallel,
4		− progress::{self, Progress},
5		−};
6		−use git_object::{borrowed, bstr::BString, owned, SHA1_SIZE};
7		−use quick_error::quick_error;
8		−use std::{collections::BTreeMap, time::Instant};
9		−
10		−quick_error! {
11		− #[derive(Debug)]
12		− pub enum Error {
13		− Io(err: std::io::Error, path: std::path::PathBuf, msg: &'static str) {
14		− display("Failed to {} at path '{}'", msg, path.display())
15		− source(err)
16		− }
17		− Graph(err: pack::graph::Error) {
18		− from()
19		− source(err)
20		− }
21		− Mismatch { expected: owned::Id, actual: owned::Id } {
22		− display("index checksum mismatch: expected {}, got {}", expected, actual)
23		− }
24		− PackChecksum(err: pack::data::verify::Error) {
25		− display("The pack of this index file failed to verify its checksums")
26		− from()
27		− source(err)
28		− }
29		− PackDecode(err: pack::data::decode::Error, id: owned::Id, offset: u64) {
30		− display("Object {} at offset {} could not be decoded", id, offset)
31		− source(err)
32		− }
33		− ObjectDecode(err: borrowed::Error, kind: git_object::Kind, oid: owned::Id) {
34		− display("{} object {} could not be decoded", kind, oid)
35		− source(err)
36		− }
37		− ObjectEncodeMismatch(kind: git_object::Kind, oid: owned::Id, expected: BString, actual: BString) {
38		− display("{} object {} wasn't re-encoded without change, wanted\n{}\n\nGOT\n\n{}", kind, oid, expected, actual)
39		− }
40		− ObjectEncode(err: std::io::Error) {
41		− from()
42		− }
43		− PackMismatch { expected: owned::Id, actual: owned::Id } {
44		− display("The packfiles checksum didn't match the index file checksum: expected {}, got {}", expected, actual)
45		− }
46		− PackObjectMismatch { expected: owned::Id, actual: owned::Id, offset: u64, kind: git_object::Kind} {
47		− display("The SHA1 of {} object at offset {} didn't match the checksum in the index file: expected {}, got {}", kind, offset, expected, actual)
48		− }
49		− Crc32Mismatch { expected: u32, actual: u32, offset: u64, kind: git_object::Kind} {
50		− display("The CRC32 of {} object at offset {} didn't match the checksum in the index file: expected {}, got {}", kind, offset, expected, actual)
51		− }
52		− }
53		−}
54		−
55		−struct TimeThroughput {
56		− then: Instant,
57		− byte_size: usize,
58		−}
59		−
60		−impl TimeThroughput {
61		− pub fn new(byte_size: usize) -> TimeThroughput {
62		− TimeThroughput {
63		− then: Instant::now(),
64		− byte_size,
65		− }
66		− }
67		−}
68		−
69		−impl Into<String> for TimeThroughput {
70		− fn into(self) -> String {
71		− let time_taken = std::time::Instant::now().duration_since(self.then).as_secs_f32();
72		− format!(
73		− "finished in {:.2}s at {}/s",
74		− time_taken,
75		− bytesize::ByteSize((self.byte_size as f32 / time_taken) as u64)
76		− )
77		− }
78		−}
79		−
80		−#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)]
81		−#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
82		−pub struct Outcome {
83		− pub average: decode::Outcome,
84		− pub objects_per_chain_length: BTreeMap<u32, u32>,
85		− /// The amount of bytes in all compressed streams, one per entry
86		− pub total_compressed_entries_size: u64,
87		− /// The amount of bytes in all decompressed streams, one per entry
88		− pub total_decompressed_entries_size: u64,
89		− /// The amount of bytes occupied by all undeltified, decompressed objects
90		− pub total_object_size: u64,
91		− /// The amount of bytes occupied by the pack itself, in bytes
92		− pub pack_size: u64,
93		−}
94		−
95		−/// Various ways in which a pack and index can be verified
96		−#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]
97		−pub enum Mode {
98		− /// Validate SHA1 and CRC32
99		− Sha1CRC32,
100		− /// Validate SHA1 and CRC32, and decode each non-Blob object
101		− Sha1CRC32Decode,
102		− /// Validate SHA1 and CRC32, and decode and encode each non-Blob object
103		− Sha1CRC32DecodeEncode,
104		−}
105		−
106		−/// The way we verify the pack
107		−#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]
108		−pub enum Algorithm {
109		− /// Build an index to allow decoding each delta and base exactly once, saving a lot of computational
110		− /// resource at the expense of resident memory, as we will use an additional `DeltaTree` to accelerate
111		− /// delta chain resolution.
112		− DeltaTreeLookup,
113		− /// We lookup each object similarly to what would happen during normal repository use.
114		− /// Uses more compute resources as it will resolve delta chains from back to front, but start right away
115		− /// without indexing or investing any memory in indices.
116		− ///
117		− /// This option may be well suited for big packs in memory-starved system that support memory mapping.
118		− Lookup,
119		−}
120		−
121		−impl Default for Algorithm {
122		− fn default() -> Self {
123		− Algorithm::DeltaTreeLookup
124		− }
125		−}
126		−
127		−mod indexed;
128		−mod lookup;
129		−mod reduce;
130		−pub(crate) use reduce::Reducer;
131		−
132		−/// Verify and validate the content of the index file
133		−impl index::File {
134		− pub fn index_checksum(&self) -> owned::Id {
135		− owned::Id::from_20_bytes(&self.data[self.data.len() - SHA1_SIZE..])
136		− }
137		−
138		− pub fn pack_checksum(&self) -> owned::Id {
139		− let from = self.data.len() - SHA1_SIZE * 2;
140		− owned::Id::from_20_bytes(&self.data[from..from + SHA1_SIZE])
141		− }
142		−
143		− pub fn verify_checksum(&self, mut progress: impl Progress) -> Result<owned::Id, Error> {
144		− let throughput = TimeThroughput::new(self.data.len());
145		− let mut hasher = git_features::hash::Sha1::default();
146		− hasher.update(&self.data[..self.data.len() - SHA1_SIZE]);
147		− let actual = owned::Id::new_sha1(hasher.digest());
148		− progress.done(throughput);
149		−
150		− let expected = self.index_checksum();
151		− if actual == expected {
152		− Ok(actual)
153		− } else {
154		− Err(Error::Mismatch { actual, expected })
155		− }
156		− }
157		−
158		− /// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index.
159		− /// It will be used to validate internal integrity of the pack before checking each objects integrity
160		− /// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash.
161		− /// redoing a lot of work across multiple objects.
162		− pub fn verify_integrity<P, C>(
163		− &self,
164		− pack: Option<(&pack::data::File, Mode, Algorithm)>,
165		− thread_limit: Option<usize>,
166		− progress: Option<P>,
167		− make_cache: impl Fn() -> C + Send + Sync,
168		− ) -> Result<(owned::Id, Option<Outcome>), Error>
169		− where
170		− P: Progress,
171		− <P as Progress>::SubProgress: Send,
172		− C: pack::cache::DecodeEntry,
173		− {
174		− let mut root = progress::DoOrDiscard::from(progress);
175		−
176		− let progress = root.add_child("Sha1 of index");
177		− let verify_self = move \|\| self.verify_checksum(progress);
178		−
179		− match pack {
180		− None => verify_self().map(\|id\| (id, None)),
181		− Some((pack, mode, algorithm)) => {
182		− if self.pack_checksum() != pack.checksum() {
183		− return Err(Error::PackMismatch {
184		− actual: pack.checksum(),
185		− expected: self.pack_checksum(),
186		− });
187		− }
188		− let mut progress = root.add_child("Sha1 of pack");
189		− let (pack_res, id) = parallel::join(
190		− move \|\| {
191		− let throughput = TimeThroughput::new(pack.data_len());
192		− let res = pack.verify_checksum();
193		− progress.done(throughput);
194		− res
195		− },
196		− verify_self,
197		− );
198		− pack_res?;
199		− let id = id?;
200		−
201		− match algorithm {
202		− Algorithm::Lookup => self.inner_verify_with_lookup(thread_limit, mode, make_cache, root, pack),
203		− Algorithm::DeltaTreeLookup => self.inner_verify_with_indexed_lookup(thread_limit, mode, root, pack),
204		− }
205		− .map(\|stats\| (id, Some(stats)))
206		− }
207		− }
208		− }
209		−}

git-odb/src/pack/index/verify/reduce.rs+0−95

@@ -1,95 +0,0 @@
1		−use crate::pack::{data::decode, index::verify};
2		−use git_features::{parallel, progress::Progress};
3		−use std::time::Instant;
4		−
5		−fn add_decode_result(lhs: &mut decode::Outcome, rhs: decode::Outcome) {
6		− lhs.num_deltas += rhs.num_deltas;
7		− lhs.decompressed_size += rhs.decompressed_size;
8		− lhs.compressed_size += rhs.compressed_size;
9		− lhs.object_size += rhs.object_size;
10		−}
11		−
12		−fn div_decode_result(lhs: &mut decode::Outcome, div: usize) {
13		− if div != 0 {
14		− lhs.num_deltas = (lhs.num_deltas as f32 / div as f32) as u32;
15		− lhs.decompressed_size /= div as u64;
16		− lhs.compressed_size /= div;
17		− lhs.object_size /= div as u64;
18		− }
19		−}
20		−
21		−pub struct Reducer<'a, P> {
22		− pub progress: &'a std::sync::Mutex<P>,
23		− pub then: Instant,
24		− pub entries_seen: u32,
25		− pub stats: verify::Outcome,
26		−}
27		−
28		−impl<'a, P> Reducer<'a, P>
29		−where
30		− P: Progress,
31		−{
32		− pub fn from_progress(progress: &'a std::sync::Mutex<P>, pack_data_len_in_bytes: usize) -> Self {
33		− Reducer {
34		− progress: &progress,
35		− then: Instant::now(),
36		− entries_seen: 0,
37		− stats: verify::Outcome {
38		− average: decode::Outcome::default_from_kind(git_object::Kind::Tree),
39		− objects_per_chain_length: Default::default(),
40		− total_compressed_entries_size: 0,
41		− total_decompressed_entries_size: 0,
42		− total_object_size: 0,
43		− pack_size: pack_data_len_in_bytes as u64,
44		− },
45		− }
46		− }
47		−}
48		−
49		−impl<'a, P> parallel::Reducer for Reducer<'a, P>
50		−where
51		− P: Progress,
52		−{
53		− type Input = Result<Vec<decode::Outcome>, verify::Error>;
54		− type Output = verify::Outcome;
55		− type Error = verify::Error;
56		−
57		− fn feed(&mut self, input: Self::Input) -> Result<(), Self::Error> {
58		− let chunk_stats: Vec<_> = input?;
59		− self.entries_seen += chunk_stats.len() as u32;
60		−
61		− let chunk_total = chunk_stats.into_iter().fold(
62		− decode::Outcome::default_from_kind(git_object::Kind::Tree),
63		− \|mut total, stats\| {
64		− *self.stats.objects_per_chain_length.entry(stats.num_deltas).or_insert(0) += 1;
65		− self.stats.total_decompressed_entries_size += stats.decompressed_size;
66		− self.stats.total_compressed_entries_size += stats.compressed_size as u64;
67		− self.stats.total_object_size += stats.object_size as u64;
68		− add_decode_result(&mut total, stats);
69		− total
70		− },
71		− );
72		−
73		− add_decode_result(&mut self.stats.average, chunk_total);
74		− self.progress.lock().unwrap().set(self.entries_seen);
75		− Ok(())
76		− }
77		−
78		− fn finalize(mut self) -> Result<Self::Output, Self::Error> {
79		− self.progress.lock().unwrap().done("finished");
80		−
81		− div_decode_result(&mut self.stats.average, self.entries_seen as usize);
82		−
83		− let elapsed_s = self.then.elapsed().as_secs_f32();
84		− let objects_per_second = (self.entries_seen as f32 / elapsed_s) as u32;
85		−
86		− self.progress.lock().unwrap().info(format!(
87		− "Verified {} objects in {:.2}s ({} objects/s, ~{}/s)",
88		− self.entries_seen,
89		− elapsed_s,
90		− objects_per_second,
91		− bytesize::ByteSize(self.stats.average.object_size * objects_per_second as u64)
92		− ));
93		− Ok(self.stats)
94		− }
95		−}

git-odb/tests/pack/index.rs+6−6

@@ -73,9 +73,9 @@ use common_macros::b_tree_map; · 72 unmodified lines
73	73	use git_features::progress::Discard;
74	74	use git_odb::pack::cache::DecodeEntryNoop;
75	75
76		−static ALGOS: &[index::verify::Algorithm] = &[
77		− index::verify::Algorithm::Lookup,
78		− index::verify::Algorithm::DeltaTreeLookup,
	76	+static ALGOS: &[index::traverse::Algorithm] = &[
	77	+ index::traverse::Algorithm::Lookup,
	78	+ index::traverse::Algorithm::DeltaTreeLookup,
79	79	];
80	80
81	81	static MODES: &[index::verify::Mode] = &[
@@ -90,7 +90,7 @@ fn pack_lookup() { · 8 unmodified lines
90	90	(
91	91	INDEX_V2,
92	92	PACK_FOR_INDEX_V2,
93		− index::verify::Outcome {
	93	+ index::traverse::Outcome {
94	94	average: Outcome {
95	95	kind: object::Kind::Tree,
96	96	num_deltas: 1,
@@ -116,7 +116,7 @@ fn pack_lookup() { · 19 unmodified lines
116	116	(
117	117	INDEX_V1,
118	118	PACK_FOR_INDEX_V1,
119		− index::verify::Outcome {
	119	+ index::traverse::Outcome {
120	120	average: Outcome {
121	121	kind: object::Kind::Tree,
122	122	num_deltas: 0,
@@ -137,7 +137,7 @@ fn pack_lookup() { · 14 unmodified lines
137	137	(
138	138	SMALL_PACK_INDEX,
139	139	SMALL_PACK,
140		− index::verify::Outcome {
	140	+ index::traverse::Outcome {
141	141	average: Outcome {
142	142	kind: object::Kind::Tree,
143	143	num_deltas: 0,

gitoxide-core/src/pack/verify.rs+8−6

@@ -33,11 +33,11 @@ impl FromStr for Algorithm { · 32 unmodified lines
33	33	}
34	34	}
35	35
36		−impl From<Algorithm> for index::verify::Algorithm {
	36	+impl From<Algorithm> for index::traverse::Algorithm {
37	37	fn from(v: Algorithm) -> Self {
38	38	match v {
39		− Algorithm::LessMemory => index::verify::Algorithm::Lookup,
40		− Algorithm::LessTime => index::verify::Algorithm::DeltaTreeLookup,
	39	+ Algorithm::LessMemory => index::traverse::Algorithm::Lookup,
	40	+ Algorithm::LessTime => index::traverse::Algorithm::DeltaTreeLookup,
41	41	}
42	42	}
43	43	}
@@ -104,12 +104,14 @@ pub fn pack_or_pack_index<P, W1, W2>( · 60 unmodified lines
104	104	thread_limit,
105	105	algorithm,
106	106	}: Context<W1, W2>,
107		−) -> Result<(owned::Id, Option<index::verify::Outcome>)>
	107	+) -> Result<(owned::Id, Option<index::traverse::Outcome>)>
108	108	where
109	109	P: Progress,
110	110	<P as Progress>::SubProgress: Send,
111	111	W1: io::Write,
112	112	W2: io::Write,
	113	+ <<P as git_features::progress::Progress>::SubProgress as git_features::progress::Progress>::SubProgress:
	114	+ std::marker::Send,
113	115	{
114	116	let path = path.as_ref();
115	117	let ext = path.extension().and_then(\|ext\| ext.to_str()).ok_or_else(\|\| {
@@ -153,7 +155,7 @@ where · 37 unmodified lines
153	155	progress,
154	156	cache,
155	157	)
156		− .with_context(\|\| "Verification failure")?
	158	+ .unwrap() // FIXME
157	159	}
158	160	ext => return Err(anyhow!("Unknown extension {:?}, expecting 'idx' or 'pack'", ext)),
159	161	};
@@ -168,7 +170,7 @@ where · 8 unmodified lines
168	170	Ok(res)
169	171	}
170	172
171		−fn print_statistics(out: &mut impl io::Write, stats: &index::verify::Outcome) -> io::Result<()> {
	173	+fn print_statistics(out: &mut impl io::Write, stats: &index::traverse::Outcome) -> io::Result<()> {
172	174	writeln!(out, "objects per delta chain length")?;
173	175	let mut chain_length_to_object: Vec<_> = stats.objects_per_chain_length.iter().map(\|(a, b)\| (a, b)).collect();
174	176	chain_length_to_object.sort_by_key(\|e\| e.0);