lael-labs/gitoxide

git-odb/src/pack/bundle.rs+3−2

         mode: pack::index::verify::Mode,
         progress: Option<P>,
         make_cache: impl Fn() -> C + Send + Sync,
-    ) -> Result<(owned::Id, Option<pack::index::verify::Outcome>), pack::index::verify::Error>
+    ) -> Result<(owned::Id, Option<pack::index::traverse::Outcome>), pack::index::traverse::Error>
     where
         P: Progress,
         <P as Progress>::SubProgress: Send,
         C: pack::cache::DecodeEntry,
+        <<P as Progress>::SubProgress as Progress>::SubProgress: std::marker::Send,
     {
         self.index.verify_integrity(
-            Some((&self.pack, mode, pack::index::verify::Algorithm::default())),
+            Some((&self.pack, mode, pack::index::traverse::Algorithm::default())),
             thread_limit,
             progress,
             make_cache,

git-odb/src/pack/index/mod.rs+0−1

@@ -116,4 +116,3 @@ pub use access::Entry; · 115 unmodified lines
116	pub mod traverse;	116	pub mod traverse;
117	pub(crate) mod util;	117	pub(crate) mod util;
118	pub mod verify;	118	pub mod verify;
119	mod verify2;

git-odb/src/pack/index/verify.rsrenamed from git-odb/src/pack/index/verify2.rs+24−24

git-odb/src/pack/index/verify/indexed.rs+0−163

@@ -1,163 +0,0 @@
1	use super::{Error, Mode, Outcome};
2	use crate::{
3	pack,
4	pack::index::access::PackOffset,
5	pack::index::{self, util},
6	};
7	use git_features::{
8	parallel::{self, in_parallel_if},
9	progress::{self, Progress},
10	};
11	use git_object::Kind;
12	use std::collections::BTreeMap;
13
14	impl index::File {
15	pub(crate) fn inner_verify_with_indexed_lookup<P>(
16	&self,
17	thread_limit: Option<usize>,
18	mode: Mode,
19	mut root: progress::DoOrDiscard<P>,
20	pack: &pack::data::File,
21	) -> Result<Outcome, Error>
22	where
23	P: Progress,
24	<P as Progress>::SubProgress: Send,
25	{
26	let sorted_entries =
27	util::index_entries_sorted_by_offset_ascending(self, root.add_child("collecting sorted index"));
28	let tree = pack::graph::DeltaTree::from_sorted_offsets(
29	sorted_entries.iter().map(\|e\| e.pack_offset),
30	pack.path(),
31	root.add_child("indexing"),
32	\|id\| self.lookup_index(id).map(\|idx\| self.pack_offset_at_index(idx)),
33	)?;
34	let if_there_are_enough_objects = \|\| self.num_objects > 10_000;
35
36	let reduce_progress = std::sync::Mutex::new({
37	let mut p = root.add_child("Checking");
38	p.init(Some(self.num_objects()), Some("objects"));
39	p
40	});
41
42	let state_per_thread = \|index\| {
43	(
44	Vec::<u8>::with_capacity(2048), // decode buffer
45	Vec::<u8>::with_capacity(2048), // re-encode buffer
46	Vec::<(pack::graph::Node, u32)>::new(),
47	reduce_progress.lock().unwrap().add_child(format!("thread {}", index)), // per thread progress
48	)
49	};
50	struct Chunks<I> {
51	size: usize,
52	iter: I,
53	}
54	impl<I> Iterator for Chunks<I>
55	where
56	I: Iterator<Item = pack::graph::Node>,
57	{
58	type Item = Vec<pack::graph::Node>;
59
60	fn next(&mut self) -> Option<Self::Item> {
61	let mut res = Vec::with_capacity(self.size);
62	let mut items_left = self.size;
63	while let Some(item) = self.iter.next() {
64	res.push(item);
65	items_left -= 1;
66	if items_left == 0 {
67	break;
68	}
69	}
70	if res.is_empty() {
71	None
72	} else {
73	Some(res)
74	}
75	}
76	}
77	let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit(1, None, thread_limit, None);
78	in_parallel_if(
79	if_there_are_enough_objects,
80	Chunks {
81	size: chunk_size,
82	iter: tree.bases(),
83	},
84	thread_limit,
85	state_per_thread,
86	\|input: Vec<pack::graph::Node>,
87	(buf, encode_buf, nodes, progress)\|
88	-> Result<Vec<pack::data::decode::Outcome>, Error> {
89	let mut stats = Vec::new();
90	let mut header_buf = [0u8; 64];
91	let mut children = Vec::new();
92	progress.init(None, Some("entries"));
93
94	struct CacheEntry {
95	kind: Kind,
96	data: Vec<u8>,
97	compressed_size: usize,
98	}
99	struct SharedCache<'a>(&'a mut BTreeMap<PackOffset, CacheEntry>);
100
101	impl<'a> pack::cache::DecodeEntry for SharedCache<'a> {
102	fn put(&mut self, pack_offset: u64, data: &[u8], kind: Kind, compressed_size: usize) {
103	self.0.entry(pack_offset).or_insert_with(\|\| CacheEntry {
104	kind,
105	data: data.to_owned(),
106	compressed_size,
107	});
108	}
109
110	fn get(&mut self, offset: u64, out: &mut Vec<u8>) -> Option<(Kind, usize)> {
111	self.0.get_mut(&offset).map(
112	\|CacheEntry {
113	kind,
114	data,
115	compressed_size,
116	}\| {
117	out.resize(data.len(), 0);
118	out.copy_from_slice(&data);
119	(kind, compressed_size)
120	},
121	)
122	}
123	}
124
125	for node in input {
126	nodes.clear();
127	nodes.push((node, 0));
128	let mut cache = BTreeMap::new();
129
130	while let Some((node, level)) = nodes.pop() {
131	let pack_offset = node.pack_offset;
132	let index_entry = sorted_entries
133	.binary_search_by(\|e\| e.pack_offset.cmp(&pack_offset))
134	.expect("tree created by our sorted entries");
135	let index_entry_of_node = &sorted_entries[index_entry];
136
137	tree.children(node, &mut children);
138
139	let shared_cache = &mut SharedCache(&mut cache);
140	let mut stat = self.process_entry(
141	mode,
142	pack,
143	shared_cache,
144	buf,
145	encode_buf,
146	progress,
147	&mut header_buf,
148	index_entry_of_node,
149	)?;
150	stat.num_deltas = level;
151	stats.push(stat);
152
153	progress.inc();
154	nodes.extend(children.iter().cloned().map(\|cn\| (cn, level + 1)));
155	}
156	}
157
158	Ok(stats)
159	},
160	index::verify::Reducer::from_progress(&reduce_progress, pack.data_len()),
161	)
162	}
163	}

git-odb/src/pack/index/verify/lookup.rs+0−168

@@ -1,168 +0,0 @@
1	use super::{Error, Mode, Outcome, Reducer};
2	use crate::pack::{self, data::decode, index, index::util};
3	use git_features::{
4	parallel::{self, in_parallel_if},
5	progress::{self, Progress},
6	};
7	use git_object::{borrowed, bstr::ByteSlice, owned};
8
9	/// Verify and validate the content of the index file
10	impl index::File {
11	pub(crate) fn inner_verify_with_lookup<P, C>(
12	&self,
13	thread_limit: Option<usize>,
14	mode: Mode,
15	make_cache: impl Fn() -> C + Send + Sync,
16	mut root: progress::DoOrDiscard<P>,
17	pack: &pack::data::File,
18	) -> Result<Outcome, Error>
19	where
20	P: Progress,
21	<P as Progress>::SubProgress: Send,
22	C: pack::cache::DecodeEntry,
23	{
24	let index_entries =
25	util::index_entries_sorted_by_offset_ascending(self, root.add_child("collecting sorted index"));
26
27	let (chunk_size, thread_limit, available_cores) =
28	parallel::optimize_chunk_size_and_thread_limit(1000, Some(index_entries.len()), thread_limit, None);
29	let there_are_enough_entries_to_process = \|\| index_entries.len() > chunk_size * available_cores;
30	let input_chunks = index_entries.chunks(chunk_size.max(chunk_size));
31	let reduce_progress = std::sync::Mutex::new({
32	let mut p = root.add_child("Checking");
33	p.init(Some(self.num_objects()), Some("objects"));
34	p
35	});
36	let state_per_thread = \|index\| {
37	(
38	make_cache(),
39	Vec::with_capacity(2048), // decode buffer
40	Vec::with_capacity(2048), // re-encode buffer
41	reduce_progress.lock().unwrap().add_child(format!("thread {}", index)), // per thread progress
42	)
43	};
44
45	in_parallel_if(
46	there_are_enough_entries_to_process,
47	input_chunks,
48	thread_limit,
49	state_per_thread,
50	\|entries: &[index::Entry], (cache, buf, encode_buf, progress)\| -> Result<Vec<decode::Outcome>, Error> {
51	progress.init(Some(entries.len() as u32), Some("entries"));
52	let mut stats = Vec::with_capacity(entries.len());
53	let mut header_buf = [0u8; 64];
54	for index_entry in entries.iter() {
55	stats.push(self.process_entry(
56	mode,
57	pack,
58	cache,
59	buf,
60	encode_buf,
61	progress,
62	&mut header_buf,
63	index_entry,
64	)?);
65	progress.inc();
66	}
67	Ok(stats)
68	},
69	Reducer::from_progress(&reduce_progress, pack.data_len()),
70	)
71	}
72
73	#[allow(clippy::too_many_arguments)]
74	pub(crate) fn process_entry<C>(
75	&self,
76	mode: Mode,
77	pack: &pack::data::File,
78	cache: &mut C,
79	buf: &mut Vec<u8>,
80	encode_buf: &mut Vec<u8>,
81	progress: &mut impl Progress,
82	header_buf: &mut [u8; 64],
83	index_entry: &pack::index::Entry,
84	) -> Result<decode::Outcome, Error>
85	where
86	C: pack::cache::DecodeEntry,
87	{
88	let pack_entry = pack.entry(index_entry.pack_offset);
89	let pack_entry_data_offset = pack_entry.data_offset;
90	let entry_stats = pack
91	.decode_entry(
92	pack_entry,
93	buf,
94	\|id, _\| {
95	self.lookup_index(id).map(\|index\| {
96	pack::data::decode::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
97	})
98	},
99	cache,
100	)
101	.map_err(\|e\| Error::PackDecode(e, index_entry.oid, index_entry.pack_offset))?;
102	let object_kind = entry_stats.kind;
103	let consumed_input = entry_stats.compressed_size;
104
105	let header_size = crate::loose::object::header::encode(object_kind, buf.len() as u64, &mut header_buf[..])
106	.expect("header buffer to be big enough");
107	let mut hasher = git_features::hash::Sha1::default();
108	hasher.update(&header_buf[..header_size]);
109	hasher.update(buf.as_slice());
110
111	let actual_oid = owned::Id::new_sha1(hasher.digest());
112	if actual_oid != index_entry.oid {
113	return Err(Error::PackObjectMismatch {
114	actual: actual_oid,
115	expected: index_entry.oid,
116	offset: index_entry.pack_offset,
117	kind: object_kind,
118	});
119	}
120	if let Some(desired_crc32) = index_entry.crc32 {
121	let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
122	let actual_crc32 = pack.entry_crc32(index_entry.pack_offset, header_size + consumed_input);
123	if actual_crc32 != desired_crc32 {
124	return Err(Error::Crc32Mismatch {
125	actual: actual_crc32,
126	expected: desired_crc32,
127	offset: index_entry.pack_offset,
128	kind: object_kind,
129	});
130	}
131	}
132	if let Mode::Sha1CRC32Decode \| Mode::Sha1CRC32DecodeEncode = mode {
133	use git_object::Kind::*;
134	match object_kind {
135	Tree \| Commit \| Tag => {
136	let borrowed_object = borrowed::Object::from_bytes(object_kind, buf.as_slice())
137	.map_err(\|err\| Error::ObjectDecode(err, object_kind, index_entry.oid))?;
138	if let Mode::Sha1CRC32DecodeEncode = mode {
139	let object = owned::Object::from(borrowed_object);
140	encode_buf.clear();
141	object.write_to(&mut *encode_buf)?;
142	if encode_buf != buf {
143	let mut should_return_error = true;
144	if let git_object::Kind::Tree = object_kind {
145	if buf.as_slice().as_bstr().find(b"100664").is_some()
146	\|\| buf.as_slice().as_bstr().find(b"100640").is_some()
147	{
148	progress.info(format!("Tree object {} would be cleaned up during re-serialization, replacing mode '100664\|100640' with '100644'", index_entry.oid));
149	should_return_error = false
150	}
151	}
152	if should_return_error {
153	return Err(Error::ObjectEncodeMismatch(
154	object_kind,
155	index_entry.oid,
156	buf.clone().into(),
157	encode_buf.clone().into(),
158	));
159	}
160	}
161	}
162	}
163	Blob => {}
164	};
165	}
166	Ok(entry_stats)
167	}
168	}

git-odb/src/pack/index/verify/mod.rs+0−209

@@ -1,209 +0,0 @@
1	use crate::{pack, pack::data::decode, pack::index};
2	use git_features::{
3	parallel,
4	progress::{self, Progress},
5	};
6	use git_object::{borrowed, bstr::BString, owned, SHA1_SIZE};
7	use quick_error::quick_error;
8	use std::{collections::BTreeMap, time::Instant};
9
10	quick_error! {
11	#[derive(Debug)]
12	pub enum Error {
13	Io(err: std::io::Error, path: std::path::PathBuf, msg: &'static str) {
14	display("Failed to {} at path '{}'", msg, path.display())
15	source(err)
16	}
17	Graph(err: pack::graph::Error) {
18	from()
19	source(err)
20	}
21	Mismatch { expected: owned::Id, actual: owned::Id } {
22	display("index checksum mismatch: expected {}, got {}", expected, actual)
23	}
24	PackChecksum(err: pack::data::verify::Error) {
25	display("The pack of this index file failed to verify its checksums")
26	from()
27	source(err)
28	}
29	PackDecode(err: pack::data::decode::Error, id: owned::Id, offset: u64) {
30	display("Object {} at offset {} could not be decoded", id, offset)
31	source(err)
32	}
33	ObjectDecode(err: borrowed::Error, kind: git_object::Kind, oid: owned::Id) {
34	display("{} object {} could not be decoded", kind, oid)
35	source(err)
36	}
37	ObjectEncodeMismatch(kind: git_object::Kind, oid: owned::Id, expected: BString, actual: BString) {
38	display("{} object {} wasn't re-encoded without change, wanted\n{}\n\nGOT\n\n{}", kind, oid, expected, actual)
39	}
40	ObjectEncode(err: std::io::Error) {
41	from()
42	}
43	PackMismatch { expected: owned::Id, actual: owned::Id } {
44	display("The packfiles checksum didn't match the index file checksum: expected {}, got {}", expected, actual)
45	}
46	PackObjectMismatch { expected: owned::Id, actual: owned::Id, offset: u64, kind: git_object::Kind} {
47	display("The SHA1 of {} object at offset {} didn't match the checksum in the index file: expected {}, got {}", kind, offset, expected, actual)
48	}
49	Crc32Mismatch { expected: u32, actual: u32, offset: u64, kind: git_object::Kind} {
50	display("The CRC32 of {} object at offset {} didn't match the checksum in the index file: expected {}, got {}", kind, offset, expected, actual)
51	}
52	}
53	}
54
55	struct TimeThroughput {
56	then: Instant,
57	byte_size: usize,
58	}
59
60	impl TimeThroughput {
61	pub fn new(byte_size: usize) -> TimeThroughput {
62	TimeThroughput {
63	then: Instant::now(),
64	byte_size,
65	}
66	}
67	}
68
69	impl Into<String> for TimeThroughput {
70	fn into(self) -> String {
71	let time_taken = std::time::Instant::now().duration_since(self.then).as_secs_f32();
72	format!(
73	"finished in {:.2}s at {}/s",
74	time_taken,
75	bytesize::ByteSize((self.byte_size as f32 / time_taken) as u64)
76	)
77	}
78	}
79
80	#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)]
81	#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
82	pub struct Outcome {
83	pub average: decode::Outcome,
84	pub objects_per_chain_length: BTreeMap<u32, u32>,
85	/// The amount of bytes in all compressed streams, one per entry
86	pub total_compressed_entries_size: u64,
87	/// The amount of bytes in all decompressed streams, one per entry
88	pub total_decompressed_entries_size: u64,
89	/// The amount of bytes occupied by all undeltified, decompressed objects
90	pub total_object_size: u64,
91	/// The amount of bytes occupied by the pack itself, in bytes
92	pub pack_size: u64,
93	}
94
95	/// Various ways in which a pack and index can be verified
96	#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]
97	pub enum Mode {
98	/// Validate SHA1 and CRC32
99	Sha1CRC32,
100	/// Validate SHA1 and CRC32, and decode each non-Blob object
101	Sha1CRC32Decode,
102	/// Validate SHA1 and CRC32, and decode and encode each non-Blob object
103	Sha1CRC32DecodeEncode,
104	}
105
106	/// The way we verify the pack
107	#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]
108	pub enum Algorithm {
109	/// Build an index to allow decoding each delta and base exactly once, saving a lot of computational
110	/// resource at the expense of resident memory, as we will use an additional `DeltaTree` to accelerate
111	/// delta chain resolution.
112	DeltaTreeLookup,
113	/// We lookup each object similarly to what would happen during normal repository use.
114	/// Uses more compute resources as it will resolve delta chains from back to front, but start right away
115	/// without indexing or investing any memory in indices.
116	///
117	/// This option may be well suited for big packs in memory-starved system that support memory mapping.
118	Lookup,
119	}
120
121	impl Default for Algorithm {
122	fn default() -> Self {
123	Algorithm::DeltaTreeLookup
124	}
125	}
126
127	mod indexed;
128	mod lookup;
129	mod reduce;
130	pub(crate) use reduce::Reducer;
131
132	/// Verify and validate the content of the index file
133	impl index::File {
134	pub fn index_checksum(&self) -> owned::Id {
135	owned::Id::from_20_bytes(&self.data[self.data.len() - SHA1_SIZE..])
136	}
137
138	pub fn pack_checksum(&self) -> owned::Id {
139	let from = self.data.len() - SHA1_SIZE * 2;
140	owned::Id::from_20_bytes(&self.data[from..from + SHA1_SIZE])
141	}
142
143	pub fn verify_checksum(&self, mut progress: impl Progress) -> Result<owned::Id, Error> {
144	let throughput = TimeThroughput::new(self.data.len());
145	let mut hasher = git_features::hash::Sha1::default();
146	hasher.update(&self.data[..self.data.len() - SHA1_SIZE]);
147	let actual = owned::Id::new_sha1(hasher.digest());
148	progress.done(throughput);
149
150	let expected = self.index_checksum();
151	if actual == expected {
152	Ok(actual)
153	} else {
154	Err(Error::Mismatch { actual, expected })
155	}
156	}
157
158	/// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index.
159	/// It will be used to validate internal integrity of the pack before checking each objects integrity
160	/// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash.
161	/// redoing a lot of work across multiple objects.
162	pub fn verify_integrity<P, C>(
163	&self,
164	pack: Option<(&pack::data::File, Mode, Algorithm)>,
165	thread_limit: Option<usize>,
166	progress: Option<P>,
167	make_cache: impl Fn() -> C + Send + Sync,
168	) -> Result<(owned::Id, Option<Outcome>), Error>
169	where
170	P: Progress,
171	<P as Progress>::SubProgress: Send,
172	C: pack::cache::DecodeEntry,
173	{
174	let mut root = progress::DoOrDiscard::from(progress);
175
176	let progress = root.add_child("Sha1 of index");
177	let verify_self = move \|\| self.verify_checksum(progress);
178
179	match pack {
180	None => verify_self().map(\|id\| (id, None)),
181	Some((pack, mode, algorithm)) => {
182	if self.pack_checksum() != pack.checksum() {
183	return Err(Error::PackMismatch {
184	actual: pack.checksum(),
185	expected: self.pack_checksum(),
186	});
187	}
188	let mut progress = root.add_child("Sha1 of pack");
189	let (pack_res, id) = parallel::join(
190	move \|\| {
191	let throughput = TimeThroughput::new(pack.data_len());
192	let res = pack.verify_checksum();
193	progress.done(throughput);
194	res
195	},
196	verify_self,
197	);
198	pack_res?;
199	let id = id?;
200
201	match algorithm {
202	Algorithm::Lookup => self.inner_verify_with_lookup(thread_limit, mode, make_cache, root, pack),
203	Algorithm::DeltaTreeLookup => self.inner_verify_with_indexed_lookup(thread_limit, mode, root, pack),
204	}
205	.map(\|stats\| (id, Some(stats)))
206	}
207	}
208	}
209	}

git-odb/src/pack/index/verify/reduce.rs+0−95

@@ -1,95 +0,0 @@
1	use crate::pack::{data::decode, index::verify};
2	use git_features::{parallel, progress::Progress};
3	use std::time::Instant;
4
5	fn add_decode_result(lhs: &mut decode::Outcome, rhs: decode::Outcome) {
6	lhs.num_deltas += rhs.num_deltas;
7	lhs.decompressed_size += rhs.decompressed_size;
8	lhs.compressed_size += rhs.compressed_size;
9	lhs.object_size += rhs.object_size;
10	}
11
12	fn div_decode_result(lhs: &mut decode::Outcome, div: usize) {
13	if div != 0 {
14	lhs.num_deltas = (lhs.num_deltas as f32 / div as f32) as u32;
15	lhs.decompressed_size /= div as u64;
16	lhs.compressed_size /= div;
17	lhs.object_size /= div as u64;
18	}
19	}
20
21	pub struct Reducer<'a, P> {
22	pub progress: &'a std::sync::Mutex<P>,
23	pub then: Instant,
24	pub entries_seen: u32,
25	pub stats: verify::Outcome,
26	}
27
28	impl<'a, P> Reducer<'a, P>
29	where
30	P: Progress,
31	{
32	pub fn from_progress(progress: &'a std::sync::Mutex<P>, pack_data_len_in_bytes: usize) -> Self {
33	Reducer {
34	progress: &progress,
35	then: Instant::now(),
36	entries_seen: 0,
37	stats: verify::Outcome {
38	average: decode::Outcome::default_from_kind(git_object::Kind::Tree),
39	objects_per_chain_length: Default::default(),
40	total_compressed_entries_size: 0,
41	total_decompressed_entries_size: 0,
42	total_object_size: 0,
43	pack_size: pack_data_len_in_bytes as u64,
44	},
45	}
46	}
47	}
48
49	impl<'a, P> parallel::Reducer for Reducer<'a, P>
50	where
51	P: Progress,
52	{
53	type Input = Result<Vec<decode::Outcome>, verify::Error>;
54	type Output = verify::Outcome;
55	type Error = verify::Error;
56
57	fn feed(&mut self, input: Self::Input) -> Result<(), Self::Error> {
58	let chunk_stats: Vec<_> = input?;
59	self.entries_seen += chunk_stats.len() as u32;
60
61	let chunk_total = chunk_stats.into_iter().fold(
62	decode::Outcome::default_from_kind(git_object::Kind::Tree),
63	\|mut total, stats\| {
64	*self.stats.objects_per_chain_length.entry(stats.num_deltas).or_insert(0) += 1;
65	self.stats.total_decompressed_entries_size += stats.decompressed_size;
66	self.stats.total_compressed_entries_size += stats.compressed_size as u64;
67	self.stats.total_object_size += stats.object_size as u64;
68	add_decode_result(&mut total, stats);
69	total
70	},
71	);
72
73	add_decode_result(&mut self.stats.average, chunk_total);
74	self.progress.lock().unwrap().set(self.entries_seen);
75	Ok(())
76	}
77
78	fn finalize(mut self) -> Result<Self::Output, Self::Error> {
79	self.progress.lock().unwrap().done("finished");
80
81	div_decode_result(&mut self.stats.average, self.entries_seen as usize);
82
83	let elapsed_s = self.then.elapsed().as_secs_f32();
84	let objects_per_second = (self.entries_seen as f32 / elapsed_s) as u32;
85
86	self.progress.lock().unwrap().info(format!(
87	"Verified {} objects in {:.2}s ({} objects/s, ~{}/s)",
88	self.entries_seen,
89	elapsed_s,
90	objects_per_second,
91	bytesize::ByteSize(self.stats.average.object_size * objects_per_second as u64)
92	));
93	Ok(self.stats)
94	}
95	}

git-odb/tests/pack/index.rs+6−6

@@ -73,9 +73,9 @@ use common_macros::b_tree_map; · 72 unmodified lines
73	use git_features::progress::Discard;	73	use git_features::progress::Discard;
74	use git_odb::pack::cache::DecodeEntryNoop;	74	use git_odb::pack::cache::DecodeEntryNoop;
75		75
76	static ALGOS: &[index::verify::Algorithm] = &[	76	static ALGOS: &[index::traverse::Algorithm] = &[
77	index::verify::Algorithm::Lookup,	77	index::traverse::Algorithm::Lookup,
78	index::verify::Algorithm::DeltaTreeLookup,	78	index::traverse::Algorithm::DeltaTreeLookup,
79	];	79	];
80		80
81	static MODES: &[index::verify::Mode] = &[	81	static MODES: &[index::verify::Mode] = &[
@@ -90,7 +90,7 @@ fn pack_lookup() { · 8 unmodified lines
90	(	90	(
91	INDEX_V2,	91	INDEX_V2,
92	PACK_FOR_INDEX_V2,	92	PACK_FOR_INDEX_V2,
93	index::verify::Outcome {	93	index::traverse::Outcome {
94	average: Outcome {	94	average: Outcome {
95	kind: object::Kind::Tree,	95	kind: object::Kind::Tree,
96	num_deltas: 1,	96	num_deltas: 1,
@@ -116,7 +116,7 @@ fn pack_lookup() { · 19 unmodified lines
116	(	116	(
117	INDEX_V1,	117	INDEX_V1,
118	PACK_FOR_INDEX_V1,	118	PACK_FOR_INDEX_V1,
119	index::verify::Outcome {	119	index::traverse::Outcome {
120	average: Outcome {	120	average: Outcome {
121	kind: object::Kind::Tree,	121	kind: object::Kind::Tree,
122	num_deltas: 0,	122	num_deltas: 0,
@@ -137,7 +137,7 @@ fn pack_lookup() { · 14 unmodified lines
137	(	137	(
138	SMALL_PACK_INDEX,	138	SMALL_PACK_INDEX,
139	SMALL_PACK,	139	SMALL_PACK,
140	index::verify::Outcome {	140	index::traverse::Outcome {
141	average: Outcome {	141	average: Outcome {
142	kind: object::Kind::Tree,	142	kind: object::Kind::Tree,
143	num_deltas: 0,	143	num_deltas: 0,

gitoxide-core/src/pack/verify.rs+8−6

     }
 }
-impl From<Algorithm> for index::verify::Algorithm {
+impl From<Algorithm> for index::traverse::Algorithm {
     fn from(v: Algorithm) -> Self {
         match v {
-            Algorithm::LessMemory => index::verify::Algorithm::Lookup,
+            Algorithm::LessMemory => index::traverse::Algorithm::Lookup,
-            Algorithm::LessTime => index::verify::Algorithm::DeltaTreeLookup,
+            Algorithm::LessTime => index::traverse::Algorithm::DeltaTreeLookup,
         }
     }
 }
         thread_limit,
         algorithm,
     }: Context<W1, W2>,
-) -> Result<(owned::Id, Option<index::verify::Outcome>)>
+) -> Result<(owned::Id, Option<index::traverse::Outcome>)>
 where
     P: Progress,
     <P as Progress>::SubProgress: Send,
     W1: io::Write,
     W2: io::Write,
+    <<P as git_features::progress::Progress>::SubProgress as git_features::progress::Progress>::SubProgress:
+        std::marker::Send,
 {
     let path = path.as_ref();
     let ext = path.extension().and_then(|ext| ext.to_str()).ok_or_else(|| {
                 progress,
                 cache,
             )
-            .with_context(|| "Verification failure")?
+            .unwrap() // FIXME
         }
         ext => return Err(anyhow!("Unknown extension {:?}, expecting 'idx' or 'pack'", ext)),
     };
     Ok(res)
 }
-fn print_statistics(out: &mut impl io::Write, stats: &index::verify::Outcome) -> io::Result<()> {
+fn print_statistics(out: &mut impl io::Write, stats: &index::traverse::Outcome) -> io::Result<()> {
     writeln!(out, "objects per delta chain length")?;
     let mut chain_length_to_object: Vec<_> = stats.objects_per_chain_length.iter().map(|(a, b)| (*a, *b)).collect();
     chain_length_to_object.sort_by_key(|e| e.0);

@@ -39,35 +39,35 @@ pub enum Mode { · 38 unmodified lines
39		39
40	/// Verify and validate the content of the index file	40	/// Verify and validate the content of the index file
41	impl index::File {	41	impl index::File {
42	// pub fn index_checksum(&self) -> owned::Id {	42	pub fn index_checksum(&self) -> owned::Id {
43	// owned::Id::from_20_bytes(&self.data[self.data.len() - SHA1_SIZE..])	43	owned::Id::from_20_bytes(&self.data[self.data.len() - SHA1_SIZE..])
44	// }	44	}
45	//	45
46	// pub fn pack_checksum(&self) -> owned::Id {	46	pub fn pack_checksum(&self) -> owned::Id {
47	// let from = self.data.len() - SHA1_SIZE * 2;	47	let from = self.data.len() - SHA1_SIZE * 2;
48	// owned::Id::from_20_bytes(&self.data[from..from + SHA1_SIZE])	48	owned::Id::from_20_bytes(&self.data[from..from + SHA1_SIZE])
49	// }	49	}
50	//	50
51	// pub fn verify_checksum(&self, mut progress: impl Progress) -> Result<owned::Id, Error> {	51	pub fn verify_checksum(&self, mut progress: impl Progress) -> Result<owned::Id, Error> {
52	// let throughput = index::util::TimeThroughput::new(self.data.len());	52	let throughput = index::util::TimeThroughput::new(self.data.len());
53	// let mut hasher = git_features::hash::Sha1::default();	53	let mut hasher = git_features::hash::Sha1::default();
54	// hasher.update(&self.data[..self.data.len() - SHA1_SIZE]);	54	hasher.update(&self.data[..self.data.len() - SHA1_SIZE]);
55	// let actual = owned::Id::new_sha1(hasher.digest());	55	let actual = owned::Id::new_sha1(hasher.digest());
56	// progress.done(throughput);	56	progress.done(throughput);
57	//	57
58	// let expected = self.index_checksum();	58	let expected = self.index_checksum();
59	// if actual == expected {	59	if actual == expected {
60	// Ok(actual)	60	Ok(actual)
61	// } else {	61	} else {
62	// Err(Error::Mismatch { actual, expected })	62	Err(Error::Mismatch { actual, expected })
63	// }	63	}
64	// }	64	}
65		65
66	/// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index.	66	/// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index.
67	/// It will be used to validate internal integrity of the pack before checking each objects integrity	67	/// It will be used to validate internal integrity of the pack before checking each objects integrity
68	/// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash.	68	/// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash.
69	/// redoing a lot of work across multiple objects.	69	/// redoing a lot of work across multiple objects.
70	pub fn verify_integrity_2<P, C>(	70	pub fn verify_integrity<P, C>(
71	&self,	71	&self,
72	pack: Option<(&pack::data::File, Mode, index::traverse::Algorithm)>,	72	pack: Option<(&pack::data::File, Mode, index::traverse::Algorithm)>,
73	thread_limit: Option<usize>,	73	thread_limit: Option<usize>,