use std::{ collections::VecDeque, sync::{ atomic::{AtomicBool, Ordering}, Arc, }, }; use git_features::{parallel, progress::Progress}; use super::{Error, SafetyCheck}; use crate::{ index::{self, util::index_entries_sorted_by_offset_ascending}, tree::traverse::Context, }; /// Traversal with index impl index::File { /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor`, using an index to reduce waste /// at the cost of memory. /// /// For more details, see the documentation on the [`traverse()`][index::File::traverse()] method. pub fn traverse_with_index
(
&self,
check: SafetyCheck,
thread_limit: Option ::SubProgress as Progress>::SubProgress,
) -> Result<(), E>,
E: std::error::Error + Send + Sync + 'static,
{
let (verify_result, traversal_result) = parallel::join(
{
let pack_progress = progress.add_child("SHA1 of pack");
let index_progress = progress.add_child("SHA1 of index");
let should_interrupt = Arc::clone(&should_interrupt);
move || {
let res = self.possibly_verify(
pack,
check,
pack_progress,
index_progress,
Arc::clone(&should_interrupt),
);
if res.is_err() {
should_interrupt.store(true, Ordering::SeqCst);
}
res
}
},
|| -> Result<_, Error<_>> {
let sorted_entries =
index_entries_sorted_by_offset_ascending(self, progress.add_child("collecting sorted index"));
let tree = crate::tree::Tree::from_offsets_in_pack(
sorted_entries.into_iter().map(EntryWithDefault::from),
|e| e.index_entry.pack_offset,
pack.path(),
progress.add_child("indexing"),
&should_interrupt,
|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
)?;
let there_are_enough_objects = || self.num_objects > 10_000;
let mut outcome = digest_statistics(tree.traverse(
there_are_enough_objects,
|slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)),
progress.add_child("Resolving"),
progress.add_child("Decoding"),
thread_limit,
&should_interrupt,
pack.pack_end() as u64,
|| (new_processor(), [0u8; 64]),
|data,
progress,
Context {
entry: pack_entry,
entry_end,
decompressed: bytes,
state: (ref mut processor, ref mut header_buf),
level,
}| {
let object_kind = pack_entry.header.as_kind().expect("non-delta object");
data.level = level;
data.decompressed_size = pack_entry.decompressed_size;
data.object_kind = object_kind;
data.compressed_size = entry_end - pack_entry.data_offset;
data.object_size = bytes.len() as u64;
let result = crate::index::traverse::process_entry(
check,
object_kind,
bytes,
progress,
header_buf,
&data.index_entry,
|| {
// TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size,
// causing index and pack to seemingly mismatch. This is surprising, and should be done differently.
// debug_assert_eq!(&data.index_entry.pack_offset, &pack_entry.pack_offset());
git_features::hash::crc32(
pack.entry_slice(data.index_entry.pack_offset..entry_end)
.expect("slice pointing into the pack (by now data is verified)"),
)
},
processor,
);
match result {
Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => {
progress.info(format!("Ignoring decode error: {}", err));
Ok(())
}
res => res,
}
},
)?);
outcome.pack_size = pack.data_len() as u64;
Ok(outcome)
},
);
let id = verify_result?;
let outcome = traversal_result?;
Ok((id, outcome, progress))
}
}
struct EntryWithDefault {
index_entry: crate::index::Entry,
object_kind: git_object::Kind,
object_size: u64,
decompressed_size: u64,
compressed_size: u64,
level: u16,
}
impl Default for EntryWithDefault {
fn default() -> Self {
EntryWithDefault {
index_entry: crate::index::Entry {
pack_offset: 0,
crc32: None,
oid: git_hash::ObjectId::null_sha1(),
},
level: 0,
object_kind: git_object::Kind::Tree,
object_size: 0,
decompressed_size: 0,
compressed_size: 0,
}
}
}
impl From