use crate::pack::{self, index};use git_features::{ parallel, progress::{self, Progress},};use git_object::owned; ///mod indexed;mod lookup;mod reduce;pub(crate) use reduce::Reducer; mod error;pub use error::Error; mod types;pub use types::{Algorithm, Options, Outcome, SafetyCheck}; /// Traversal of pack data files using an index fileimpl index::File { /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor`. /// The return value is (pack-checksum, [`Outcome`], `progress`), thus the pack traversal will always verify /// the whole packs checksum to assure it was correct. In case of bit-rod, the operation will abort early without /// verifying all objects using the [interrupt mechanism][git_features::interrupt] mechanism. /// /// # Algorithms /// /// Using the [`Options::algorithm`] field one can chose between two algorithms providing different tradeoffs. Both invoke /// `new_processor()` to create functions receiving decoded objects, their object kind, index entry and a progress instance to provide /// progress information. /// /// * [`Algorithm::DeltaTreeLookup`] builds an index to avoid any unnecessary computation while resolving objects, avoiding /// the need for a cache entirely, rendering `new_cache()` unused. /// One could also call [`traverse_with_index()`][index::File::traverse_with_index()] directly. /// * [`Algorithm::Lookup`] uses a cache created by `new_cache()` to avoid having to re-compute all bases of a delta-chain while /// decoding objects. /// One could also call [`traverse_with_lookup()`][index::File::traverse_with_lookup()] directly. /// /// Use [`thread_limit`][Options::thread_limit] to further control parallelism and [`check`][SafetyCheck] to define how much the passed /// objects shall be verified beforehand. pub fn traverse<P, C, Processor, E>( &self, pack: &pack::data::File, progress: Option<P>, new_processor: impl Fn() -> Processor + Send + Sync, new_cache: impl Fn() -> C + Send + Sync, Options { algorithm, thread_limit, check, }: Options, ) -> Result<(owned::Id, Outcome, Option<P>), Error<E>> where P: Progress, C: pack::cache::DecodeEntry, E: std::error::Error + Send + Sync + 'static, Processor: FnMut( git_object::Kind, &[u8], &index::Entry, &mut progress::DoOrDiscard<<<P as Progress>::SubProgress as Progress>::SubProgress>, ) -> Result<(), E>, { let progress = progress::DoOrDiscard::from(progress); match algorithm { Algorithm::Lookup => { self.traverse_with_lookup(check, thread_limit, new_processor, new_cache, progress, pack) } Algorithm::DeltaTreeLookup => self.traverse_with_index(check, thread_limit, new_processor, progress, pack), } .map(|(a, b, p)| (a, b, p.into_inner())) } pub(crate) fn possibly_verify<E>( &self, pack: &pack::data::File, check: SafetyCheck, pack_progress: impl Progress, index_progress: impl Progress, ) -> Result<owned::Id, Error<E>> where E: std::error::Error + Send + Sync + 'static, { Ok(if check.file_checksum() { if self.pack_checksum() != pack.checksum() { return Err(Error::PackMismatch { actual: pack.checksum(), expected: self.pack_checksum(), }); } let (pack_res, id) = parallel::join( move || pack.verify_checksum(pack_progress), move || self.verify_checksum(index_progress), ); pack_res?; id? } else { self.index_checksum() }) } #[allow(clippy::too_many_arguments)] pub(crate) fn decode_and_process_entry<C, P, E>( &self, check: SafetyCheck, pack: &pack::data::File, cache: &mut C, buf: &mut Vec<u8>, progress: &mut P, header_buf: &mut [u8; 64], index_entry: &pack::index::Entry, processor: &mut impl FnMut(git_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>, ) -> Result<pack::data::decode::Outcome, Error<E>> where C: pack::cache::DecodeEntry, P: Progress, E: std::error::Error + Send + Sync + 'static, { let pack_entry = pack.entry(index_entry.pack_offset); let pack_entry_data_offset = pack_entry.data_offset; let entry_stats = pack .decode_entry( pack_entry, buf, |id, _| { self.lookup(id).map(|index| { pack::data::decode::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index))) }) }, cache, ) .map_err(|e| Error::PackDecode { source: e, id: index_entry.oid, offset: index_entry.pack_offset, })?; let object_kind = entry_stats.kind; let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize; let entry_len = header_size + entry_stats.compressed_size; process_entry( check, object_kind, &buf, progress, header_buf, index_entry, || pack.entry_crc32(index_entry.pack_offset, entry_len), processor, )?; Ok(entry_stats) }} #[allow(clippy::too_many_arguments)]pub(crate) fn process_entry<P, E>( check: SafetyCheck, object_kind: git_object::Kind, decompressed: &[u8], progress: &mut P, header_buf: &mut [u8; 64], index_entry: &pack::index::Entry, pack_entry_crc32: impl FnOnce() -> u32, processor: &mut impl FnMut(git_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>,) -> Result<(), Error<E>>where P: Progress, E: std::error::Error + Send + Sync + 'static,{ if check.object_checksum() { let header_size = crate::loose::object::header::encode(object_kind, decompressed.len() as u64, &mut header_buf[..]) .expect("header buffer to be big enough"); let mut hasher = git_features::hash::Sha1::default(); hasher.update(&header_buf[..header_size]); hasher.update(decompressed); let actual_oid = owned::Id::new_sha1(hasher.digest()); if actual_oid != index_entry.oid { return Err(Error::PackObjectMismatch { actual: actual_oid, expected: index_entry.oid, offset: index_entry.pack_offset, kind: object_kind, }); } if let Some(desired_crc32) = index_entry.crc32 { let actual_crc32 = pack_entry_crc32(); if actual_crc32 != desired_crc32 { return Err(Error::Crc32Mismatch { actual: actual_crc32, expected: desired_crc32, offset: index_entry.pack_offset, kind: object_kind, }); } } } processor(object_kind, decompressed, &index_entry, progress).map_err(Error::Processor)}
use crate::pack::{self, index};
use git_features::{
parallel,
progress::{self, Progress},
};
use git_object::owned;
///
mod indexed;
mod lookup;
mod reduce;
pub(crate) use reduce::Reducer;
mod error;
pub use error::Error;
mod types;
pub use types::{Algorithm, Options, Outcome, SafetyCheck};
/// Traversal of pack data files using an index file
impl index::File {
/// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor`.
/// The return value is (pack-checksum, [`Outcome`], `progress`), thus the pack traversal will always verify
/// the whole packs checksum to assure it was correct. In case of bit-rod, the operation will abort early without
/// verifying all objects using the [interrupt mechanism][git_features::interrupt] mechanism.
///
/// # Algorithms
///
/// Using the [`Options::algorithm`] field one can chose between two algorithms providing different tradeoffs. Both invoke
/// `new_processor()` to create functions receiving decoded objects, their object kind, index entry and a progress instance to provide
/// progress information.
///
/// * [`Algorithm::DeltaTreeLookup`] builds an index to avoid any unnecessary computation while resolving objects, avoiding
/// the need for a cache entirely, rendering `new_cache()` unused.
/// One could also call [`traverse_with_index()`][index::File::traverse_with_index()] directly.
/// * [`Algorithm::Lookup`] uses a cache created by `new_cache()` to avoid having to re-compute all bases of a delta-chain while
/// decoding objects.
/// One could also call [`traverse_with_lookup()`][index::File::traverse_with_lookup()] directly.
///
/// Use [`thread_limit`][Options::thread_limit] to further control parallelism and [`check`][SafetyCheck] to define how much the passed
/// objects shall be verified beforehand.
pub fn traverse<P, C, Processor, E>(
&self,
pack: &pack::data::File,
progress: Option<P>,
new_processor: impl Fn() -> Processor + Send + Sync,
new_cache: impl Fn() -> C + Send + Sync,
Options {
algorithm,
thread_limit,
check,
}: Options,
) -> Result<(owned::Id, Outcome, Option<P>), Error<E>>
where
P: Progress,
C: pack::cache::DecodeEntry,
E: std::error::Error + Send + Sync + 'static,
Processor: FnMut(
git_object::Kind,
&[u8],
&index::Entry,
&mut progress::DoOrDiscard<<<P as Progress>::SubProgress as Progress>::SubProgress>,
) -> Result<(), E>,
{
let progress = progress::DoOrDiscard::from(progress);
match algorithm {
Algorithm::Lookup => {
self.traverse_with_lookup(check, thread_limit, new_processor, new_cache, progress, pack)
}
Algorithm::DeltaTreeLookup => self.traverse_with_index(check, thread_limit, new_processor, progress, pack),
}
.map(|(a, b, p)| (a, b, p.into_inner()))
}
pub(crate) fn possibly_verify<E>(
&self,
pack: &pack::data::File,
check: SafetyCheck,
pack_progress: impl Progress,
index_progress: impl Progress,
) -> Result<owned::Id, Error<E>>
where
E: std::error::Error + Send + Sync + 'static,
{
Ok(if check.file_checksum() {
if self.pack_checksum() != pack.checksum() {
return Err(Error::PackMismatch {
actual: pack.checksum(),
expected: self.pack_checksum(),
});
}
let (pack_res, id) = parallel::join(
move || pack.verify_checksum(pack_progress),
move || self.verify_checksum(index_progress),
);
pack_res?;
id?
} else {
self.index_checksum()
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn decode_and_process_entry<C, P, E>(
&self,
check: SafetyCheck,
pack: &pack::data::File,
cache: &mut C,
buf: &mut Vec<u8>,
progress: &mut P,
header_buf: &mut [u8; 64],
index_entry: &pack::index::Entry,
processor: &mut impl FnMut(git_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>,
) -> Result<pack::data::decode::Outcome, Error<E>>
where
C: pack::cache::DecodeEntry,
P: Progress,
E: std::error::Error + Send + Sync + 'static,
{
let pack_entry = pack.entry(index_entry.pack_offset);
let pack_entry_data_offset = pack_entry.data_offset;
let entry_stats = pack
.decode_entry(
pack_entry,
buf,
|id, _| {
self.lookup(id).map(|index| {
pack::data::decode::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
})
},
cache,
)
.map_err(|e| Error::PackDecode {
source: e,
id: index_entry.oid,
offset: index_entry.pack_offset,
})?;
let object_kind = entry_stats.kind;
let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize;
let entry_len = header_size + entry_stats.compressed_size;
process_entry(
check,
object_kind,
&buf,
progress,
header_buf,
index_entry,
|| pack.entry_crc32(index_entry.pack_offset, entry_len),
processor,
)?;
Ok(entry_stats)
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn process_entry<P, E>(
check: SafetyCheck,
object_kind: git_object::Kind,
decompressed: &[u8],
progress: &mut P,
header_buf: &mut [u8; 64],
index_entry: &pack::index::Entry,
pack_entry_crc32: impl FnOnce() -> u32,
processor: &mut impl FnMut(git_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>,
) -> Result<(), Error<E>>
where
P: Progress,
E: std::error::Error + Send + Sync + 'static,
{
if check.object_checksum() {
let header_size =
crate::loose::object::header::encode(object_kind, decompressed.len() as u64, &mut header_buf[..])
.expect("header buffer to be big enough");
let mut hasher = git_features::hash::Sha1::default();
hasher.update(&header_buf[..header_size]);
hasher.update(decompressed);
let actual_oid = owned::Id::new_sha1(hasher.digest());
if actual_oid != index_entry.oid {
return Err(Error::PackObjectMismatch {
actual: actual_oid,
expected: index_entry.oid,
offset: index_entry.pack_offset,
kind: object_kind,
});
}
if let Some(desired_crc32) = index_entry.crc32 {
let actual_crc32 = pack_entry_crc32();
if actual_crc32 != desired_crc32 {
return Err(Error::Crc32Mismatch {
actual: actual_crc32,
expected: desired_crc32,
offset: index_entry.pack_offset,
kind: object_kind,
});
}
}
}
processor(object_kind, decompressed, &index_entry, progress).map_err(Error::Processor)
}