computing hashes for bases from decompressed in-memory store works

Sebastian Thiel committed 6 years ago

Commit 7c19fe61 parent a95ce9c3 files · +135 −62Browse files

Unified · Split

3 files

git-odb/src/pack/index/write/mod.rs +64 −17
git-odb/src/pack/index/write/types.rs +31 −7
git-odb/tests/pack/index.rs +40 −38

git-odb/src/pack/index/write/mod.rs+64−17

-use crate::{hash, pack, pack::index::util::Chunks, pack::index::V2_SIGNATURE};
+use crate::{hash, loose, pack, pack::index::util::Chunks, pack::index::V2_SIGNATURE};
 use byteorder::{BigEndian, WriteBytesExt};
 use git_features::{parallel, parallel::in_parallel_if, progress::Progress};
-use git_object::owned;
+use git_object::{owned, HashKind};
 use smallvec::alloc::collections::BTreeMap;
 use std::io;
     {
         use io::Write;
-        let mut out = hash::Write::new(out, kind.hash());
         if kind != pack::index::Kind::default() {
             return Err(Error::Unsupported(kind));
         }
                         .ok_or_else(|| {
                             Error::IteratorInvariantBasesBeforeDeltasNeedThem(pack_offset, base_pack_offset)
                         })?
-                        .child_count += 1;
+                        .increment_child_count();
                     (
                         mode.delta_cache(compressed, decompressed),
                         ObjectKind::OfsDelta(base_pack_offset),
                 }
             };
-            cache_by_offset.insert(
+            cache_by_offset.insert(pack_offset, CacheEntry::new(cache));
-                pack_offset,
-                CacheEntry {
-                    child_count: 0,
-                    _cache: cache,
-                },
-            );
             index_entries.push(Entry {
                 pack_offset,
                 kind,
                 thread_limit,
                 |_| (),
                 |base_pack_offsets, state| {
-                    apply_deltas(base_pack_offsets, state, &index_entries, &_cache_by_offset, &mode)
+                    apply_deltas(
+                        base_pack_offsets,
+                        state,
+                        &index_entries,
+                        &_cache_by_offset,
+                        &mode,
+                        kind.hash(),
+                    )
                 },
                 Reducer::new(num_objects),
             )?;
         drop(index_entries);
         // Write header
+        let mut out = hash::Write::new(out, kind.hash());
         out.write_all(V2_SIGNATURE)?;
         out.write_u32::<BigEndian>(kind as u32)?;
         // todo: write fanout
-        let _index_hash = out.hash.digest();
+        let index_hash = out.hash.digest().into();
         Ok(Outcome {
             index_kind: kind,
-            index_hash: owned::Id::from([0u8; 20]),
+            index_hash,
             pack_hash: last_seen_trailer.ok_or(Error::IteratorInvariantTrailer)?,
             num_objects,
         })
 }
 fn apply_deltas<F>(
-    _base_entries: Vec<&Entry>,
+    base_entries: Vec<&Entry>,
     _state: &mut (),
     _entries: &[Entry],
-    _caches: &parking_lot::Mutex<BTreeMap<u64, CacheEntry>>,
+    caches: &parking_lot::Mutex<BTreeMap<u64, CacheEntry>>,
     _mode: &Mode<F>,
+    hash_kind: HashKind,
 ) -> Vec<(u64, owned::Id)>
 where
     F: for<'r> Fn(u64, &'r mut Vec<u8>) -> Option<(pack::data::Header, u64)> + Send + Sync,
 {
-    Vec::new()
+    let decompressed_bytes_from_cache = |pack_offset: &u64| -> (bool, Vec<u8>) {
+        let cache = caches
+            .lock()
+            .get_mut(pack_offset)
+            .expect("an entry for every pack offset")
+            .cache();
+        let (is_borrowed, cache) = match cache {
+            Bytes::Borrowed(b) => (true, b),
+            Bytes::Owned(b) => (false, b),
+        };
+        let bytes = match cache {
+            Cache::Decompressed(b) => b,
+            Cache::Compressed(_, _) => unimplemented!("decompress once"),
+            Cache::Unset => unimplemented!("use resolver"),
+        };
+        (is_borrowed, bytes)
+    };
+    let possibly_return_to_cache = |pack_offset: &u64, is_borrowed: bool, bytes: Vec<u8>| {
+        if is_borrowed {
+            caches
+                .lock()
+                .get_mut(pack_offset)
+                .expect("an entry for every pack offset")
+                .set_decompressed(bytes);
+        }
+    };
+    let compute_hash = |kind: git_object::Kind, bytes: &[u8]| -> owned::Id {
+        let mut write = hash::Write::new(io::sink(), hash_kind);
+        loose::object::header::encode(kind, bytes.len() as u64, &mut write)
+            .expect("write to sink and hash cannot fail");
+        write.hash.update(bytes);
+        owned::Id::from(write.hash.digest())
+    };
+    let mut out = Vec::with_capacity(base_entries.len()); // perfectly conservative guess
+    for Entry { pack_offset, kind, .. } in base_entries {
+        let (is_borrowed, base_bytes) = decompressed_bytes_from_cache(pack_offset);
+        out.push((
+            *pack_offset,
+            compute_hash(kind.to_kind().expect("base object"), &base_bytes),
+        ));
+        possibly_return_to_cache(pack_offset, is_borrowed, base_bytes);
+    }
+    out.shrink_to_fit();
+    out
 }

git-odb/src/pack/index/write/types.rs+31−7

             ObjectKind::OfsDelta(_) => false,
         }
     }
+    pub fn to_kind(&self) -> Option<git_object::Kind> {
+        match self {
+            ObjectKind::Base(kind) => Some(*kind),
+            ObjectKind::OfsDelta(_) => None,
+        }
+    }
 }
 pub(crate) struct Entry {
 }
 pub(crate) struct CacheEntry {
-    pub _cache: Cache,
+    cache: Cache,
     /// When it reaches zero, the cache can be freed
-    pub child_count: u32,
+    child_count: u32,
 }
-pub(crate) enum _Bytes {
+pub(crate) enum Bytes {
     Owned(Cache),
     Borrowed(Cache),
 }
+/// Note that every operation in the CacheEntry must be fast, as these happen behind a lock
 impl CacheEntry {
-    pub fn _decr(&mut self) -> _Bytes {
+    pub fn new(cache: Cache) -> Self {
+        CacheEntry {
+            child_count: 0,
+            cache: cache,
+        }
+    }
+    pub fn increment_child_count(&mut self) {
+        self.child_count += 1;
+    }
+    pub fn _decr(&mut self) -> Bytes {
         self.child_count -= 1;
-        let cache = std::mem::replace(&mut self._cache, Cache::Unset);
+        self.cache()
+    }
+    pub fn cache(&mut self) -> Bytes {
+        let cache = std::mem::replace(&mut self.cache, Cache::Unset);
         if self.child_count == 0 {
-            _Bytes::Owned(cache)
+            Bytes::Owned(cache)
         } else {
-            _Bytes::Borrowed(cache)
+            Bytes::Borrowed(cache)
         }
     }
+    pub fn set_decompressed(&mut self, bytes: Vec<u8>) {
+        assert_ne!(self.child_count, 0, "Do not return decompressed bytes once nobody is interested in the data anymore, i.e. from `Bytes::Owned(…)`");
+        self.cache = Cache::Decompressed(bytes);
+    }
 }
 /// The function resolves pack_offset: u64 into compressed bytes to &mut Vec<u8> and returns (object kind, decompressed size)

git-odb/tests/pack/index.rs+40−38

@@ -77,46 +77,48 @@ mod method { · 76 unmodified lines
77	fn write_to_stream() -> Result<(), Box<dyn std::error::Error>> {	77	fn write_to_stream() -> Result<(), Box<dyn std::error::Error>> {
78	for mode in &[Mode::AsIs, Mode::Verify, Mode::Restore] {	78	for mode in &[Mode::AsIs, Mode::Verify, Mode::Restore] {
79	for (index_path, data_path) in V2_PACKS_AND_INDICES {	79	for (index_path, data_path) in V2_PACKS_AND_INDICES {
80	let pack_iter = pack::data::Iter::new_from_header(	80	for memory_mode in vec![pack::index::write::Mode::in_memory_decompressed()] {
81	io::BufReader::new(fs::File::open(fixture_path(data_path))?),	81	let pack_iter = pack::data::Iter::new_from_header(
82	*mode,	82	io::BufReader::new(fs::File::open(fixture_path(data_path))?),
83	)?;	83	*mode,
		84	)?;
84		85
85	let mut actual = Vec::<u8>::new();	86	let mut actual = Vec::<u8>::new();
86	let desired_kind = pack::index::Kind::default();	87	let desired_kind = pack::index::Kind::default();
87	let num_objects = pack_iter.len() as u32;	88	let num_objects = pack_iter.len() as u32;
88	let outcome = pack::index::File::write_data_iter_to_stream(	89	let outcome = pack::index::File::write_data_iter_to_stream(
89	desired_kind,	90	desired_kind,
90	pack::index::write::Mode::in_memory_decompressed(),	91	memory_mode,
91	pack_iter,	92	pack_iter,
92	None,	93	None,
93	progress::Discard,	94	progress::Discard,
94	&mut actual,	95	&mut actual,
95	)?;	96	)?;
96		97
97	let expected = fs::read(fixture_path(index_path))?;	98	let expected = fs::read(fixture_path(index_path))?;
98	let end_of_header = 4 * 2;	99	let end_of_header = 4 * 2;
99	assert_eq!(actual, &expected[..end_of_header], "we should get the header right");	100	assert_eq!(actual, &expected[..end_of_header], "we should get the header right");
100	// let end_of_fanout_table = end_of_header + 256 * 4;	101	// let end_of_fanout_table = end_of_header + 256 * 4;
101	// assert_eq!(	102	// assert_eq!(
102	// actual,	103	// actual,
103	// &expected[..end_of_fanout_table],	104	// &expected[..end_of_fanout_table],
104	// "we should get the fanout table right"	105	// "we should get the fanout table right"
105	// );	106	// );
106	// TODO: comment this in for the final test - keep the above anyway though, useful if something breaks	107	// TODO: comment this in for the final test - keep the above anyway though, useful if something breaks
107	// assert_eq!(	108	// assert_eq!(
108	// actual, expected,	109	// actual, expected,
109	// "we should be writing a bit-exact version of the original V2 index"	110	// "we should be writing a bit-exact version of the original V2 index"
110	// );	111	// );
111	assert_eq!(	112	assert_eq!(
112	outcome.num_objects, num_objects,	113	outcome.num_objects, num_objects,
113	"it wrote the entire iterator worth of entries"	114	"it wrote the entire iterator worth of entries"
114	);	115	);
115	assert_eq!(outcome.index_kind, desired_kind);	116	assert_eq!(outcome.index_kind, desired_kind);
116	assert_eq!(	117	assert_eq!(
117	outcome.index_hash,	118	outcome.index_hash,
118	pack::index::File::at(fixture_path(index_path))?.index_checksum()	119	pack::index::File::at(fixture_path(index_path))?.index_checksum()
119	);	120	);
		121	}
120	}	122	}
121	}	123	}
122	Ok(())	124	Ok(())

computing hashes for bases from decompressed in-memory store works

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code