Unified · Split
| @@ -1,7 +1,7 @@ | ||
| 1 | −use crate::{hash, pack, pack::index::util::Chunks, pack::index::V2_SIGNATURE}; | |
| 1 | +use crate::{hash, loose, pack, pack::index::util::Chunks, pack::index::V2_SIGNATURE}; | |
| 2 | 2 | use byteorder::{BigEndian, WriteBytesExt}; |
| 3 | 3 | use git_features::{parallel, parallel::in_parallel_if, progress::Progress}; |
| 4 | −use git_object::owned; | |
| 4 | +use git_object::{owned, HashKind}; | |
| 5 | 5 | use smallvec::alloc::collections::BTreeMap; |
| 6 | 6 | use std::io; |
| 7 | 7 | |
| @@ -28,7 +28,6 @@ impl pack::index::File { · 20 unmodified lines | ||
| 28 | 28 | { |
| 29 | 29 | use io::Write; |
| 30 | 30 | |
| 31 | − let mut out = hash::Write::new(out, kind.hash()); | |
| 32 | 31 | if kind != pack::index::Kind::default() { |
| 33 | 32 | return Err(Error::Unsupported(kind)); |
| 34 | 33 | } |
| @@ -81,7 +80,7 @@ impl pack::index::File { · 46 unmodified lines | ||
| 81 | 80 | .ok_or_else(|| { |
| 82 | 81 | Error::IteratorInvariantBasesBeforeDeltasNeedThem(pack_offset, base_pack_offset) |
| 83 | 82 | })? |
| 84 | − .child_count += 1; | |
| 83 | + .increment_child_count(); | |
| 85 | 84 | ( |
| 86 | 85 | mode.delta_cache(compressed, decompressed), |
| 87 | 86 | ObjectKind::OfsDelta(base_pack_offset), |
| @@ -89,13 +88,7 @@ impl pack::index::File { · 1 unmodified line | ||
| 89 | 88 | } |
| 90 | 89 | }; |
| 91 | 90 | |
| 92 | − cache_by_offset.insert( | |
| 93 | − pack_offset, | |
| 94 | − CacheEntry { | |
| 95 | − child_count: 0, | |
| 96 | − _cache: cache, | |
| 97 | − }, | |
| 98 | − ); | |
| 91 | + cache_by_offset.insert(pack_offset, CacheEntry::new(cache)); | |
| 99 | 92 | index_entries.push(Entry { |
| 100 | 93 | pack_offset, |
| 101 | 94 | kind, |
| @@ -121,7 +114,14 @@ impl pack::index::File { · 19 unmodified lines | ||
| 121 | 114 | thread_limit, |
| 122 | 115 | |_| (), |
| 123 | 116 | |base_pack_offsets, state| { |
| 124 | − apply_deltas(base_pack_offsets, state, &index_entries, &_cache_by_offset, &mode) | |
| 117 | + apply_deltas( | |
| 118 | + base_pack_offsets, | |
| 119 | + state, | |
| 120 | + &index_entries, | |
| 121 | + &_cache_by_offset, | |
| 122 | + &mode, | |
| 123 | + kind.hash(), | |
| 124 | + ) | |
| 125 | 125 | }, |
| 126 | 126 | Reducer::new(num_objects), |
| 127 | 127 | )?; |
| @@ -139,15 +139,16 @@ impl pack::index::File { · 11 unmodified lines | ||
| 139 | 139 | drop(index_entries); |
| 140 | 140 | |
| 141 | 141 | // Write header |
| 142 | + let mut out = hash::Write::new(out, kind.hash()); | |
| 142 | 143 | out.write_all(V2_SIGNATURE)?; |
| 143 | 144 | out.write_u32::<BigEndian>(kind as u32)?; |
| 144 | 145 | |
| 145 | 146 | // todo: write fanout |
| 146 | 147 | |
| 147 | − let _index_hash = out.hash.digest(); | |
| 148 | + let index_hash = out.hash.digest().into(); | |
| 148 | 149 | Ok(Outcome { |
| 149 | 150 | index_kind: kind, |
| 150 | − index_hash: owned::Id::from([0u8; 20]), | |
| 151 | + index_hash, | |
| 151 | 152 | pack_hash: last_seen_trailer.ok_or(Error::IteratorInvariantTrailer)?, |
| 152 | 153 | num_objects, |
| 153 | 154 | }) |
| @@ -155,14 +156,60 @@ impl pack::index::File { · 1 unmodified line | ||
| 155 | 156 | } |
| 156 | 157 | |
| 157 | 158 | fn apply_deltas<F>( |
| 158 | − _base_entries: Vec<&Entry>, | |
| 159 | + base_entries: Vec<&Entry>, | |
| 159 | 160 | _state: &mut (), |
| 160 | 161 | _entries: &[Entry], |
| 161 | − _caches: &parking_lot::Mutex<BTreeMap<u64, CacheEntry>>, | |
| 162 | + caches: &parking_lot::Mutex<BTreeMap<u64, CacheEntry>>, | |
| 162 | 163 | _mode: &Mode<F>, |
| 164 | + hash_kind: HashKind, | |
| 163 | 165 | ) -> Vec<(u64, owned::Id)> |
| 164 | 166 | where |
| 165 | 167 | F: for<'r> Fn(u64, &'r mut Vec<u8>) -> Option<(pack::data::Header, u64)> + Send + Sync, |
| 166 | 168 | { |
| 167 | − Vec::new() | |
| 169 | + let decompressed_bytes_from_cache = |pack_offset: &u64| -> (bool, Vec<u8>) { | |
| 170 | + let cache = caches | |
| 171 | + .lock() | |
| 172 | + .get_mut(pack_offset) | |
| 173 | + .expect("an entry for every pack offset") | |
| 174 | + .cache(); | |
| 175 | + let (is_borrowed, cache) = match cache { | |
| 176 | + Bytes::Borrowed(b) => (true, b), | |
| 177 | + Bytes::Owned(b) => (false, b), | |
| 178 | + }; | |
| 179 | + let bytes = match cache { | |
| 180 | + Cache::Decompressed(b) => b, | |
| 181 | + Cache::Compressed(_, _) => unimplemented!("decompress once"), | |
| 182 | + Cache::Unset => unimplemented!("use resolver"), | |
| 183 | + }; | |
| 184 | + (is_borrowed, bytes) | |
| 185 | + }; | |
| 186 | + let possibly_return_to_cache = |pack_offset: &u64, is_borrowed: bool, bytes: Vec<u8>| { | |
| 187 | + if is_borrowed { | |
| 188 | + caches | |
| 189 | + .lock() | |
| 190 | + .get_mut(pack_offset) | |
| 191 | + .expect("an entry for every pack offset") | |
| 192 | + .set_decompressed(bytes); | |
| 193 | + } | |
| 194 | + }; | |
| 195 | + let compute_hash = |kind: git_object::Kind, bytes: &[u8]| -> owned::Id { | |
| 196 | + let mut write = hash::Write::new(io::sink(), hash_kind); | |
| 197 | + loose::object::header::encode(kind, bytes.len() as u64, &mut write) | |
| 198 | + .expect("write to sink and hash cannot fail"); | |
| 199 | + write.hash.update(bytes); | |
| 200 | + owned::Id::from(write.hash.digest()) | |
| 201 | + }; | |
| 202 | + let mut out = Vec::with_capacity(base_entries.len()); // perfectly conservative guess | |
| 203 | + | |
| 204 | + for Entry { pack_offset, kind, .. } in base_entries { | |
| 205 | + let (is_borrowed, base_bytes) = decompressed_bytes_from_cache(pack_offset); | |
| 206 | + out.push(( | |
| 207 | + *pack_offset, | |
| 208 | + compute_hash(kind.to_kind().expect("base object"), &base_bytes), | |
| 209 | + )); | |
| 210 | + possibly_return_to_cache(pack_offset, is_borrowed, base_bytes); | |
| 211 | + } | |
| 212 | + | |
| 213 | + out.shrink_to_fit(); | |
| 214 | + out | |
| 168 | 215 | } |
| @@ -30,6 +30,12 @@ impl ObjectKind { · 29 unmodified lines | ||
| 30 | 30 | ObjectKind::OfsDelta(_) => false, |
| 31 | 31 | } |
| 32 | 32 | } |
| 33 | + pub fn to_kind(&self) -> Option<git_object::Kind> { | |
| 34 | + match self { | |
| 35 | + ObjectKind::Base(kind) => Some(*kind), | |
| 36 | + ObjectKind::OfsDelta(_) => None, | |
| 37 | + } | |
| 38 | + } | |
| 33 | 39 | } |
| 34 | 40 | |
| 35 | 41 | pub(crate) struct Entry { |
| @@ -39,26 +45,44 @@ pub(crate) struct Entry { · 3 unmodified lines | ||
| 39 | 45 | } |
| 40 | 46 | |
| 41 | 47 | pub(crate) struct CacheEntry { |
| 42 | − pub _cache: Cache, | |
| 48 | + cache: Cache, | |
| 43 | 49 | /// When it reaches zero, the cache can be freed |
| 44 | − pub child_count: u32, | |
| 50 | + child_count: u32, | |
| 45 | 51 | } |
| 46 | 52 | |
| 47 | −pub(crate) enum _Bytes { | |
| 53 | +pub(crate) enum Bytes { | |
| 48 | 54 | Owned(Cache), |
| 49 | 55 | Borrowed(Cache), |
| 50 | 56 | } |
| 51 | 57 | |
| 58 | +/// Note that every operation in the CacheEntry must be fast, as these happen behind a lock | |
| 52 | 59 | impl CacheEntry { |
| 53 | − pub fn _decr(&mut self) -> _Bytes { | |
| 60 | + pub fn new(cache: Cache) -> Self { | |
| 61 | + CacheEntry { | |
| 62 | + child_count: 0, | |
| 63 | + cache: cache, | |
| 64 | + } | |
| 65 | + } | |
| 66 | + pub fn increment_child_count(&mut self) { | |
| 67 | + self.child_count += 1; | |
| 68 | + } | |
| 69 | + pub fn _decr(&mut self) -> Bytes { | |
| 54 | 70 | self.child_count -= 1; |
| 55 | − let cache = std::mem::replace(&mut self._cache, Cache::Unset); | |
| 71 | + self.cache() | |
| 72 | + } | |
| 73 | + | |
| 74 | + pub fn cache(&mut self) -> Bytes { | |
| 75 | + let cache = std::mem::replace(&mut self.cache, Cache::Unset); | |
| 56 | 76 | if self.child_count == 0 { |
| 57 | − _Bytes::Owned(cache) | |
| 77 | + Bytes::Owned(cache) | |
| 58 | 78 | } else { |
| 59 | − _Bytes::Borrowed(cache) | |
| 79 | + Bytes::Borrowed(cache) | |
| 60 | 80 | } |
| 61 | 81 | } |
| 82 | + pub fn set_decompressed(&mut self, bytes: Vec<u8>) { | |
| 83 | + assert_ne!(self.child_count, 0, "Do not return decompressed bytes once nobody is interested in the data anymore, i.e. from `Bytes::Owned(…)`"); | |
| 84 | + self.cache = Cache::Decompressed(bytes); | |
| 85 | + } | |
| 62 | 86 | } |
| 63 | 87 | |
| 64 | 88 | /// The function resolves pack_offset: u64 into compressed bytes to &mut Vec<u8> and returns (object kind, decompressed size) |
| @@ -77,46 +77,48 @@ mod method { · 76 unmodified lines | ||
| 77 | 77 | fn write_to_stream() -> Result<(), Box<dyn std::error::Error>> { |
| 78 | 78 | for mode in &[Mode::AsIs, Mode::Verify, Mode::Restore] { |
| 79 | 79 | for (index_path, data_path) in V2_PACKS_AND_INDICES { |
| 80 | − let pack_iter = pack::data::Iter::new_from_header( | |
| 81 | − io::BufReader::new(fs::File::open(fixture_path(data_path))?), | |
| 82 | − *mode, | |
| 83 | − )?; | |
| 80 | + for memory_mode in vec![pack::index::write::Mode::in_memory_decompressed()] { | |
| 81 | + let pack_iter = pack::data::Iter::new_from_header( | |
| 82 | + io::BufReader::new(fs::File::open(fixture_path(data_path))?), | |
| 83 | + *mode, | |
| 84 | + )?; | |
| 84 | 85 | |
| 85 | − let mut actual = Vec::<u8>::new(); | |
| 86 | − let desired_kind = pack::index::Kind::default(); | |
| 87 | − let num_objects = pack_iter.len() as u32; | |
| 88 | − let outcome = pack::index::File::write_data_iter_to_stream( | |
| 89 | − desired_kind, | |
| 90 | − pack::index::write::Mode::in_memory_decompressed(), | |
| 91 | − pack_iter, | |
| 92 | − None, | |
| 93 | − progress::Discard, | |
| 94 | − &mut actual, | |
| 95 | − )?; | |
| 86 | + let mut actual = Vec::<u8>::new(); | |
| 87 | + let desired_kind = pack::index::Kind::default(); | |
| 88 | + let num_objects = pack_iter.len() as u32; | |
| 89 | + let outcome = pack::index::File::write_data_iter_to_stream( | |
| 90 | + desired_kind, | |
| 91 | + memory_mode, | |
| 92 | + pack_iter, | |
| 93 | + None, | |
| 94 | + progress::Discard, | |
| 95 | + &mut actual, | |
| 96 | + )?; | |
| 96 | 97 | |
| 97 | − let expected = fs::read(fixture_path(index_path))?; | |
| 98 | − let end_of_header = 4 * 2; | |
| 99 | − assert_eq!(actual, &expected[..end_of_header], "we should get the header right"); | |
| 100 | − // let end_of_fanout_table = end_of_header + 256 * 4; | |
| 101 | − // assert_eq!( | |
| 102 | − // actual, | |
| 103 | − // &expected[..end_of_fanout_table], | |
| 104 | − // "we should get the fanout table right" | |
| 105 | − // ); | |
| 106 | − // TODO: comment this in for the final test - keep the above anyway though, useful if something breaks | |
| 107 | − // assert_eq!( | |
| 108 | − // actual, expected, | |
| 109 | − // "we should be writing a bit-exact version of the original V2 index" | |
| 110 | − // ); | |
| 111 | − assert_eq!( | |
| 112 | − outcome.num_objects, num_objects, | |
| 113 | − "it wrote the entire iterator worth of entries" | |
| 114 | − ); | |
| 115 | − assert_eq!(outcome.index_kind, desired_kind); | |
| 116 | − assert_eq!( | |
| 117 | − outcome.index_hash, | |
| 118 | − pack::index::File::at(fixture_path(index_path))?.index_checksum() | |
| 119 | − ); | |
| 98 | + let expected = fs::read(fixture_path(index_path))?; | |
| 99 | + let end_of_header = 4 * 2; | |
| 100 | + assert_eq!(actual, &expected[..end_of_header], "we should get the header right"); | |
| 101 | + // let end_of_fanout_table = end_of_header + 256 * 4; | |
| 102 | + // assert_eq!( | |
| 103 | + // actual, | |
| 104 | + // &expected[..end_of_fanout_table], | |
| 105 | + // "we should get the fanout table right" | |
| 106 | + // ); | |
| 107 | + // TODO: comment this in for the final test - keep the above anyway though, useful if something breaks | |
| 108 | + // assert_eq!( | |
| 109 | + // actual, expected, | |
| 110 | + // "we should be writing a bit-exact version of the original V2 index" | |
| 111 | + // ); | |
| 112 | + assert_eq!( | |
| 113 | + outcome.num_objects, num_objects, | |
| 114 | + "it wrote the entire iterator worth of entries" | |
| 115 | + ); | |
| 116 | + assert_eq!(outcome.index_kind, desired_kind); | |
| 117 | + assert_eq!( | |
| 118 | + outcome.index_hash, | |
| 119 | + pack::index::File::at(fixture_path(index_path))?.index_checksum() | |
| 120 | + ); | |
| 121 | + } | |
| 120 | 122 | } |
| 121 | 123 | } |
| 122 | 124 | Ok(()) |