Unified · Split
| @@ -1,19 +1,10 @@ | |||
| 1 | use super::{Error, Reducer, SafetyCheck}; | 1 | use super::{Error, SafetyCheck}; |
| 2 | use crate::{ | 2 | use crate::{ |
| 3 | pack, | 3 | pack, |
| 4 | pack::index::access::PackOffset, | 4 | pack::index::{self, util::index_entries_sorted_by_offset_ascending}, |
| 5 | pack::index::{ | ||
| 6 | self, | ||
| 7 | util::{index_entries_sorted_by_offset_ascending, Chunks}, | ||
| 8 | }, | ||
| 9 | pack::tree::traverse::Context, | 5 | pack::tree::traverse::Context, |
| 10 | }; | 6 | }; |
| 11 | use git_features::{ | 7 | use git_features::progress::Progress; |
| 12 | parallel::{self, in_parallel_if}, | ||
| 13 | progress::Progress, | ||
| 14 | }; | ||
| 15 | use git_object::Kind; | ||
| 16 | use std::collections::BTreeMap; | ||
| 17 | 8 | ||
| 18 | impl index::File { | 9 | impl index::File { |
| 19 | pub(crate) fn traverse_with_index_lookup<P, Processor>( | 10 | pub(crate) fn traverse_with_index_lookup<P, Processor>( |
| @@ -35,21 +26,15 @@ impl index::File { · 15 unmodified lines | |||
| 35 | ) -> Result<(), Box<dyn std::error::Error + Send + Sync>>, | 26 | ) -> Result<(), Box<dyn std::error::Error + Send + Sync>>, |
| 36 | { | 27 | { |
| 37 | let sorted_entries = index_entries_sorted_by_offset_ascending(self, root.add_child("collecting sorted index")); | 28 | let sorted_entries = index_entries_sorted_by_offset_ascending(self, root.add_child("collecting sorted index")); |
| 38 | let graph = pack::graph::DeltaTree::from_offsets_in_pack( | ||
| 39 | sorted_entries.iter().map(|e| e.pack_offset), | ||
| 40 | pack.path(), | ||
| 41 | root.add_child("indexing"), | ||
| 42 | |id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)), | ||
| 43 | )?; | ||
| 44 | let tree = pack::tree::Tree::from_offsets_in_pack( | 29 | let tree = pack::tree::Tree::from_offsets_in_pack( |
| 45 | sorted_entries.clone().into_iter().map(|e| EntryWithDefault::from(e)), | 30 | sorted_entries.into_iter().map(|e| EntryWithDefault::from(e)), |
| 46 | |e| e.index_entry.pack_offset, | 31 | |e| e.index_entry.pack_offset, |
| 47 | pack.path(), | 32 | pack.path(), |
| 48 | root.add_child("indexing"), | 33 | root.add_child("indexing"), |
| 49 | |id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)), | 34 | |id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)), |
| 50 | )?; | 35 | )?; |
| 51 | let there_are_enough_objects = || self.num_objects > 10_000; | 36 | let there_are_enough_objects = || self.num_objects > 10_000; |
| 52 | let _items = tree.traverse( | 37 | let outcome = digest_statistics(tree.traverse( |
| 53 | there_are_enough_objects, | 38 | there_are_enough_objects, |
| 54 | |slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)), | 39 | |slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)), |
| 55 | root.add_child("Resolving"), | 40 | root.add_child("Resolving"), |
| @@ -96,115 +81,8 @@ impl index::File { · 40 unmodified lines | |||
| 96 | res => res, | 81 | res => res, |
| 97 | } | 82 | } |
| 98 | }, | 83 | }, |
| 99 | )?; | 84 | )?); |
| 100 | 85 | Ok((outcome, root)) | |
| 101 | let reduce_progress = parking_lot::Mutex::new({ | ||
| 102 | let mut p = root.add_child("Iterating"); | ||
| 103 | p.init(Some(self.num_objects()), Some("objects")); | ||
| 104 | p | ||
| 105 | }); | ||
| 106 | |||
| 107 | let state_per_thread = |index| { | ||
| 108 | ( | ||
| 109 | new_processor(), | ||
| 110 | Vec::<u8>::with_capacity(2048), // decode buffer | ||
| 111 | Vec::<(pack::graph::Node, u32)>::new(), | ||
| 112 | reduce_progress.lock().add_child(format!("thread {}", index)), // per thread progress | ||
| 113 | ) | ||
| 114 | }; | ||
| 115 | let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit(1, None, thread_limit, None); | ||
| 116 | in_parallel_if( | ||
| 117 | there_are_enough_objects, | ||
| 118 | Chunks { | ||
| 119 | size: chunk_size, | ||
| 120 | iter: graph.bases(), | ||
| 121 | }, | ||
| 122 | thread_limit, | ||
| 123 | state_per_thread, | ||
| 124 | |input: Vec<pack::graph::Node>, | ||
| 125 | (ref mut processor, buf, nodes, progress)| | ||
| 126 | -> Result<Vec<pack::data::decode::Outcome>, Error> { | ||
| 127 | let mut stats = Vec::new(); | ||
| 128 | let mut header_buf = [0u8; 64]; | ||
| 129 | let mut children = Vec::new(); | ||
| 130 | progress.init(None, Some("entries")); | ||
| 131 | |||
| 132 | struct CacheEntry { | ||
| 133 | kind: Kind, | ||
| 134 | data: Vec<u8>, | ||
| 135 | compressed_size: usize, | ||
| 136 | } | ||
| 137 | struct SharedCache<'a>(&'a mut BTreeMap<PackOffset, CacheEntry>); | ||
| 138 | |||
| 139 | impl<'a> pack::cache::DecodeEntry for SharedCache<'a> { | ||
| 140 | fn put(&mut self, pack_offset: u64, data: &[u8], kind: Kind, compressed_size: usize) { | ||
| 141 | self.0.entry(pack_offset).or_insert_with(|| CacheEntry { | ||
| 142 | kind, | ||
| 143 | data: data.to_owned(), | ||
| 144 | compressed_size, | ||
| 145 | }); | ||
| 146 | } | ||
| 147 | |||
| 148 | fn get(&mut self, offset: u64, out: &mut Vec<u8>) -> Option<(Kind, usize)> { | ||
| 149 | self.0.get_mut(&offset).map( | ||
| 150 | |CacheEntry { | ||
| 151 | kind, | ||
| 152 | data, | ||
| 153 | compressed_size, | ||
| 154 | }| { | ||
| 155 | out.resize(data.len(), 0); | ||
| 156 | out.copy_from_slice(&data); | ||
| 157 | (*kind, *compressed_size) | ||
| 158 | }, | ||
| 159 | ) | ||
| 160 | } | ||
| 161 | } | ||
| 162 | |||
| 163 | for node in input { | ||
| 164 | nodes.clear(); | ||
| 165 | nodes.push((node, 0)); | ||
| 166 | let mut cache = BTreeMap::new(); | ||
| 167 | |||
| 168 | while let Some((node, level)) = nodes.pop() { | ||
| 169 | let pack_offset = node.pack_offset; | ||
| 170 | let index_entry = sorted_entries | ||
| 171 | .binary_search_by_key(&pack_offset, |e| e.pack_offset) | ||
| 172 | .expect("tree created by our sorted entries"); | ||
| 173 | let index_entry_of_node = &sorted_entries[index_entry]; | ||
| 174 | |||
| 175 | graph.children(node, &mut children); | ||
| 176 | |||
| 177 | let shared_cache = &mut SharedCache(&mut cache); | ||
| 178 | let result = self.process_entry_dispatch( | ||
| 179 | check, | ||
| 180 | pack, | ||
| 181 | shared_cache, | ||
| 182 | buf, | ||
| 183 | progress, | ||
| 184 | &mut header_buf, | ||
| 185 | index_entry_of_node, | ||
| 186 | processor, | ||
| 187 | ); | ||
| 188 | progress.inc(); | ||
| 189 | nodes.extend(children.iter().cloned().map(|cn| (cn, level + 1))); | ||
| 190 | |||
| 191 | let mut stat = match result { | ||
| 192 | Err(err @ Error::PackDecode(_, _, _)) if !check.fatal_decode_error() => { | ||
| 193 | progress.info(format!("Ignoring decode error: {}", err)); | ||
| 194 | continue; | ||
| 195 | } | ||
| 196 | res => res, | ||
| 197 | }?; | ||
| 198 | stat.num_deltas = level; | ||
| 199 | stats.push(stat); | ||
| 200 | } | ||
| 201 | } | ||
| 202 | |||
| 203 | Ok(stats) | ||
| 204 | }, | ||
| 205 | Reducer::from_progress(&reduce_progress, pack.data_len(), check), | ||
| 206 | ) | ||
| 207 | .map(|res| (res, root)) | ||
| 208 | } | 86 | } |
| 209 | } | 87 | } |
| 210 | 88 | ||
| @@ -249,3 +127,7 @@ impl From<pack::index::Entry> for EntryWithDefault { · 38 unmodified lines | |||
| 249 | } | 127 | } |
| 250 | } | 128 | } |
| 251 | } | 129 | } |
| 130 | |||
| 131 | fn digest_statistics(items: Vec<pack::tree::Item<EntryWithDefault>>) -> index::traverse::Outcome { | ||
| 132 | unimplemented!(); | ||
| 133 | } | ||