History

prepare for computing indexed statistics

Sebastian Thiel committed 6 years ago

Commit 082c2461 parent bfbae901 file · +11 −129Browse files

Unified · Split

git-odb/src/pack/index/traverse/indexed.rs+11−129
@@ -1,19 +1,10 @@
1−use super::{Error, Reducer, SafetyCheck};
1+use super::{Error, SafetyCheck};
22 use crate::{
33 pack,
4− pack::index::access::PackOffset,
5− pack::index::{
6− self,
7− util::{index_entries_sorted_by_offset_ascending, Chunks},
8− },
4+ pack::index::{self, util::index_entries_sorted_by_offset_ascending},
95 pack::tree::traverse::Context,
106 };
11−use git_features::{
12− parallel::{self, in_parallel_if},
13− progress::Progress,
14−};
15−use git_object::Kind;
16−use std::collections::BTreeMap;
7+use git_features::progress::Progress;
178
189 impl index::File {
1910 pub(crate) fn traverse_with_index_lookup<P, Processor>(
@@ -35,21 +26,15 @@ impl index::File { · 15 unmodified lines
3526 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>>,
3627 {
3728 let sorted_entries = index_entries_sorted_by_offset_ascending(self, root.add_child("collecting sorted index"));
38− let graph = pack::graph::DeltaTree::from_offsets_in_pack(
39− sorted_entries.iter().map(|e| e.pack_offset),
40− pack.path(),
41− root.add_child("indexing"),
42− |id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
43− )?;
4429 let tree = pack::tree::Tree::from_offsets_in_pack(
45− sorted_entries.clone().into_iter().map(|e| EntryWithDefault::from(e)),
30+ sorted_entries.into_iter().map(|e| EntryWithDefault::from(e)),
4631 |e| e.index_entry.pack_offset,
4732 pack.path(),
4833 root.add_child("indexing"),
4934 |id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
5035 )?;
5136 let there_are_enough_objects = || self.num_objects > 10_000;
52− let _items = tree.traverse(
37+ let outcome = digest_statistics(tree.traverse(
5338 there_are_enough_objects,
5439 |slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)),
5540 root.add_child("Resolving"),
@@ -96,115 +81,8 @@ impl index::File { · 40 unmodified lines
9681 res => res,
9782 }
9883 },
99− )?;
100−
101− let reduce_progress = parking_lot::Mutex::new({
102− let mut p = root.add_child("Iterating");
103− p.init(Some(self.num_objects()), Some("objects"));
104− p
105− });
106−
107− let state_per_thread = |index| {
108− (
109− new_processor(),
110− Vec::<u8>::with_capacity(2048), // decode buffer
111− Vec::<(pack::graph::Node, u32)>::new(),
112− reduce_progress.lock().add_child(format!("thread {}", index)), // per thread progress
113− )
114− };
115− let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit(1, None, thread_limit, None);
116− in_parallel_if(
117− there_are_enough_objects,
118− Chunks {
119− size: chunk_size,
120− iter: graph.bases(),
121− },
122− thread_limit,
123− state_per_thread,
124− |input: Vec<pack::graph::Node>,
125− (ref mut processor, buf, nodes, progress)|
126− -> Result<Vec<pack::data::decode::Outcome>, Error> {
127− let mut stats = Vec::new();
128− let mut header_buf = [0u8; 64];
129− let mut children = Vec::new();
130− progress.init(None, Some("entries"));
131−
132− struct CacheEntry {
133− kind: Kind,
134− data: Vec<u8>,
135− compressed_size: usize,
136− }
137− struct SharedCache<'a>(&'a mut BTreeMap<PackOffset, CacheEntry>);
138−
139− impl<'a> pack::cache::DecodeEntry for SharedCache<'a> {
140− fn put(&mut self, pack_offset: u64, data: &[u8], kind: Kind, compressed_size: usize) {
141− self.0.entry(pack_offset).or_insert_with(|| CacheEntry {
142− kind,
143− data: data.to_owned(),
144− compressed_size,
145− });
146− }
147−
148− fn get(&mut self, offset: u64, out: &mut Vec<u8>) -> Option<(Kind, usize)> {
149− self.0.get_mut(&offset).map(
150− |CacheEntry {
151− kind,
152− data,
153− compressed_size,
154− }| {
155− out.resize(data.len(), 0);
156− out.copy_from_slice(&data);
157− (*kind, *compressed_size)
158− },
159− )
160− }
161− }
162−
163− for node in input {
164− nodes.clear();
165− nodes.push((node, 0));
166− let mut cache = BTreeMap::new();
167−
168− while let Some((node, level)) = nodes.pop() {
169− let pack_offset = node.pack_offset;
170− let index_entry = sorted_entries
171− .binary_search_by_key(&pack_offset, |e| e.pack_offset)
172− .expect("tree created by our sorted entries");
173− let index_entry_of_node = &sorted_entries[index_entry];
174−
175− graph.children(node, &mut children);
176−
177− let shared_cache = &mut SharedCache(&mut cache);
178− let result = self.process_entry_dispatch(
179− check,
180− pack,
181− shared_cache,
182− buf,
183− progress,
184− &mut header_buf,
185− index_entry_of_node,
186− processor,
187− );
188− progress.inc();
189− nodes.extend(children.iter().cloned().map(|cn| (cn, level + 1)));
190−
191− let mut stat = match result {
192− Err(err @ Error::PackDecode(_, _, _)) if !check.fatal_decode_error() => {
193− progress.info(format!("Ignoring decode error: {}", err));
194− continue;
195− }
196− res => res,
197− }?;
198− stat.num_deltas = level;
199− stats.push(stat);
200− }
201− }
202−
203− Ok(stats)
204− },
205− Reducer::from_progress(&reduce_progress, pack.data_len(), check),
206− )
207− .map(|res| (res, root))
84+ )?);
85+ Ok((outcome, root))
20886 }
20987 }
21088
@@ -249,3 +127,7 @@ impl From<pack::index::Entry> for EntryWithDefault { · 38 unmodified lines
249127 }
250128 }
251129 }
130+
131+fn digest_statistics(items: Vec<pack::tree::Item<EntryWithDefault>>) -> index::traverse::Outcome {
132+ unimplemented!();
133+}