History

Make use of shared reducer in upcoming indexed verify implementation

Sebastian Thiel committed 6 years ago

Commit 290eae11 parent 10fc88d4 files · +47 −44Browse files

Unified · Split

4 files
  1. git-odb/src/pack/index/verify/indexed.rs +11 −21
  2. git-odb/src/pack/index/verify/lookup.rs +7 −21
  3. git-odb/src/pack/index/verify/mod.rs +7 −2
  4. git-odb/src/pack/index/verify/reduce.rs +22 −0
git-odb/src/pack/index/verify/indexed.rs+11−21
@@ -1,7 +1,7 @@
1use super::{Error, Mode, Outcome};1use super::{Error, Mode, Outcome};
2use crate::{pack, pack::index};2use crate::{pack, pack::index};
3use git_features::{3use git_features::{
4 parallel::{self, in_parallel_if},4 parallel::in_parallel_if,
5 progress::{self, Progress},5 progress::{self, Progress},
6};6};
7use std::time::SystemTime;7use std::time::SystemTime;
@@ -12,7 +12,7 @@ impl index::File { · 4 unmodified lines
12 thread_limit: Option<usize>,12 thread_limit: Option<usize>,
13 _mode: Mode,13 _mode: Mode,
14 _make_cache: impl Fn() -> C + Send + Sync,14 _make_cache: impl Fn() -> C + Send + Sync,
15 mut progress: progress::DoOrDiscard<P>,15 mut root: progress::DoOrDiscard<P>,
16 pack: &pack::data::File,16 pack: &pack::data::File,
17 ) -> Result<Outcome, Error>17 ) -> Result<Outcome, Error>
18 where18 where
@@ -21,7 +21,7 @@ impl index::File { · 2 unmodified lines
21 C: pack::cache::DecodeEntry,21 C: pack::cache::DecodeEntry,
22 {22 {
23 let offsets = {23 let offsets = {
24 let mut indexing_progress = progress.add_child("preparing pack offsets");24 let mut indexing_progress = root.add_child("preparing pack offsets");
25 indexing_progress.init(Some(self.num_objects), Some("objects"));25 indexing_progress.init(Some(self.num_objects), Some("objects"));
26 let then = SystemTime::now();26 let then = SystemTime::now();
27 let iter = self.sorted_offsets().into_iter();27 let iter = self.sorted_offsets().into_iter();
@@ -33,31 +33,21 @@ impl index::File { · 5 unmodified lines
33 ));33 ));
34 iter34 iter
35 };35 };
36 let tree = pack::graph::DeltaTree::from_sorted_offsets(offsets, pack.path(), progress.add_child("indexing"))?;36 let tree = pack::graph::DeltaTree::from_sorted_offsets(offsets, pack.path(), root.add_child("indexing"))?;
37 let if_there_are_enough_objects = || self.num_objects > 10_000;37 let if_there_are_enough_objects = || self.num_objects > 10_000;
38 struct Reducer;
39 38
40 impl parallel::Reducer for Reducer {39 let reduce_progress = std::sync::Mutex::new({
41 type Input = ();40 let mut p = root.add_child("Checking");
42 type Output = Outcome;41 p.init(Some(self.num_objects()), Some("objects"));
43 type Error = Error;42 p
44 43 });
45 fn feed(&mut self, input: Self::Input) -> Result<(), Self::Error> {
46 unimplemented!()
47 }
48
49 fn finalize(self) -> Result<Self::Output, Self::Error> {
50 unimplemented!()
51 }
52 }
53 let reduce = Reducer;
54 in_parallel_if(44 in_parallel_if(
55 if_there_are_enough_objects,45 if_there_are_enough_objects,
56 tree.bases(),46 tree.bases(),
57 thread_limit,47 thread_limit,
58 |_index| (),48 |_index| (),
59 |node: pack::graph::Node, state: &mut ()| (),49 |node: pack::graph::Node, state: &mut ()| Ok::<_, Error>(Vec::new()),
60 reduce,50 index::verify::Reducer::from_progress(&reduce_progress, pack.data_len()),
61 )?;51 )?;
62 52
63 unimplemented!()53 unimplemented!()
git-odb/src/pack/index/verify/lookup.rs+7−21
@@ -1,11 +1,10 @@
1use super::{reduce::Reducer, Error, Mode, Outcome};1use super::{Error, Mode, Outcome, Reducer};
2use crate::{pack, pack::data::decode, pack::index};2use crate::{pack, pack::data::decode, pack::index};
3use git_features::{3use git_features::{
4 parallel::in_parallel_if,4 parallel::in_parallel_if,
5 progress::{self, Progress},5 progress::{self, Progress},
6};6};
7use git_object::{borrowed, bstr::ByteSlice, owned};7use git_object::{borrowed, bstr::ByteSlice, owned};
8use std::time::Instant;
9 8
10/// Verify and validate the content of the index file9/// Verify and validate the content of the index file
11impl index::File {10impl index::File {
@@ -33,11 +32,11 @@ impl index::File { · 21 unmodified lines
33 const CHUNK_SIZE: usize = 1000;32 const CHUNK_SIZE: usize = 1000;
34 let there_are_enough_entries_to_process = || index_entries.len() > CHUNK_SIZE * 2;33 let there_are_enough_entries_to_process = || index_entries.len() > CHUNK_SIZE * 2;
35 let input_chunks = index_entries.chunks(CHUNK_SIZE.max(index_entries.len() / CHUNK_SIZE));34 let input_chunks = index_entries.chunks(CHUNK_SIZE.max(index_entries.len() / CHUNK_SIZE));
36 let reduce_progress = std::sync::Mutex::new(root.add_child("Checking"));35 let reduce_progress = std::sync::Mutex::new({
37 reduce_progress36 let mut p = root.add_child("Checking");
38 .lock()37 p.init(Some(self.num_objects()), Some("objects"));
39 .unwrap()38 p
40 .init(Some(self.num_objects()), Some("objects"));39 });
41 let state_per_thread = |index| {40 let state_per_thread = |index| {
42 (41 (
43 make_cache(),42 make_cache(),
@@ -141,20 +140,7 @@ impl index::File { · 97 unmodified lines
141 }140 }
142 Ok(stats)141 Ok(stats)
143 },142 },
144 Reducer {143 Reducer::from_progress(&reduce_progress, pack.data_len()),
145 progress: &reduce_progress,
146 then: Instant::now(),
147 entries_seen: 0,
148 chunks_seen: 0,
149 stats: Outcome {
150 average: decode::Outcome::default_from_kind(git_object::Kind::Tree),
151 objects_per_chain_length: Default::default(),
152 total_compressed_entries_size: 0,
153 total_decompressed_entries_size: 0,
154 total_object_size: 0,
155 pack_size: pack.data_len() as u64,
156 },
157 },
158 )144 )
159 }145 }
160}146}
git-odb/src/pack/index/verify/mod.rs+7−2
@@ -107,10 +107,14 @@ pub enum Mode { · 106 unmodified lines
107#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]107#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)]
108pub enum Algorithm {108pub enum Algorithm {
109 /// We lookup each object similarly to what would happen during normal repository use.109 /// We lookup each object similarly to what would happen during normal repository use.
110 /// Uses more compute resources as it will resolve delta chains from back to front, potentially110 /// Uses more compute resources as it will resolve delta chains from back to front, but start right away
111 /// without indexing or investing any memory in indices.
112 ///
113 /// This option may be well suited for big packs in memory-starved system that support memory mapping.
111 Lookup,114 Lookup,
112 /// Build an index to allow decoding each delta and base exactly once, saving a lot of computational115 /// Build an index to allow decoding each delta and base exactly once, saving a lot of computational
113 /// resource at the expense of resident memory, as we will use an additional DeltaTree to make that happen.116 /// resource at the expense of resident memory, as we will use an additional `DeltaTree` to accelerate
117 /// delta chain resolution.
114 DeltaTreeLookup,118 DeltaTreeLookup,
115}119}
116 120
@@ -123,6 +127,7 @@ impl Default for Algorithm { · 6 unmodified lines
123mod indexed;127mod indexed;
124mod lookup;128mod lookup;
125mod reduce;129mod reduce;
130pub(crate) use reduce::Reducer;
126 131
127/// Verify and validate the content of the index file132/// Verify and validate the content of the index file
128impl index::File {133impl index::File {
git-odb/src/pack/index/verify/reduce.rs+22−0
@@ -24,6 +24,28 @@ pub struct Reducer<'a, P> { · 23 unmodified lines
24 pub stats: verify::Outcome,24 pub stats: verify::Outcome,
25}25}
26 26
27impl<'a, P> Reducer<'a, P>
28where
29 P: Progress,
30{
31 pub fn from_progress(progress: &'a std::sync::Mutex<P>, pack_data_len_in_bytes: usize) -> Self {
32 Reducer {
33 progress: &progress,
34 then: Instant::now(),
35 entries_seen: 0,
36 chunks_seen: 0,
37 stats: verify::Outcome {
38 average: decode::Outcome::default_from_kind(git_object::Kind::Tree),
39 objects_per_chain_length: Default::default(),
40 total_compressed_entries_size: 0,
41 total_decompressed_entries_size: 0,
42 total_object_size: 0,
43 pack_size: pack_data_len_in_bytes as u64,
44 },
45 }
46 }
47}
48
27impl<'a, P> parallel::Reducer for Reducer<'a, P>49impl<'a, P> parallel::Reducer for Reducer<'a, P>
28where50where
29 P: Progress,51 P: Progress,