However, it seems slightly slower and doesn't necessarily improve the progress printing. Also, we have ot content over two counters, not only one, which might be noticable and is probably not worth it.
Unified · Split
| @@ -108,7 +108,7 @@ where · 107 unmodified lines | ||
| 108 | 108 | Options { |
| 109 | 109 | thread_limit, |
| 110 | 110 | object_progress, |
| 111 | − size_progress, | |
| 111 | + mut size_progress, | |
| 112 | 112 | should_interrupt, |
| 113 | 113 | object_hash, |
| 114 | 114 | }: Options<'_, P1, P2>, |
| @@ -128,6 +128,16 @@ where · 13 unmodified lines | ||
| 128 | 128 | // allowing threads to be more busy overall. This, however, needs some refactorings to allow operation |
| 129 | 129 | // on a single item efficiently while providing real-time feedback. |
| 130 | 130 | let num_objects = self.num_items(); |
| 131 | + | |
| 132 | + let object_counter = { | |
| 133 | + let mut progress = lock(&object_progress); | |
| 134 | + progress.init(Some(num_objects), progress::count("objects")); | |
| 135 | + progress.counter() | |
| 136 | + }; | |
| 137 | + size_progress.init(None, progress::bytes()); | |
| 138 | + let size_counter = size_progress.counter(); | |
| 139 | + | |
| 140 | + let start = std::time::Instant::now(); | |
| 131 | 141 | in_parallel_if( |
| 132 | 142 | should_run_in_parallel, |
| 133 | 143 | self.iter_root_chunks(chunk_size), |
| @@ -144,9 +154,25 @@ where · 10 unmodified lines | ||
| 144 | 154 | ) |
| 145 | 155 | } |
| 146 | 156 | }, |
| 147 | − move |root_nodes, state| resolve::deltas(root_nodes, state, object_hash.len_in_bytes()), | |
| 148 | − Reducer::new(num_objects, object_progress, size_progress, should_interrupt), | |
| 157 | + { | |
| 158 | + let object_counter = object_counter.clone(); | |
| 159 | + let size_counter = size_counter.clone(); | |
| 160 | + move |root_nodes, state| { | |
| 161 | + resolve::deltas( | |
| 162 | + object_counter.clone(), | |
| 163 | + size_counter.clone(), | |
| 164 | + root_nodes, | |
| 165 | + state, | |
| 166 | + object_hash.len_in_bytes(), | |
| 167 | + ) | |
| 168 | + } | |
| 169 | + }, | |
| 170 | + Reducer::new(should_interrupt), | |
| 149 | 171 | )?; |
| 172 | + | |
| 173 | + lock(&object_progress).show_throughput(start); | |
| 174 | + size_progress.show_throughput(start); | |
| 175 | + | |
| 150 | 176 | Ok(Outcome { |
| 151 | 177 | roots: self.root_items, |
| 152 | 178 | children: self.child_items, |
| @@ -154,61 +180,31 @@ where · 1 unmodified line | ||
| 154 | 180 | } |
| 155 | 181 | } |
| 156 | 182 | |
| 157 | −struct Reducer<'a, P1, P2> { | |
| 158 | − item_count: usize, | |
| 159 | − progress: OwnShared<Mutable<P1>>, | |
| 160 | − start: std::time::Instant, | |
| 161 | − size_progress: P2, | |
| 183 | +struct Reducer<'a> { | |
| 162 | 184 | should_interrupt: &'a AtomicBool, |
| 163 | 185 | } |
| 164 | 186 | |
| 165 | −impl<'a, P1, P2> Reducer<'a, P1, P2> | |
| 166 | −where | |
| 167 | − P1: Progress, | |
| 168 | − P2: Progress, | |
| 169 | −{ | |
| 170 | − pub fn new( | |
| 171 | − num_objects: usize, | |
| 172 | − progress: OwnShared<Mutable<P1>>, | |
| 173 | − mut size_progress: P2, | |
| 174 | − should_interrupt: &'a AtomicBool, | |
| 175 | − ) -> Self { | |
| 176 | − lock(&progress).init(Some(num_objects), progress::count("objects")); | |
| 177 | − size_progress.init(None, progress::bytes()); | |
| 178 | − Reducer { | |
| 179 | − item_count: 0, | |
| 180 | − progress, | |
| 181 | − start: std::time::Instant::now(), | |
| 182 | − size_progress, | |
| 183 | − should_interrupt, | |
| 184 | − } | |
| 187 | +impl<'a> Reducer<'a> { | |
| 188 | + pub fn new(should_interrupt: &'a AtomicBool) -> Self { | |
| 189 | + Reducer { should_interrupt } | |
| 185 | 190 | } |
| 186 | 191 | } |
| 187 | 192 | |
| 188 | −impl<'a, P1, P2> parallel::Reduce for Reducer<'a, P1, P2> | |
| 189 | −where | |
| 190 | − P1: Progress, | |
| 191 | − P2: Progress, | |
| 192 | −{ | |
| 193 | − type Input = Result<(usize, u64), Error>; | |
| 193 | +impl<'a> parallel::Reduce for Reducer<'a> { | |
| 194 | + type Input = Result<(), Error>; | |
| 194 | 195 | type FeedProduce = (); |
| 195 | 196 | type Output = (); |
| 196 | 197 | type Error = Error; |
| 197 | 198 | |
| 198 | 199 | fn feed(&mut self, input: Self::Input) -> Result<(), Self::Error> { |
| 199 | − let (num_objects, decompressed_size) = input?; | |
| 200 | − self.item_count += num_objects; | |
| 201 | − self.size_progress.inc_by(decompressed_size as usize); | |
| 202 | − lock(&self.progress).set(self.item_count); | |
| 200 | + input?; | |
| 203 | 201 | if self.should_interrupt.load(Ordering::SeqCst) { |
| 204 | 202 | return Err(Error::Interrupted); |
| 205 | 203 | } |
| 206 | 204 | Ok(()) |
| 207 | 205 | } |
| 208 | 206 | |
| 209 | − fn finalize(mut self) -> Result<Self::Output, Self::Error> { | |
| 210 | − lock(&self.progress).show_throughput(self.start); | |
| 211 | − self.size_progress.show_throughput(self.start); | |
| 207 | + fn finalize(self) -> Result<Self::Output, Self::Error> { | |
| 212 | 208 | Ok(()) |
| 213 | 209 | } |
| 214 | 210 | } |
| @@ -1,3 +1,4 @@ | ||
| 1 | +use std::sync::atomic::Ordering; | |
| 1 | 2 | use std::{cell::RefCell, collections::BTreeMap}; |
| 2 | 3 | |
| 3 | 4 | use git_features::{ |
| @@ -11,10 +12,12 @@ use crate::{ · 7 unmodified lines | ||
| 11 | 12 | }; |
| 12 | 13 | |
| 13 | 14 | pub(crate) fn deltas<T, F, P, MBFN, S, E>( |
| 15 | + object_counter: Option<git_features::progress::StepShared>, | |
| 16 | + size_counter: Option<git_features::progress::StepShared>, | |
| 14 | 17 | nodes: crate::cache::delta::Chunk<'_, T>, |
| 15 | 18 | (bytes_buf, ref mut progress, state, resolve, modify_base): &mut (Vec<u8>, P, S, F, MBFN), |
| 16 | 19 | hash_len: usize, |
| 17 | −) -> Result<(usize, u64), Error> | |
| 20 | +) -> Result<(), Error> | |
| 18 | 21 | where |
| 19 | 22 | F: for<'r> Fn(EntryRange, &'r mut Vec<u8>) -> Option<()>, |
| 20 | 23 | P: Progress, |
| @@ -23,8 +26,6 @@ where · 2 unmodified lines | ||
| 23 | 26 | { |
| 24 | 27 | let mut decompressed_bytes_by_pack_offset = BTreeMap::new(); |
| 25 | 28 | let bytes_buf = RefCell::new(bytes_buf); |
| 26 | − let mut num_objects = 0; | |
| 27 | − let mut decompressed_bytes: u64 = 0; | |
| 28 | 29 | let decompress_from_resolver = |slice: EntryRange| -> Result<(crate::data::Entry, u64, Vec<u8>), Error> { |
| 29 | 30 | let mut bytes_buf = bytes_buf.borrow_mut(); |
| 30 | 31 | bytes_buf.resize((slice.end - slice.start) as usize, 0); |
| @@ -75,9 +76,10 @@ where · 44 unmodified lines | ||
| 75 | 76 | }, |
| 76 | 77 | ) |
| 77 | 78 | .map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)?; |
| 78 | − num_objects += 1; | |
| 79 | − decompressed_bytes += base_bytes.len() as u64; | |
| 80 | − progress.inc(); | |
| 79 | + object_counter.as_ref().map(|c| c.fetch_add(1, Ordering::SeqCst)); | |
| 80 | + size_counter | |
| 81 | + .as_ref() | |
| 82 | + .map(|c| c.fetch_add(base_bytes.len(), Ordering::SeqCst)); | |
| 81 | 83 | } |
| 82 | 84 | |
| 83 | 85 | for mut child in base.into_child_iter() { |
| @@ -118,14 +120,15 @@ where · 34 unmodified lines | ||
| 118 | 120 | }, |
| 119 | 121 | ) |
| 120 | 122 | .map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)?; |
| 121 | − num_objects += 1; | |
| 122 | − decompressed_bytes += fully_resolved_delta_bytes.len() as u64; | |
| 123 | − progress.inc(); | |
| 123 | + object_counter.as_ref().map(|c| c.fetch_add(1, Ordering::SeqCst)); | |
| 124 | + size_counter | |
| 125 | + .as_ref() | |
| 126 | + .map(|c| c.fetch_add(base_bytes.len(), Ordering::SeqCst)); | |
| 124 | 127 | } |
| 125 | 128 | } |
| 126 | 129 | } |
| 127 | 130 | |
| 128 | − Ok((num_objects, decompressed_bytes)) | |
| 131 | + Ok(()) | |
| 129 | 132 | } |
| 130 | 133 | |
| 131 | 134 | fn decompress_all_at_once(b: &[u8], decompressed_len: usize) -> Result<Vec<u8>, Error> { |