History

[traverse-tree] multi-threaded gathering of unique objects

Sebastian Thiel committed 5 years ago

Commit 8f741b91 parent 3f9ee234 files · +156 −59Browse files

Unified · Split

4 files
  1. Cargo.lock +1 −2
  2. experiments/traversal/Cargo.toml +1 −2
  3. experiments/traversal/README.md +1 −0
  4. experiments/traversal/src/main.rs +153 −55
Cargo.lock+1−2
@@ -2104,13 +2104,12 @@ name = "traversal" · 2103 unmodified lines
2104version = "0.1.0"2104version = "0.1.0"
2105dependencies = [2105dependencies = [
2106 "anyhow",2106 "anyhow",
2107 "git-diff",2107 "dashmap",
2108 "git-hash",2108 "git-hash",
2109 "git-object",2109 "git-object",
2110 "git-odb",2110 "git-odb",
2111 "git-traverse",2111 "git-traverse",
2112 "git2",2112 "git2",
2113 "memory-lru",
2114 "rayon",2113 "rayon",
2115]2114]
2116 2115
experiments/traversal/Cargo.toml+1−2
@@ -13,7 +13,6 @@ git-odb = { version = "^0.14", path = "../../git-odb", features = ["pack-cache-l · 12 unmodified lines
13git-traverse = { version = "0.1", path = "../../git-traverse" }13git-traverse = { version = "0.1", path = "../../git-traverse" }
14git-hash = { version = "^0.3", path = "../../git-hash" }14git-hash = { version = "^0.3", path = "../../git-hash" }
15git-object = { version = "^0.8", path = "../../git-object" }15git-object = { version = "^0.8", path = "../../git-object" }
16git-diff = { version = "^0.2", path = "../../git-diff" }
17git2 = "0.13"16git2 = "0.13"
18rayon = "1.5.0"17rayon = "1.5.0"
19memory-lru = "0.1.0"18dashmap = "4.0.2"
experiments/traversal/README.md+1−0
@@ -6,3 +6,4 @@ An experiment to see how fast we can traverse · 5 unmodified lines
6and compare that to libgit2.6and compare that to libgit2.
7 7
8This is relevant for learning which objects to add to a pack in case of a clone.8This is relevant for learning which objects to add to a pack in case of a clone.
9[Here is the discussion with additional performance data.](https://github.com/Byron/gitoxide/discussions/76).
experiments/traversal/src/main.rs+153−55
@@ -1,4 +1,5 @@
1use anyhow::anyhow;1use anyhow::anyhow;
2use dashmap::DashSet;
2use git_hash::{bstr::BStr, bstr::ByteSlice, ObjectId};3use git_hash::{bstr::BStr, bstr::ByteSlice, ObjectId};
3use git_object::immutable::tree::Entry;4use git_object::immutable::tree::Entry;
4use git_odb::Find;5use git_odb::Find;
@@ -65,13 +66,33 @@ fn main() -> anyhow::Result<()> { · 60 unmodified lines
65 );66 );
66 67
67 let start = Instant::now();68 let start = Instant::now();
68 let (unique, entries) = do_gitoxide_tree_dag_traversal(&all_commits, &db, || {69 let (unique, entries) = do_gitoxide_tree_dag_traversal(
69 git_odb::pack::cache::lru::MemoryCappedHashmap::new(GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES)70 &all_commits,
70 })?;71 &db,
72 || git_odb::pack::cache::lru::StaticLinkedList::<64>::default(),
73 Computation::MultiThreaded,
74 )?;
71 let elapsed = start.elapsed();75 let elapsed = start.elapsed();
72 println!(76 println!(
73 "gitoxide (cache = {:.0}MB): confirmed {} entries ({} unique objects) in {} trees in {:?} ({:0.0} entries/s, {:0.0} trees/s)",77 "gitoxide PARALLEL (cache = 64 entries: confirmed {} entries ({} unique objects) in {} trees in {:?} ({:0.0} entries/s, {:0.0} trees/s)",
74 GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES as f32 / (1024 * 1024) as f32,78 entries,
79 unique,
80 all_commits.len(),
81 elapsed,
82 entries as f32 / elapsed.as_secs_f32(),
83 all_commits.len() as f32 / elapsed.as_secs_f32()
84 );
85
86 let start = Instant::now();
87 let (unique, entries) = do_gitoxide_tree_dag_traversal(
88 &all_commits,
89 &db,
90 || git_odb::pack::cache::lru::StaticLinkedList::<64>::default(),
91 Computation::SingleThreaded,
92 )?;
93 let elapsed = start.elapsed();
94 println!(
95 "gitoxide (cache = 64 entries: confirmed {} entries ({} unique objects) in {} trees in {:?} ({:0.0} entries/s, {:0.0} trees/s)",
75 entries,96 entries,
76 unique,97 unique,
77 all_commits.len(),98 all_commits.len(),
@@ -172,69 +193,146 @@ where · 94 unmodified lines
172 Ok(commits)193 Ok(commits)
173}194}
174 195
196enum Computation {
197 SingleThreaded,
198 MultiThreaded,
199}
200
175fn do_gitoxide_tree_dag_traversal<C>(201fn do_gitoxide_tree_dag_traversal<C>(
176 commits: &[ObjectId],202 commits: &[ObjectId],
177 db: &git_odb::linked::Db,203 db: &git_odb::linked::Db,
178 new_cache: impl FnOnce() -> C,204 new_cache: impl Fn() -> C + Sync + Send,
205 mode: Computation,
179) -> anyhow::Result<(usize, u64)>206) -> anyhow::Result<(usize, u64)>
180where207where
181 C: git_odb::pack::cache::DecodeEntry,208 C: git_odb::pack::cache::DecodeEntry,
182{209{
183 #[derive(Default)]210 match mode {
184 struct Count {211 Computation::SingleThreaded => {
185 entries: usize,212 #[derive(Default)]
186 seen: HashSet<ObjectId>,213 struct Count {
187 }214 entries: usize,
215 seen: HashSet<ObjectId>,
216 }
188 217
189 impl tree::visit::Visit for Count {218 impl tree::visit::Visit for Count {
190 type PathId = ();219 type PathId = ();
191 fn set_current_path(&mut self, _id: Self::PathId) {}220 fn set_current_path(&mut self, _id: Self::PathId) {}
192 fn push_tracked_path_component(&mut self, _component: &BStr) -> Self::PathId {}221 fn push_tracked_path_component(&mut self, _component: &BStr) -> Self::PathId {}
193 fn push_path_component(&mut self, _component: &BStr) {}222 fn push_path_component(&mut self, _component: &BStr) {}
194 fn pop_path_component(&mut self) {}223 fn pop_path_component(&mut self) {}
195 fn visit_tree(&mut self, entry: &Entry<'_>) -> Action {224 fn visit_tree(&mut self, entry: &Entry<'_>) -> Action {
196 self.entries += 1;225 self.entries += 1;
197 let inserted = self.seen.insert(entry.oid.to_owned());226 let inserted = self.seen.insert(entry.oid.to_owned());
198 if !inserted {227 if !inserted {
199 tree::visit::Action::Skip228 tree::visit::Action::Skip
200 } else {229 } else {
201 tree::visit::Action::Continue230 tree::visit::Action::Continue
231 }
232 }
233 fn visit_nontree(&mut self, entry: &Entry<'_>) -> Action {
234 self.entries += 1;
235 self.seen.insert(entry.oid.to_owned());
236 tree::visit::Action::Continue
237 }
202 }238 }
203 }
204 fn visit_nontree(&mut self, entry: &Entry<'_>) -> Action {
205 self.entries += 1;
206 self.seen.insert(entry.oid.to_owned());
207 tree::visit::Action::Continue
208 }
209 }
210 239
211 let mut cache = new_cache();240 let mut cache = new_cache();
212 let mut buf = Vec::new();241 let mut buf = Vec::new();
213 let mut seen = HashSet::new();242 let mut state = tree::breadthfirst::State::default();
214 let mut entries = 0;243 let mut seen = HashSet::new();
244 let mut entries = 0;
215 245
216 for commit in commits {246 for commit in commits {
217 let tid = db247 let tid = db
218 .find(commit, &mut buf, &mut cache)?248 .find(commit, &mut buf, &mut cache)?
219 .and_then(|o| o.into_commit_iter().and_then(|mut c| c.tree_id()))249 .and_then(|o| o.into_commit_iter().and_then(|mut c| c.tree_id()))
220 .expect("commit as starting point");250 .expect("commit as starting point");
251
252 let mut count = Count { entries: 0, seen };
253 tree::breadthfirst::traverse(
254 tid,
255 &mut state,
256 |oid, buf| {
257 db.find(oid, buf, &mut cache)
258 .ok()
259 .flatten()
260 .and_then(|o| o.into_tree_iter())
261 },
262 &mut count,
263 )?;
264 entries += count.entries as u64;
265 seen = count.seen;
266 }
267 Ok((seen.len(), entries))
268 }
269 Computation::MultiThreaded => {
270 struct Count<'a> {
271 entries: usize,
272 seen: &'a DashSet<ObjectId>,
273 }
221 274
222 let mut count = Count { entries: 0, seen };275 impl<'a> tree::visit::Visit for Count<'a> {
223 tree::breadthfirst::traverse(276 type PathId = ();
224 tid,277 fn set_current_path(&mut self, _id: Self::PathId) {}
225 tree::breadthfirst::State::default(),278 fn push_tracked_path_component(&mut self, _component: &BStr) -> Self::PathId {}
226 |oid, buf| {279 fn push_path_component(&mut self, _component: &BStr) {}
227 db.find(oid, buf, &mut cache)280 fn pop_path_component(&mut self) {}
228 .ok()281 fn visit_tree(&mut self, entry: &Entry<'_>) -> Action {
229 .flatten()282 self.entries += 1;
230 .and_then(|o| o.into_tree_iter())283 let inserted = self.seen.insert(entry.oid.to_owned());
231 },284 if !inserted {
232 &mut count,285 tree::visit::Action::Skip
233 )?;286 } else {
234 entries += count.entries as u64;287 tree::visit::Action::Continue
235 seen = count.seen;288 }
289 }
290 fn visit_nontree(&mut self, entry: &Entry<'_>) -> Action {
291 self.entries += 1;
292 self.seen.insert(entry.oid.to_owned());
293 tree::visit::Action::Continue
294 }
295 }
296 use rayon::prelude::*;
297 let seen = DashSet::new();
298 let entries = std::sync::atomic::AtomicU64::new(0);
299
300 commits
301 .into_par_iter()
302 .try_for_each_init::<_, _, _, anyhow::Result<_>>(
303 {
304 let new_cache = &new_cache;
305 let seen = &seen;
306 move || {
307 (
308 Count {
309 entries: 0,
310 seen: &seen,
311 },
312 Vec::<u8>::new(),
313 new_cache(),
314 tree::breadthfirst::State::default(),
315 )
316 }
317 },
318 |(count, buf, cache, state), commit| {
319 let tid = db
320 .find(commit, buf, cache)?
321 .and_then(|o| o.into_commit_iter().and_then(|mut c| c.tree_id()))
322 .expect("commit as starting point");
323 tree::breadthfirst::traverse(
324 tid,
325 state,
326 |oid, buf| db.find(oid, buf, cache).ok().flatten().and_then(|o| o.into_tree_iter()),
327 count,
328 )?;
329 entries.fetch_add(count.entries as u64, std::sync::atomic::Ordering::Relaxed);
330 Ok(())
331 },
332 )?;
333 Ok((seen.len(), entries.load(std::sync::atomic::Ordering::Acquire)))
334 }
236 }335 }
237 Ok((seen.len(), entries))
238}336}
239 337
240fn do_libgit2_tree_dag_traversal(commits: &[ObjectId], db: &git2::Repository) -> anyhow::Result<(usize, u64)> {338fn do_libgit2_tree_dag_traversal(commits: &[ObjectId], db: &git2::Repository) -> anyhow::Result<(usize, u64)> {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t