use std::time::Instant; use anyhow::anyhow; use diff::tree::visit::{Action, Change}; use git_repository::{ diff, easy::object, hash::{oid, ObjectId}, objs::{bstr::BStr, TreeRefIter}, odb, prelude::*, refs::{file::ReferenceExt, peel}, }; use rayon::prelude::*; const GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES: usize = 60_000_000; fn cache_size() -> usize { std::env::var("GITOXIDE_OBJECT_CACHE_IN_BYTES") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES) } fn main() -> anyhow::Result<()> { let mut args = std::env::args(); let repo_git_dir = args .nth(1) .ok_or_else(|| anyhow!("First argument is the .git directory to work in"))?; let repo = git_repository::discover(repo_git_dir)?; let name = args.next().unwrap_or_else(|| "HEAD".into()); let commit_id = repo .refs .find(&name)? .peel_to_id_in_place(&repo.refs, peel::none)? .to_owned(); let db = &repo.odb; let start = Instant::now(); let all_commits = commit_id .ancestors(|oid, buf| db.find_commit_iter(oid, buf, &mut odb::pack::cache::Never).ok()) .collect::, _>>()?; let num_diffs = all_commits.len(); let elapsed = start.elapsed(); println!( "gitoxide (uncached): collect all {} commits in {:?} ({:0.0} commits/s)", all_commits.len(), elapsed, all_commits.len() as f32 / elapsed.as_secs_f32() ); struct ObjectInfo { kind: object::Kind, data: Vec, } impl memory_lru::ResidentSize for ObjectInfo { fn resident_size(&self) -> usize { self.data.len() } } fn find_with_lru_mem_cache<'b>( oid: &oid, buf: &'b mut Vec, obj_cache: &mut memory_lru::MemoryLruCache, db: &odb::linked::Store, pack_cache: &mut impl odb::pack::cache::DecodeEntry, ) -> Option> { let oid = oid.to_owned(); match obj_cache.get(&oid) { Some(ObjectInfo { kind, data }) => { buf.resize(data.len(), 0); buf.copy_from_slice(data); Some(odb::data::Object::new(*kind, buf)) } None => { let obj = db.find(oid, buf, pack_cache).ok(); if let Some(ref obj) = obj { obj_cache.insert( oid, ObjectInfo { kind: obj.kind, data: obj.data.to_owned(), }, ); } obj } } } let start = Instant::now(); let num_deltas = do_gitoxide_tree_diff( &all_commits, || { let mut pack_cache = odb::pack::cache::lru::MemoryCappedHashmap::new(cache_size()); let db = &db; let mut obj_cache = memory_lru::MemoryLruCache::new(cache_size()); move |oid, buf: &mut Vec| find_with_lru_mem_cache(oid, buf, &mut obj_cache, db, &mut pack_cache) }, Computation::MultiThreaded, )?; let elapsed = start.elapsed(); println!( "gitoxide-deltas PARALLEL (cache = memory-LRU -> {:.0}MB | pack -> {:.0}MB): collect {} tree deltas of {} trees-diffs in {:?} ({:0.0} deltas/s, {:0.0} tree-diffs/s)", cache_size() as f32 / (1024 * 1024) as f32, cache_size() as f32 / (1024 * 1024) as f32, num_deltas, num_diffs, elapsed, num_deltas as f32 / elapsed.as_secs_f32(), num_diffs as f32 / elapsed.as_secs_f32() ); let start = Instant::now(); let num_deltas = do_gitoxide_tree_diff( &all_commits, || { let mut pack_cache = odb::pack::cache::lru::MemoryCappedHashmap::new(cache_size()); let db = &db; let mut obj_cache = memory_lru::MemoryLruCache::new(cache_size()); move |oid, buf: &mut Vec| find_with_lru_mem_cache(oid, buf, &mut obj_cache, db, &mut pack_cache) }, Computation::SingleThreaded, )?; let elapsed = start.elapsed(); println!( "gitoxide-deltas (cache = memory-LRU -> {:.0}MB | pack -> {:.0}MB): collect {} tree deltas of {} trees-diffs in {:?} ({:0.0} deltas/s, {:0.0} tree-diffs/s)", cache_size() as f32 / (1024 * 1024) as f32, cache_size() as f32 / (1024 * 1024) as f32, num_deltas, num_diffs, elapsed, num_deltas as f32 / elapsed.as_secs_f32(), num_diffs as f32 / elapsed.as_secs_f32() ); let start = Instant::now(); let num_deltas = do_libgit2_treediff(&all_commits, repo.git_dir(), Computation::MultiThreaded)?; let elapsed = start.elapsed(); println!( "libgit2 PARALLEL: collect {} tree deltas of {} trees-diffs in {:?} ({:0.0} deltas/s, {:0.0} tree-diffs/s)", num_deltas, num_diffs, elapsed, num_deltas as f32 / elapsed.as_secs_f32(), num_diffs as f32 / elapsed.as_secs_f32() ); let start = Instant::now(); let num_deltas = do_libgit2_treediff(&all_commits, repo.git_dir(), Computation::SingleThreaded)?; let elapsed = start.elapsed(); println!( "libgit2: collect {} tree deltas of {} trees-diffs in {:?} ({:0.0} deltas/s, {:0.0} tree-diffs/s)", num_deltas, num_diffs, elapsed, num_deltas as f32 / elapsed.as_secs_f32(), num_diffs as f32 / elapsed.as_secs_f32() ); Ok(()) } enum Computation { SingleThreaded, MultiThreaded, } fn do_libgit2_treediff(commits: &[ObjectId], repo_dir: &std::path::Path, mode: Computation) -> anyhow::Result { Ok(match mode { Computation::SingleThreaded => { let db = git2::Repository::open(repo_dir)?; let mut changes = 0; for pair in commits.windows(2) { let (ca, cb) = (pair[0], pair[1]); let ta = db.find_commit(git2::Oid::from_bytes(ca.as_bytes())?)?.tree()?; let tb = db.find_commit(git2::Oid::from_bytes(cb.as_bytes())?)?.tree()?; let diff = db.diff_tree_to_tree(Some(&ta), Some(&tb), None)?; changes += diff.deltas().count(); } changes } Computation::MultiThreaded => { let changes = std::sync::atomic::AtomicUsize::new(0); commits.par_windows(2).try_for_each_init::<_, _, _, anyhow::Result<_>>( || git2::Repository::open(repo_dir).expect("git directory is valid"), |db, pair| { let (ca, cb) = (pair[0], pair[1]); let ta = db.find_commit(git2::Oid::from_bytes(ca.as_bytes())?)?.tree()?; let tb = db.find_commit(git2::Oid::from_bytes(cb.as_bytes())?)?.tree()?; let diff = db.diff_tree_to_tree(Some(&ta), Some(&tb), None)?; changes.fetch_add(diff.deltas().count(), std::sync::atomic::Ordering::Relaxed); Ok(()) }, )?; changes.load(std::sync::atomic::Ordering::Acquire) } }) } fn do_gitoxide_tree_diff(commits: &[ObjectId], make_find: C, mode: Computation) -> anyhow::Result where C: Fn() -> L + Sync, L: for<'b> FnMut(&oid, &'b mut Vec) -> Option>, { let changes: usize = match mode { Computation::MultiThreaded => { let changes = std::sync::atomic::AtomicUsize::new(0); commits.par_windows(2).try_for_each_init::<_, _, _, anyhow::Result<_>>( || { ( diff::tree::State::default(), Vec::::new(), Vec::::new(), make_find(), ) }, |(state, buf1, buf2, find), pair| { let (ca, cb) = (pair[0], pair[1]); let (ta, tb) = ( tree_iter_by_commit(&ca, buf1, &mut *find), tree_iter_by_commit(&cb, buf2, &mut *find), ); let mut count = Count::default(); ta.changes_needed(tb, state, |id, buf| find_tree_iter(id, buf, &mut *find), &mut count)?; changes.fetch_add(count.0, std::sync::atomic::Ordering::Relaxed); Ok(()) }, )?; changes.load(std::sync::atomic::Ordering::Acquire) } Computation::SingleThreaded => { let mut state = diff::tree::State::default(); let mut find = make_find(); let mut buf: Vec = Vec::new(); let mut buf2: Vec = Vec::new(); let mut changes = 0; for pair in commits.windows(2) { let (ca, cb) = (pair[0], pair[1]); let (ta, tb) = ( tree_iter_by_commit(&ca, &mut buf, &mut find), tree_iter_by_commit(&cb, &mut buf2, &mut find), ); let mut count = Count::default(); ta.changes_needed(tb, &mut state, |id, buf| find_tree_iter(id, buf, &mut find), &mut count)?; changes += count.0; } changes } }; fn find_tree_iter<'b, L>(id: &oid, buf: &'b mut Vec, mut find: L) -> Option> where L: for<'a> FnMut(&oid, &'a mut Vec) -> Option>, { find(id, buf).and_then(|o| o.try_into_tree_iter()) } fn tree_iter_by_commit<'b, L>(id: &oid, buf: &'b mut Vec, mut find: L) -> TreeRefIter<'b> where L: for<'a> FnMut(&oid, &'a mut Vec) -> Option>, { let tid = find(id, buf) .expect("commit present") .try_into_commit_iter() .expect("a commit") .tree_id() .expect("tree id present and decodable"); find_tree_iter(&tid, buf, find).expect("tree available") } #[derive(Default)] struct Count(usize); impl diff::tree::Visit for Count { fn pop_front_tracked_path_and_set_current(&mut self) {} fn push_back_tracked_path_component(&mut self, _component: &BStr) {} fn push_path_component(&mut self, _component: &BStr) {} fn pop_path_component(&mut self) {} fn visit(&mut self, _change: Change) -> Action { self.0 += 1; Action::Continue } } Ok(changes) }