use std::{path::Path, sync::Arc, time::Instant}; use anyhow::anyhow; use git_repository::{hash::ObjectId, odb, threading::OwnShared, ThreadSafeRepository}; use crate::odb::Cache; const GITOXIDE_STATIC_CACHE_SIZE: usize = 64; const GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES: usize = 60_000_000; fn main() -> anyhow::Result<()> { let repo_git_dir = std::env::args() .nth(1) .ok_or_else(|| anyhow!("First argument is the .git directory to work in"))?; let repo = git_repository::discover(&repo_git_dir)?.into_sync(); let hashes = { let start = Instant::now(); let repo = git_repository::discover(repo_git_dir)?; let hashes = repo.objects.iter()?.collect::, _>>()?; let elapsed = start.elapsed(); println!("gitoxide: {} objects (collected in {:?}", hashes.len(), elapsed); hashes }; let objs_per_sec = |elapsed: std::time::Duration| hashes.len() as f32 / elapsed.as_secs_f32(); let start = Instant::now(); do_gitoxide_in_parallel_sync(&hashes, &repo, || odb::pack::cache::Never, AccessMode::ObjectExists)?; let elapsed = start.elapsed(); println!( "parallel gitoxide : confirmed {} objects exists in {:?} ({:0.0} objects/s)", hashes.len(), elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let (object_store, _) = do_new_gitoxide_store_in_parallel( &hashes, repo.objects_dir(), || odb::pack::cache::Never, AccessMode::ObjectExists, None, )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (new store): confirmed {} objects exists in {:?} ({:0.0} objects/s)", hashes.len(), elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); do_new_gitoxide_store_in_parallel( &hashes, repo.objects_dir(), || odb::pack::cache::Never, AccessMode::ObjectExists, Some(object_store), )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (new store, warm): confirmed {} objects exists in {:?} ({:0.0} objects/s)", hashes.len(), elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); do_gitoxide_in_parallel_through_arc( &hashes, repo.objects_dir(), odb::pack::cache::Never::default, AccessMode::ObjectExists, )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (Arc): confirmed {} objects exists in {:?} ({:0.0} objects/s)", hashes.len(), elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); do_parallel_git2(hashes.as_slice(), repo.git_dir(), AccessMode::ObjectExists)?; let elapsed = start.elapsed(); println!( "parallel libgit2: confirmed {} exist in {:?} ({:0.0} objects/s)", hashes.len(), elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); do_git2(hashes.as_slice(), repo.git_dir(), AccessMode::ObjectExists)?; let elapsed = start.elapsed(); println!( "libgit2: confirmed {} exist in {:?} ({:0.0} objects/s)", hashes.len(), elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let (object_store, bytes) = do_new_gitoxide_store_in_parallel( &hashes, repo.objects_dir(), || odb::pack::cache::Never, AccessMode::ObjectData, None, )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (new store, uncached): confirmed {} bytes exists in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let (object_store, bytes) = do_new_gitoxide_store_in_parallel( &hashes, repo.objects_dir(), || odb::pack::cache::Never, AccessMode::ObjectData, Some(object_store), )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (new store, uncached, warm): confirmed {} bytes exists in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let (_, bytes) = do_new_gitoxide_store_in_parallel( &hashes, repo.objects_dir(), || odb::pack::cache::lru::MemoryCappedHashmap::new(GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES), AccessMode::ObjectData, Some(object_store), )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (new store, cache = {:.0}MB), warm): confirmed {} bytes exists in {:?} ({:0.0} objects/s)", GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES as f32 / (1024 * 1024) as f32, bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_gitoxide_in_parallel_sync(&hashes, &repo, odb::pack::cache::Never::default, AccessMode::ObjectData)?; let elapsed = start.elapsed(); println!( "parallel gitoxide (uncached): confirmed {} bytes in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_gitoxide_in_parallel_through_arc( &hashes, repo.objects_dir(), odb::pack::cache::Never::default, AccessMode::ObjectData, )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (uncached, Arc): confirmed {} bytes in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_gitoxide_in_parallel_sync( &hashes, &repo, || odb::pack::cache::lru::MemoryCappedHashmap::new(GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES), AccessMode::ObjectData, )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (cache = {:.0}MB): confirmed {} bytes in {:?} ({:0.0} objects/s)", GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES as f32 / (1024 * 1024) as f32, bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_gitoxide_in_parallel_sync( &hashes, &repo, odb::pack::cache::lru::StaticLinkedList::::default, AccessMode::ObjectData, )?; let elapsed = start.elapsed(); println!( "parallel gitoxide (small static cache): confirmed {} bytes in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_git2(hashes.as_slice(), repo.git_dir(), AccessMode::ObjectData)?; let elapsed = start.elapsed(); println!( "libgit2: confirmed {} bytes in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_gitoxide(&hashes, &repo, || { odb::pack::cache::lru::MemoryCappedHashmap::new(GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES) })?; let elapsed = start.elapsed(); let objs_per_sec = |elapsed: std::time::Duration| hashes.len() as f32 / elapsed.as_secs_f32(); println!( "gitoxide (cache = {:.0}MB): confirmed {} bytes in {:?} ({:0.0} objects/s)", GITOXIDE_CACHED_OBJECT_DATA_PER_THREAD_IN_BYTES as f32 / (1024 * 1024) as f32, bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_gitoxide( &hashes, &repo, odb::pack::cache::lru::StaticLinkedList::::default, )?; let elapsed = start.elapsed(); println!( "gitoxide (small static cache): confirmed {} bytes in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_gitoxide(&hashes, &repo, odb::pack::cache::Never::default)?; let elapsed = start.elapsed(); println!( "gitoxide (uncached): confirmed {} bytes in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); let start = Instant::now(); let bytes = do_parallel_git2(hashes.as_slice(), repo.git_dir(), AccessMode::ObjectData)?; let elapsed = start.elapsed(); println!( "parallel libgit2: confirmed {} bytes in {:?} ({:0.0} objects/s)", bytes, elapsed, objs_per_sec(elapsed) ); Ok(()) } fn do_git2(hashes: &[ObjectId], git_dir: &Path, mode: AccessMode) -> anyhow::Result { git2::opts::strict_hash_verification(false); let repo = git2::Repository::open(git_dir)?; let odb = repo.odb()?; let mut bytes = 0u64; for hash in hashes { let hash = git2::Oid::from_bytes(hash.as_slice())?; match mode { AccessMode::ObjectData => { let obj = odb.read(hash)?; bytes += obj.len() as u64; } AccessMode::ObjectExists => { assert!(odb.exists(hash)); } } } Ok(bytes) } fn do_parallel_git2(hashes: &[ObjectId], git_dir: &Path, mode: AccessMode) -> anyhow::Result { use rayon::prelude::*; git2::opts::strict_hash_verification(false); let bytes = std::sync::atomic::AtomicU64::default(); hashes.par_iter().try_for_each_init::<_, _, _, anyhow::Result<_>>( || git2::Repository::open(git_dir).expect("git directory is valid"), |repo, hash| { let odb = repo.odb()?; let hash = git2::Oid::from_bytes(hash.as_slice())?; match mode { AccessMode::ObjectData => { let obj = odb.read(hash)?; bytes.fetch_add(obj.len() as u64, std::sync::atomic::Ordering::Relaxed); } AccessMode::ObjectExists => { assert!(odb.exists(hash)); } } Ok(()) }, )?; Ok(bytes.load(std::sync::atomic::Ordering::Acquire)) } fn do_gitoxide( hashes: &[ObjectId], repo: &ThreadSafeRepository, new_cache: impl Fn() -> C + Send + Sync + 'static, ) -> anyhow::Result where C: odb::pack::cache::DecodeEntry + Send + 'static, { use git_repository::prelude::FindExt; let mut buf = Vec::new(); let mut bytes = 0u64; let handle = repo.objects.to_cache().with_pack_cache(move || Box::new(new_cache())); for hash in hashes { let obj = handle.find(hash, &mut buf)?; bytes += obj.data.len() as u64; } Ok(bytes) } enum AccessMode { ObjectData, ObjectExists, } fn do_gitoxide_in_parallel_sync( hashes: &[ObjectId], repo: &ThreadSafeRepository, new_cache: impl Fn() -> C + Send + Clone + Sync + 'static, mode: AccessMode, ) -> anyhow::Result where C: odb::pack::cache::DecodeEntry + Send + 'static, { use git_repository::prelude::FindExt; use rayon::prelude::*; let bytes = std::sync::atomic::AtomicU64::default(); hashes.par_iter().try_for_each_init::<_, _, _, anyhow::Result<_>>( move || { ( Vec::new(), repo.objects.to_cache().with_pack_cache({ let new_cache = new_cache.clone(); move || Box::new(new_cache()) }), ) }, |(buf, objects), hash| { match mode { AccessMode::ObjectData => { let obj = objects.find(hash, buf)?; bytes.fetch_add(obj.data.len() as u64, std::sync::atomic::Ordering::Relaxed); } AccessMode::ObjectExists => { use git_repository::prelude::Find; assert!(objects.contains(hash), "each traversed object exists"); } } Ok(()) }, )?; Ok(bytes.load(std::sync::atomic::Ordering::Acquire)) } fn do_new_gitoxide_store_in_parallel( hashes: &[ObjectId], objects_dir: &Path, new_cache: impl Fn() -> C + Send + Sync + 'static, mode: AccessMode, store: Option>, ) -> anyhow::Result<(std::sync::Arc, u64)> where C: odb::pack::cache::DecodeEntry + Send + 'static, { use git_repository::prelude::FindExt; let bytes = std::sync::atomic::AtomicU64::default(); let slots = std::env::args() .nth(2) .and_then(|num| num.parse().ok().map(odb::store::init::Slots::Given)) .unwrap_or_default(); let store = match store { Some(store) => store, None => OwnShared::new(odb::Store::at_opts( objects_dir, Vec::new(), git_repository::odb::store::init::Options { slots, ..Default::default() }, )?), }; let handle = Cache::from(store.to_handle()).with_pack_cache(move || Box::new(new_cache())); git_repository::parallel::in_parallel( hashes.chunks(1000), None, move |_| (Vec::new(), handle.clone()), |hashes, (buf, cache)| { for hash in hashes { match mode { AccessMode::ObjectData => { let obj = cache.find(hash, buf)?; bytes.fetch_add(obj.data.len() as u64, std::sync::atomic::Ordering::Relaxed); } AccessMode::ObjectExists => { use git_repository::prelude::Find; assert!(cache.contains(hash), "each traversed object exists"); } } } Ok(()) }, git_repository::parallel::reduce::IdentityWithResult::<(), anyhow::Error>::default(), )?; Ok((store, bytes.load(std::sync::atomic::Ordering::Acquire))) } fn do_gitoxide_in_parallel_through_arc( hashes: &[ObjectId], objects_dir: &Path, new_cache: impl Fn() -> C + Send + Sync + Clone + 'static, mode: AccessMode, ) -> anyhow::Result where C: odb::pack::cache::DecodeEntry + Send + 'static, { use git_repository::prelude::FindExt; let bytes = std::sync::atomic::AtomicU64::default(); #[allow(deprecated)] let odb = Arc::new(odb::linked::Store::at(objects_dir)?); git_repository::parallel::in_parallel( hashes.chunks(1000), None, move |_| { ( Vec::new(), odb.to_cache_arc().with_pack_cache({ let new_cache = new_cache.clone(); move || Box::new(new_cache()) }), ) }, |hashes, (buf, odb)| { for hash in hashes { match mode { AccessMode::ObjectData => { let obj = odb.find(hash, buf)?; bytes.fetch_add(obj.data.len() as u64, std::sync::atomic::Ordering::Relaxed); } AccessMode::ObjectExists => { use git_repository::prelude::Find; assert!(odb.contains(hash), "each traversed object exists"); } } } Ok(()) }, git_repository::parallel::reduce::IdentityWithResult::<(), anyhow::Error>::default(), )?; Ok(bytes.load(std::sync::atomic::Ordering::Acquire)) }