#[cfg(any(feature = "tar", feature = "tar_gz", feature = "zip"))] use gix_error::ResultExt; use gix_error::{ErrorExt, message}; use gix_worktree_stream::{Entry, Stream}; use crate::{Error, Format, Options}; #[cfg(feature = "zip")] use std::io::Write; /// Write all stream entries in `stream` as provided by `next_entry(stream)` to `out` configured according to `opts` which /// also includes the streaming format. /// /// ### Performance /// /// * The caller should be sure `out` is fast enough. If in doubt, wrap in [`std::io::BufWriter`]. /// * Further, big files aren't suitable for archival into `tar` archives as they require the size of the stream to be known /// prior to writing the header of each entry. #[cfg_attr(not(feature = "tar"), allow(unused_mut, unused_variables))] pub fn write_stream( stream: &mut Stream, mut next_entry: NextFn, out: impl std::io::Write, opts: Options, ) -> Result<(), Error> where NextFn: FnMut(&mut Stream) -> Result>, gix_error::Exn>, { if opts.format == Format::InternalTransientNonPersistable { return Err(message("The internal format cannot be used as an archive, it's merely a debugging tool").raise()); } #[cfg(any(feature = "tar", feature = "tar_gz"))] { enum State { #[cfg(feature = "tar")] Tar((tar::Builder, Vec)), #[cfg(feature = "tar_gz")] TarGz((tar::Builder>, Vec)), } impl State { pub fn new(format: Format, mtime: gix_date::SecondsSinceUnixEpoch, out: W) -> Result { match format { Format::InternalTransientNonPersistable => unreachable!("handled earlier"), Format::Zip { .. } => { Err(message("Cannot create a zip archive if output stream does not support seek").raise()) } Format::Tar => { #[cfg(feature = "tar")] { Ok(State::Tar(( { let mut ar = tar::Builder::new(out); ar.mode(tar::HeaderMode::Deterministic); ar }, Vec::with_capacity(64 * 1024), ))) } #[cfg(not(feature = "tar"))] { Err(message!("Support for the format '{:?}' was not compiled in", Format::Tar).raise()) } } Format::TarGz { compression_level } => { #[cfg(feature = "tar_gz")] { Ok(State::TarGz(( { let gz = flate2::GzBuilder::new().mtime(mtime as u32).write( out, match compression_level { None => flate2::Compression::default(), Some(level) => flate2::Compression::new(u32::from(level)), }, ); let mut ar = tar::Builder::new(gz); ar.mode(tar::HeaderMode::Deterministic); ar }, Vec::with_capacity(64 * 1024), ))) } #[cfg(not(feature = "tar_gz"))] { Err(message!( "Support for the format '{:?}' was not compiled in", Format::TarGz { compression_level: None, } ) .raise()) } } } } } let mut state = State::new(opts.format, opts.modification_time, out)?; while let Some(entry) = next_entry(stream).or_raise(|| message("Could not retrieve the next entry from the stream"))? { match &mut state { #[cfg(feature = "tar")] State::Tar((ar, buf)) => { append_tar_entry(ar, buf, entry, opts.modification_time, &opts)?; } #[cfg(feature = "tar_gz")] State::TarGz((ar, buf)) => { append_tar_entry(ar, buf, entry, opts.modification_time, &opts)?; } } } match state { #[cfg(feature = "tar")] State::Tar((mut ar, _)) => { ar.finish().or_raise(|| message("Could not finish tar archive"))?; } #[cfg(feature = "tar_gz")] State::TarGz((ar, _)) => { ar.into_inner() .or_raise(|| message("Could not finish tar.gz archive"))? .finish() .or_raise(|| message("Could not finish gzip stream"))?; } } } #[cfg(not(any(feature = "tar", feature = "tar_gz")))] { let _ = (next_entry, out); return Err(message!("Support for the format '{:?}' was not compiled in", opts.format).raise()); } #[allow( unreachable_code, reason = "Reachability depends on the enabled archive format features." )] Ok(()) } /// Like [`write_stream()`], but requires [`std::io::Seek`] for `out`. /// /// Note that `zip` is able to stream big files, which our `tar` implementation is not able to do, which makes it the /// only suitable container to support huge files from `git-lfs` without consuming excessive amounts of memory. #[cfg_attr(not(feature = "zip"), allow(unused_mut, unused_variables))] pub fn write_stream_seek( stream: &mut Stream, mut next_entry: NextFn, out: impl std::io::Write + std::io::Seek, opts: Options, ) -> Result<(), Error> where NextFn: FnMut(&mut Stream) -> Result>, gix_error::Exn>, { let compression_level = match opts.format { Format::Zip { compression_level } => compression_level.map(i64::from), _other => return write_stream(stream, next_entry, out, opts), }; #[cfg(feature = "zip")] { let mut ar = rawzip::ZipArchiveWriter::new(out); let mut buf = Vec::new(); let mtime = rawzip::time::UtcDateTime::from_unix(opts.modification_time); while let Some(entry) = next_entry(stream).or_raise(|| message("Could not retrieve the next entry from the stream"))? { append_zip_entry( &mut ar, entry, &mut buf, mtime, compression_level, opts.tree_prefix.as_ref(), )?; } ar.finish().or_raise(|| message("Could not finish zip archive"))?; } #[cfg(not(feature = "zip"))] { let _ = compression_level; #[expect( clippy::needless_return, reason = "the explicit return keeps feature-dependent branches structurally consistent" )] return Err(message!( "Support for the format '{:?}' was not compiled in", Format::Zip { compression_level: None } ) .raise()); } #[cfg(feature = "zip")] Ok(()) } #[cfg(feature = "zip")] fn append_zip_entry( ar: &mut rawzip::ZipArchiveWriter, mut entry: gix_worktree_stream::Entry<'_>, buf: &mut Vec, mtime: rawzip::time::UtcDateTime, compression_level: Option, tree_prefix: Option<&bstr::BString>, ) -> Result<(), Error> { use bstr::ByteSlice; let path = add_prefix(entry.relative_path(), tree_prefix).into_owned(); let unix_permissions = if entry.mode.is_executable() { 0o755 } else { 0o644 }; let path = path .to_str() .or_raise(|| message!("Invalid UTF-8 in entry path: {path:?}"))?; match entry.mode.kind() { gix_object::tree::EntryKind::Blob | gix_object::tree::EntryKind::BlobExecutable => { let file_builder = ar .new_file(path) .compression_method(rawzip::CompressionMethod::Deflate) .last_modified(mtime) .unix_permissions(unix_permissions); let (mut zip_entry, config) = file_builder .start() .or_raise(|| message("Could not start zip file entry"))?; // Use flate2 for compression. Level 9 is the maximum compression level for deflate. let encoder = flate2::write::DeflateEncoder::new( &mut zip_entry, match compression_level { None => flate2::Compression::default(), Some(level) => flate2::Compression::new(level.clamp(0, 9) as u32), }, ); let mut writer = config.wrap(encoder); std::io::copy(&mut entry, &mut writer).or_raise(|| message("Could not write zip entry data"))?; let (encoder, descriptor) = writer .finish() .or_raise(|| message("Could not finish zip entry writer"))?; encoder .finish() .or_raise(|| message("Could not finish deflate encoder"))?; zip_entry .finish(descriptor) .or_raise(|| message("Could not finish zip entry"))?; } gix_object::tree::EntryKind::Tree | gix_object::tree::EntryKind::Commit => { // rawzip requires directory paths to end with '/' let mut dir_path = path.to_owned(); if !dir_path.ends_with('/') { dir_path.push('/'); } ar.new_dir(&dir_path) .last_modified(mtime) .unix_permissions(unix_permissions) .create() .or_raise(|| message("Could not create zip directory entry"))?; } gix_object::tree::EntryKind::Link => { buf.clear(); std::io::copy(&mut entry, buf).or_raise(|| message("Could not read symlink target"))?; // For symlinks, we need to create a file with symlink permissions let symlink_path = path; let target = buf.as_bstr().to_str().or_raise(|| { message!( "Invalid UTF-8 in symlink target for entry '{symlink_path}': {:?}", buf.as_bstr() ) })?; let (mut zip_entry, config) = ar .new_file(symlink_path) .compression_method(rawzip::CompressionMethod::Store) .last_modified(mtime) .unix_permissions(0o120644) // Symlink mode .start() .or_raise(|| message("Could not start zip symlink entry"))?; let mut writer = config.wrap(&mut zip_entry); writer .write_all(target.as_bytes()) .or_raise(|| message("Could not write symlink target"))?; let (_, descriptor) = writer .finish() .or_raise(|| message("Could not finish zip symlink writer"))?; zip_entry .finish(descriptor) .or_raise(|| message("Could not finish zip symlink entry"))?; } } Ok(()) } #[cfg(any(feature = "tar", feature = "tar_gz"))] fn append_tar_entry( ar: &mut tar::Builder, buf: &mut Vec, mut entry: gix_worktree_stream::Entry<'_>, mtime_seconds_since_epoch: i64, opts: &Options, ) -> Result<(), Error> { let mut header = tar::Header::new_gnu(); header.set_mtime(mtime_seconds_since_epoch as u64); header.set_entry_type(tar_entry_type(entry.mode)); header.set_mode(if entry.mode.is_executable() { 0o755 } else { 0o644 }); buf.clear(); std::io::copy(&mut entry, buf).or_raise(|| message("Could not read entry data"))?; let path = gix_path::from_bstr(add_prefix(entry.relative_path(), opts.tree_prefix.as_ref())); header.set_size(buf.len() as u64); if entry.mode.is_link() { use bstr::ByteSlice; let target = gix_path::from_bstr(buf.as_bstr()); header.set_entry_type(tar::EntryType::Symlink); header.set_size(0); ar.append_link(&mut header, path, target) .or_raise(|| message("Could not append symlink to tar archive"))?; } else { ar.append_data(&mut header, path, buf.as_slice()) .or_raise(|| message("Could not append data to tar archive"))?; } Ok(()) } #[cfg(any(feature = "tar", feature = "tar_gz"))] fn tar_entry_type(mode: gix_object::tree::EntryMode) -> tar::EntryType { use gix_object::tree::EntryKind; use tar::EntryType; match mode.kind() { EntryKind::Tree | EntryKind::Commit => EntryType::Directory, EntryKind::Blob => EntryType::Regular, EntryKind::BlobExecutable => EntryType::Regular, EntryKind::Link => EntryType::Link, } } #[cfg(any(feature = "tar", feature = "tar_gz", feature = "zip"))] fn add_prefix<'a>(relative_path: &'a bstr::BStr, prefix: Option<&bstr::BString>) -> std::borrow::Cow<'a, bstr::BStr> { use std::borrow::Cow; match prefix { None => Cow::Borrowed(relative_path), Some(prefix) => { use bstr::ByteVec; let mut buf = prefix.clone(); buf.push_str(relative_path); Cow::Owned(buf) } } }