#[cfg(any(feature = "tar", feature = "tar_gz", feature = "zip"))]use gix_error::ResultExt;use gix_error::{ErrorExt, message};use gix_worktree_stream::{Entry, Stream}; use crate::{Error, Format, Options}; #[cfg(feature = "zip")]use std::io::Write; /// Write all stream entries in `stream` as provided by `next_entry(stream)` to `out` configured according to `opts` which/// also includes the streaming format.////// ### Performance////// * The caller should be sure `out` is fast enough. If in doubt, wrap in [`std::io::BufWriter`]./// * Further, big files aren't suitable for archival into `tar` archives as they require the size of the stream to be known/// prior to writing the header of each entry.#[cfg_attr(not(feature = "tar"), allow(unused_mut, unused_variables))]pub fn write_stream<NextFn>( stream: &mut Stream, mut next_entry: NextFn, out: impl std::io::Write, opts: Options,) -> Result<(), Error>where NextFn: FnMut(&mut Stream) -> Result<Option<Entry<'_>>, gix_error::Exn>,{ if opts.format == Format::InternalTransientNonPersistable { return Err(message("The internal format cannot be used as an archive, it's merely a debugging tool").raise()); } #[cfg(any(feature = "tar", feature = "tar_gz"))] { enum State<W: std::io::Write> { #[cfg(feature = "tar")] Tar((tar::Builder<W>, Vec<u8>)), #[cfg(feature = "tar_gz")] TarGz((tar::Builder<flate2::write::GzEncoder<W>>, Vec<u8>)), } impl<W: std::io::Write> State<W> { pub fn new(format: Format, mtime: gix_date::SecondsSinceUnixEpoch, out: W) -> Result<Self, Error> { match format { Format::InternalTransientNonPersistable => unreachable!("handled earlier"), Format::Zip { .. } => { Err(message("Cannot create a zip archive if output stream does not support seek").raise()) } Format::Tar => { #[cfg(feature = "tar")] { Ok(State::Tar(( { let mut ar = tar::Builder::new(out); ar.mode(tar::HeaderMode::Deterministic); ar }, Vec::with_capacity(64 * 1024), ))) } #[cfg(not(feature = "tar"))] { Err(message!("Support for the format '{:?}' was not compiled in", Format::Tar).raise()) } } Format::TarGz { compression_level } => { #[cfg(feature = "tar_gz")] { Ok(State::TarGz(( { let gz = flate2::GzBuilder::new().mtime(mtime as u32).write( out, match compression_level { None => flate2::Compression::default(), Some(level) => flate2::Compression::new(u32::from(level)), }, ); let mut ar = tar::Builder::new(gz); ar.mode(tar::HeaderMode::Deterministic); ar }, Vec::with_capacity(64 * 1024), ))) } #[cfg(not(feature = "tar_gz"))] { Err(message!( "Support for the format '{:?}' was not compiled in", Format::TarGz { compression_level: None, } ) .raise()) } } } } } let mut state = State::new(opts.format, opts.modification_time, out)?; while let Some(entry) = next_entry(stream).or_raise(|| message("Could not retrieve the next entry from the stream"))? { match &mut state { #[cfg(feature = "tar")] State::Tar((ar, buf)) => { append_tar_entry(ar, buf, entry, opts.modification_time, &opts)?; } #[cfg(feature = "tar_gz")] State::TarGz((ar, buf)) => { append_tar_entry(ar, buf, entry, opts.modification_time, &opts)?; } } } match state { #[cfg(feature = "tar")] State::Tar((mut ar, _)) => { ar.finish().or_raise(|| message("Could not finish tar archive"))?; } #[cfg(feature = "tar_gz")] State::TarGz((ar, _)) => { ar.into_inner() .or_raise(|| message("Could not finish tar.gz archive"))? .finish() .or_raise(|| message("Could not finish gzip stream"))?; } } } #[cfg(not(any(feature = "tar", feature = "tar_gz")))] { let _ = (next_entry, out); return Err(message!("Support for the format '{:?}' was not compiled in", opts.format).raise()); } #[allow( unreachable_code, reason = "Reachability depends on the enabled archive format features." )] Ok(())} /// Like [`write_stream()`], but requires [`std::io::Seek`] for `out`.////// Note that `zip` is able to stream big files, which our `tar` implementation is not able to do, which makes it the/// only suitable container to support huge files from `git-lfs` without consuming excessive amounts of memory.#[cfg_attr(not(feature = "zip"), allow(unused_mut, unused_variables))]pub fn write_stream_seek<NextFn>( stream: &mut Stream, mut next_entry: NextFn, out: impl std::io::Write + std::io::Seek, opts: Options,) -> Result<(), Error>where NextFn: FnMut(&mut Stream) -> Result<Option<Entry<'_>>, gix_error::Exn>,{ let compression_level = match opts.format { Format::Zip { compression_level } => compression_level.map(i64::from), _other => return write_stream(stream, next_entry, out, opts), }; #[cfg(feature = "zip")] { let mut ar = rawzip::ZipArchiveWriter::new(out); let mut buf = Vec::new(); let mtime = rawzip::time::UtcDateTime::from_unix(opts.modification_time); while let Some(entry) = next_entry(stream).or_raise(|| message("Could not retrieve the next entry from the stream"))? { append_zip_entry( &mut ar, entry, &mut buf, mtime, compression_level, opts.tree_prefix.as_ref(), )?; } ar.finish().or_raise(|| message("Could not finish zip archive"))?; } #[cfg(not(feature = "zip"))] { let _ = compression_level; #[expect( clippy::needless_return, reason = "the explicit return keeps feature-dependent branches structurally consistent" )] return Err(message!( "Support for the format '{:?}' was not compiled in", Format::Zip { compression_level: None } ) .raise()); } #[cfg(feature = "zip")] Ok(())} #[cfg(feature = "zip")]fn append_zip_entry<W: std::io::Write + std::io::Seek>( ar: &mut rawzip::ZipArchiveWriter<W>, mut entry: gix_worktree_stream::Entry<'_>, buf: &mut Vec<u8>, mtime: rawzip::time::UtcDateTime, compression_level: Option<i64>, tree_prefix: Option<&bstr::BString>,) -> Result<(), Error> { use bstr::ByteSlice; let path = add_prefix(entry.relative_path(), tree_prefix).into_owned(); let unix_permissions = if entry.mode.is_executable() { 0o755 } else { 0o644 }; let path = path .to_str() .or_raise(|| message!("Invalid UTF-8 in entry path: {path:?}"))?; match entry.mode.kind() { gix_object::tree::EntryKind::Blob | gix_object::tree::EntryKind::BlobExecutable => { let file_builder = ar .new_file(path) .compression_method(rawzip::CompressionMethod::Deflate) .last_modified(mtime) .unix_permissions(unix_permissions); let (mut zip_entry, config) = file_builder .start() .or_raise(|| message("Could not start zip file entry"))?; // Use flate2 for compression. Level 9 is the maximum compression level for deflate. let encoder = flate2::write::DeflateEncoder::new( &mut zip_entry, match compression_level { None => flate2::Compression::default(), Some(level) => flate2::Compression::new(level.clamp(0, 9) as u32), }, ); let mut writer = config.wrap(encoder); std::io::copy(&mut entry, &mut writer).or_raise(|| message("Could not write zip entry data"))?; let (encoder, descriptor) = writer .finish() .or_raise(|| message("Could not finish zip entry writer"))?; encoder .finish() .or_raise(|| message("Could not finish deflate encoder"))?; zip_entry .finish(descriptor) .or_raise(|| message("Could not finish zip entry"))?; } gix_object::tree::EntryKind::Tree | gix_object::tree::EntryKind::Commit => { // rawzip requires directory paths to end with '/' let mut dir_path = path.to_owned(); if !dir_path.ends_with('/') { dir_path.push('/'); } ar.new_dir(&dir_path) .last_modified(mtime) .unix_permissions(unix_permissions) .create() .or_raise(|| message("Could not create zip directory entry"))?; } gix_object::tree::EntryKind::Link => { buf.clear(); std::io::copy(&mut entry, buf).or_raise(|| message("Could not read symlink target"))?; // For symlinks, we need to create a file with symlink permissions let symlink_path = path; let target = buf.as_bstr().to_str().or_raise(|| { message!( "Invalid UTF-8 in symlink target for entry '{symlink_path}': {:?}", buf.as_bstr() ) })?; let (mut zip_entry, config) = ar .new_file(symlink_path) .compression_method(rawzip::CompressionMethod::Store) .last_modified(mtime) .unix_permissions(0o120644) // Symlink mode .start() .or_raise(|| message("Could not start zip symlink entry"))?; let mut writer = config.wrap(&mut zip_entry); writer .write_all(target.as_bytes()) .or_raise(|| message("Could not write symlink target"))?; let (_, descriptor) = writer .finish() .or_raise(|| message("Could not finish zip symlink writer"))?; zip_entry .finish(descriptor) .or_raise(|| message("Could not finish zip symlink entry"))?; } } Ok(())} #[cfg(any(feature = "tar", feature = "tar_gz"))]fn append_tar_entry<W: std::io::Write>( ar: &mut tar::Builder<W>, buf: &mut Vec<u8>, mut entry: gix_worktree_stream::Entry<'_>, mtime_seconds_since_epoch: i64, opts: &Options,) -> Result<(), Error> { let mut header = tar::Header::new_gnu(); header.set_mtime(mtime_seconds_since_epoch as u64); header.set_entry_type(tar_entry_type(entry.mode)); header.set_mode(if entry.mode.is_executable() { 0o755 } else { 0o644 }); buf.clear(); std::io::copy(&mut entry, buf).or_raise(|| message("Could not read entry data"))?; let path = gix_path::from_bstr(add_prefix(entry.relative_path(), opts.tree_prefix.as_ref())); header.set_size(buf.len() as u64); if entry.mode.is_link() { use bstr::ByteSlice; let target = gix_path::from_bstr(buf.as_bstr()); header.set_entry_type(tar::EntryType::Symlink); header.set_size(0); ar.append_link(&mut header, path, target) .or_raise(|| message("Could not append symlink to tar archive"))?; } else { ar.append_data(&mut header, path, buf.as_slice()) .or_raise(|| message("Could not append data to tar archive"))?; } Ok(())} #[cfg(any(feature = "tar", feature = "tar_gz"))]fn tar_entry_type(mode: gix_object::tree::EntryMode) -> tar::EntryType { use gix_object::tree::EntryKind; use tar::EntryType; match mode.kind() { EntryKind::Tree | EntryKind::Commit => EntryType::Directory, EntryKind::Blob => EntryType::Regular, EntryKind::BlobExecutable => EntryType::Regular, EntryKind::Link => EntryType::Link, }} #[cfg(any(feature = "tar", feature = "tar_gz", feature = "zip"))]fn add_prefix<'a>(relative_path: &'a bstr::BStr, prefix: Option<&bstr::BString>) -> std::borrow::Cow<'a, bstr::BStr> { use std::borrow::Cow; match prefix { None => Cow::Borrowed(relative_path), Some(prefix) => { use bstr::ByteVec; let mut buf = prefix.clone(); buf.push_str(relative_path); Cow::Owned(buf) } }}
#[cfg(any(feature = "tar", feature = "tar_gz", feature = "zip"))]
use gix_error::ResultExt;
use gix_error::{ErrorExt, message};
use gix_worktree_stream::{Entry, Stream};
use crate::{Error, Format, Options};
#[cfg(feature = "zip")]
use std::io::Write;
/// Write all stream entries in `stream` as provided by `next_entry(stream)` to `out` configured according to `opts` which
/// also includes the streaming format.
///
/// ### Performance
///
/// * The caller should be sure `out` is fast enough. If in doubt, wrap in [`std::io::BufWriter`].
/// * Further, big files aren't suitable for archival into `tar` archives as they require the size of the stream to be known
/// prior to writing the header of each entry.
#[cfg_attr(not(feature = "tar"), allow(unused_mut, unused_variables))]
pub fn write_stream<NextFn>(
stream: &mut Stream,
mut next_entry: NextFn,
out: impl std::io::Write,
opts: Options,
) -> Result<(), Error>
where
NextFn: FnMut(&mut Stream) -> Result<Option<Entry<'_>>, gix_error::Exn>,
{
if opts.format == Format::InternalTransientNonPersistable {
return Err(message("The internal format cannot be used as an archive, it's merely a debugging tool").raise());
}
#[cfg(any(feature = "tar", feature = "tar_gz"))]
{
enum State<W: std::io::Write> {
#[cfg(feature = "tar")]
Tar((tar::Builder<W>, Vec<u8>)),
#[cfg(feature = "tar_gz")]
TarGz((tar::Builder<flate2::write::GzEncoder<W>>, Vec<u8>)),
}
impl<W: std::io::Write> State<W> {
pub fn new(format: Format, mtime: gix_date::SecondsSinceUnixEpoch, out: W) -> Result<Self, Error> {
match format {
Format::InternalTransientNonPersistable => unreachable!("handled earlier"),
Format::Zip { .. } => {
Err(message("Cannot create a zip archive if output stream does not support seek").raise())
}
Format::Tar => {
#[cfg(feature = "tar")]
{
Ok(State::Tar((
{
let mut ar = tar::Builder::new(out);
ar.mode(tar::HeaderMode::Deterministic);
ar
},
Vec::with_capacity(64 * 1024),
)))
}
#[cfg(not(feature = "tar"))]
{
Err(message!("Support for the format '{:?}' was not compiled in", Format::Tar).raise())
}
}
Format::TarGz { compression_level } => {
#[cfg(feature = "tar_gz")]
{
Ok(State::TarGz((
{
let gz = flate2::GzBuilder::new().mtime(mtime as u32).write(
out,
match compression_level {
None => flate2::Compression::default(),
Some(level) => flate2::Compression::new(u32::from(level)),
},
);
let mut ar = tar::Builder::new(gz);
ar.mode(tar::HeaderMode::Deterministic);
ar
},
Vec::with_capacity(64 * 1024),
)))
}
#[cfg(not(feature = "tar_gz"))]
{
Err(message!(
"Support for the format '{:?}' was not compiled in",
Format::TarGz {
compression_level: None,
}
)
.raise())
}
}
}
}
}
let mut state = State::new(opts.format, opts.modification_time, out)?;
while let Some(entry) =
next_entry(stream).or_raise(|| message("Could not retrieve the next entry from the stream"))?
{
match &mut state {
#[cfg(feature = "tar")]
State::Tar((ar, buf)) => {
append_tar_entry(ar, buf, entry, opts.modification_time, &opts)?;
}
#[cfg(feature = "tar_gz")]
State::TarGz((ar, buf)) => {
append_tar_entry(ar, buf, entry, opts.modification_time, &opts)?;
}
}
}
match state {
#[cfg(feature = "tar")]
State::Tar((mut ar, _)) => {
ar.finish().or_raise(|| message("Could not finish tar archive"))?;
}
#[cfg(feature = "tar_gz")]
State::TarGz((ar, _)) => {
ar.into_inner()
.or_raise(|| message("Could not finish tar.gz archive"))?
.finish()
.or_raise(|| message("Could not finish gzip stream"))?;
}
}
}
#[cfg(not(any(feature = "tar", feature = "tar_gz")))]
{
let _ = (next_entry, out);
return Err(message!("Support for the format '{:?}' was not compiled in", opts.format).raise());
}
#[allow(
unreachable_code,
reason = "Reachability depends on the enabled archive format features."
)]
Ok(())
}
/// Like [`write_stream()`], but requires [`std::io::Seek`] for `out`.
///
/// Note that `zip` is able to stream big files, which our `tar` implementation is not able to do, which makes it the
/// only suitable container to support huge files from `git-lfs` without consuming excessive amounts of memory.
#[cfg_attr(not(feature = "zip"), allow(unused_mut, unused_variables))]
pub fn write_stream_seek<NextFn>(
stream: &mut Stream,
mut next_entry: NextFn,
out: impl std::io::Write + std::io::Seek,
opts: Options,
) -> Result<(), Error>
where
NextFn: FnMut(&mut Stream) -> Result<Option<Entry<'_>>, gix_error::Exn>,
{
let compression_level = match opts.format {
Format::Zip { compression_level } => compression_level.map(i64::from),
_other => return write_stream(stream, next_entry, out, opts),
};
#[cfg(feature = "zip")]
{
let mut ar = rawzip::ZipArchiveWriter::new(out);
let mut buf = Vec::new();
let mtime = rawzip::time::UtcDateTime::from_unix(opts.modification_time);
while let Some(entry) =
next_entry(stream).or_raise(|| message("Could not retrieve the next entry from the stream"))?
{
append_zip_entry(
&mut ar,
entry,
&mut buf,
mtime,
compression_level,
opts.tree_prefix.as_ref(),
)?;
}
ar.finish().or_raise(|| message("Could not finish zip archive"))?;
}
#[cfg(not(feature = "zip"))]
{
let _ = compression_level;
#[expect(
clippy::needless_return,
reason = "the explicit return keeps feature-dependent branches structurally consistent"
)]
return Err(message!(
"Support for the format '{:?}' was not compiled in",
Format::Zip {
compression_level: None
}
)
.raise());
}
#[cfg(feature = "zip")]
Ok(())
}
#[cfg(feature = "zip")]
fn append_zip_entry<W: std::io::Write + std::io::Seek>(
ar: &mut rawzip::ZipArchiveWriter<W>,
mut entry: gix_worktree_stream::Entry<'_>,
buf: &mut Vec<u8>,
mtime: rawzip::time::UtcDateTime,
compression_level: Option<i64>,
tree_prefix: Option<&bstr::BString>,
) -> Result<(), Error> {
use bstr::ByteSlice;
let path = add_prefix(entry.relative_path(), tree_prefix).into_owned();
let unix_permissions = if entry.mode.is_executable() { 0o755 } else { 0o644 };
let path = path
.to_str()
.or_raise(|| message!("Invalid UTF-8 in entry path: {path:?}"))?;
match entry.mode.kind() {
gix_object::tree::EntryKind::Blob | gix_object::tree::EntryKind::BlobExecutable => {
let file_builder = ar
.new_file(path)
.compression_method(rawzip::CompressionMethod::Deflate)
.last_modified(mtime)
.unix_permissions(unix_permissions);
let (mut zip_entry, config) = file_builder
.start()
.or_raise(|| message("Could not start zip file entry"))?;
// Use flate2 for compression. Level 9 is the maximum compression level for deflate.
let encoder = flate2::write::DeflateEncoder::new(
&mut zip_entry,
match compression_level {
None => flate2::Compression::default(),
Some(level) => flate2::Compression::new(level.clamp(0, 9) as u32),
},
);
let mut writer = config.wrap(encoder);
std::io::copy(&mut entry, &mut writer).or_raise(|| message("Could not write zip entry data"))?;
let (encoder, descriptor) = writer
.finish()
.or_raise(|| message("Could not finish zip entry writer"))?;
encoder
.finish()
.or_raise(|| message("Could not finish deflate encoder"))?;
zip_entry
.finish(descriptor)
.or_raise(|| message("Could not finish zip entry"))?;
}
gix_object::tree::EntryKind::Tree | gix_object::tree::EntryKind::Commit => {
// rawzip requires directory paths to end with '/'
let mut dir_path = path.to_owned();
if !dir_path.ends_with('/') {
dir_path.push('/');
}
ar.new_dir(&dir_path)
.last_modified(mtime)
.unix_permissions(unix_permissions)
.create()
.or_raise(|| message("Could not create zip directory entry"))?;
}
gix_object::tree::EntryKind::Link => {
buf.clear();
std::io::copy(&mut entry, buf).or_raise(|| message("Could not read symlink target"))?;
// For symlinks, we need to create a file with symlink permissions
let symlink_path = path;
let target = buf.as_bstr().to_str().or_raise(|| {
message!(
"Invalid UTF-8 in symlink target for entry '{symlink_path}': {:?}",
buf.as_bstr()
)
})?;
let (mut zip_entry, config) = ar
.new_file(symlink_path)
.compression_method(rawzip::CompressionMethod::Store)
.last_modified(mtime)
.unix_permissions(0o120644) // Symlink mode
.start()
.or_raise(|| message("Could not start zip symlink entry"))?;
let mut writer = config.wrap(&mut zip_entry);
writer
.write_all(target.as_bytes())
.or_raise(|| message("Could not write symlink target"))?;
let (_, descriptor) = writer
.finish()
.or_raise(|| message("Could not finish zip symlink writer"))?;
zip_entry
.finish(descriptor)
.or_raise(|| message("Could not finish zip symlink entry"))?;
}
}
Ok(())
}
#[cfg(any(feature = "tar", feature = "tar_gz"))]
fn append_tar_entry<W: std::io::Write>(
ar: &mut tar::Builder<W>,
buf: &mut Vec<u8>,
mut entry: gix_worktree_stream::Entry<'_>,
mtime_seconds_since_epoch: i64,
opts: &Options,
) -> Result<(), Error> {
let mut header = tar::Header::new_gnu();
header.set_mtime(mtime_seconds_since_epoch as u64);
header.set_entry_type(tar_entry_type(entry.mode));
header.set_mode(if entry.mode.is_executable() { 0o755 } else { 0o644 });
buf.clear();
std::io::copy(&mut entry, buf).or_raise(|| message("Could not read entry data"))?;
let path = gix_path::from_bstr(add_prefix(entry.relative_path(), opts.tree_prefix.as_ref()));
header.set_size(buf.len() as u64);
if entry.mode.is_link() {
use bstr::ByteSlice;
let target = gix_path::from_bstr(buf.as_bstr());
header.set_entry_type(tar::EntryType::Symlink);
header.set_size(0);
ar.append_link(&mut header, path, target)
.or_raise(|| message("Could not append symlink to tar archive"))?;
} else {
ar.append_data(&mut header, path, buf.as_slice())
.or_raise(|| message("Could not append data to tar archive"))?;
}
Ok(())
}
#[cfg(any(feature = "tar", feature = "tar_gz"))]
fn tar_entry_type(mode: gix_object::tree::EntryMode) -> tar::EntryType {
use gix_object::tree::EntryKind;
use tar::EntryType;
match mode.kind() {
EntryKind::Tree | EntryKind::Commit => EntryType::Directory,
EntryKind::Blob => EntryType::Regular,
EntryKind::BlobExecutable => EntryType::Regular,
EntryKind::Link => EntryType::Link,
}
}
#[cfg(any(feature = "tar", feature = "tar_gz", feature = "zip"))]
fn add_prefix<'a>(relative_path: &'a bstr::BStr, prefix: Option<&bstr::BString>) -> std::borrow::Cow<'a, bstr::BStr> {
use std::borrow::Cow;
match prefix {
None => Cow::Borrowed(relative_path),
Some(prefix) => {
use bstr::ByteVec;
let mut buf = prefix.clone();
buf.push_str(relative_path);
Cow::Owned(buf)
}
}
}