feat!: make zlib compression levels configurable

Introduce a validated Compression type and require callers to choose a level when creating deflate streams. Optional serde support allows options in dependent crates to retain their serialization APIs.

Amey Pawar committed 4 months ago

Commit 6936e851 parent 9884f486 files · +82 −26Browse files

Unified · Split

6 files

Cargo.lock +2 −0
gix-zlib/Cargo.toml +4 −0
gix-zlib/src/lib.rs +34 −0
gix-zlib/src/stream/deflate.rs +14 −17
gix-zlib/tests/zlib/stream/deflate.rs +27 −8
gix-zlib/tests/zlib/stream/inflate.rs +1 −1

Cargo.lock+2−0

  "gix-worktree",
  "gix-worktree-state",
  "gix-worktree-stream",
+ "gix-zlib",
  "insta",
  "is_ci",
  "nonempty",
 version = "0.1.0"
 dependencies = [
  "bstr",
+ "serde",
  "thiserror 2.0.18",
  "zlib-rs",
 ]

gix-zlib/Cargo.toml+4−0

 doctest = false
 test = false
+[features]
+serde = ["dep:serde"]
 [dependencies]
 zlib-rs = "0.6.2"
 thiserror = "2.0.18"
+serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
 [dev-dependencies]
 bstr = { version = "1.12.0", default-features = false }

gix-zlib/src/lib.rs+34−0

@@ -1,6 +1,40 @@
1	#![deny(missing_docs)]	1	#![deny(missing_docs)]
2	//! Streaming compression and decompression utilities used by gitoxide.	2	//! Streaming compression and decompression utilities used by gitoxide.
3		3
		4	/// The compression level to use for zlib-based streams, in the range from 0 (no compression)
		5	/// to 9 (best compression, slowest).
		6	///
		7	/// Note that `git` maps its configured level of `-1` to the zlib default, which is level 6
		8	/// and available as [`Compression::DEFAULT`].
		9	#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
		10	#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
		11	pub struct Compression(i32);
		12
		13	impl Compression {
		14	/// Do not compress at all, while still producing a valid zlib stream.
		15	pub const NONE: Compression = Compression(0);
		16	/// The fastest compression, with the lowest compression ratio, also known as level 1.
		17	///
		18	/// This is what `git` uses for loose objects unless configured otherwise with `core.looseCompression`.
		19	pub const BEST_SPEED: Compression = Compression(1);
		20	/// The default compromise between speed and compression ratio, also known as level 6.
		21	///
		22	/// This is what `git` uses when writing packs unless configured otherwise with `pack.compression`.
		23	pub const DEFAULT: Compression = Compression(6);
		24	/// The best compression ratio at the expense of speed, also known as level 9.
		25	pub const BEST: Compression = Compression(9);
		26
		27	/// Create a new instance from `level` if it is within the valid range from 0 to 9, inclusive.
		28	pub fn new(level: i32) -> Option<Self> {
		29	(0..=9).contains(&level).then_some(Compression(level))
		30	}
		31
		32	/// Return the compression level as integer in the range from 0 to 9, inclusive.
		33	pub fn level(&self) -> i32 {
		34	self.0
		35	}
		36	}
		37
4	/// A type to hold all state needed for decompressing a ZLIB encoded stream.	38	/// A type to hold all state needed for decompressing a ZLIB encoded stream.
5	pub struct Decompress(zlib_rs::Inflate);	39	pub struct Decompress(zlib_rs::Inflate);
6		40

gix-zlib/src/stream/deflate.rs+14−17

 //! Compression state and a [`std::io::Write`] adapter for producing zlib streams.
-use crate::Status;
+use crate::{Compression, Status};
 use zlib_rs::DeflateError;
 const BUF_SIZE: usize = 4096 * 8;
 /// Be sure to call `flush()` when done to finalize the deflate stream.
 pub struct Write<W> {
     compressor: Compress,
+    compression: Compression,
     inner: W,
     buf: [u8; BUF_SIZE],
 }
 {
     fn clone(&self) -> Self {
         Write {
-            compressor: impls::new_compress(),
+            compressor: impls::new_compress(self.compression),
+            compression: self.compression,
             inner: self.inner.clone(),
             buf: self.buf,
         }
 /// Hold all state needed for compressing data.
 pub struct Compress(zlib_rs::Deflate);
-impl Default for Compress {
-    fn default() -> Self {
-        Self::new()
-    }
-}
 impl Compress {
     /// The number of bytes that were read from the input.
     pub fn total_in(&self) -> u64 {
         self.0.total_out()
     }
-    /// Create a new instance - this allocates so should be done with care.
+    /// Create a new instance compressing with `level` - this allocates so should be done with care.
-    pub fn new() -> Self {
+    pub fn new(level: Compression) -> Self {
-        let config = zlib_rs::DeflateConfig::best_speed();
+        let config = zlib_rs::DeflateConfig::new(level.level());
         let header = true;
         let inner = zlib_rs::Deflate::new(config.level, header, config.window_bits as u8);
         Self(inner)
 mod impls {
     use std::io;
-    use crate::Status;
     use crate::stream::deflate::{self, Compress, FlushCompress};
+    use crate::{Compression, Status};
-    pub(crate) fn new_compress() -> Compress {
+    pub(crate) fn new_compress(level: Compression) -> Compress {
-        Compress::new()
+        Compress::new(level)
     }
     impl<W> deflate::Write<W>
     where
         W: io::Write,
     {
-        /// Create a new instance writing compressed bytes to `inner`.
+        /// Create a new instance writing bytes compressed with `level` to `inner`.
-        pub fn new(inner: W) -> deflate::Write<W> {
+        pub fn new(inner: W, level: Compression) -> deflate::Write<W> {
             deflate::Write {
-                compressor: new_compress(),
+                compressor: new_compress(level),
+                compression: level,
                 inner,
                 buf: [0; deflate::BUF_SIZE],
             }

gix-zlib/tests/zlib/stream/deflate.rs+27−8

 use bstr::ByteSlice;
-use gix_zlib::Decompress;
 use gix_zlib::stream::deflate::{self, Compress, FlushCompress};
+use gix_zlib::{Compression, Decompress};
 pub(crate) fn compressed(data: &[u8]) -> Vec<u8> {
-    let mut writer = deflate::Write::new(Vec::new());
+    let mut writer = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
     writer.write_all(data).expect("in-memory writes never fail");
     writer.flush().expect("in-memory flushes never fail");
     writer.into_inner()
 #[test]
 fn all_at_once() -> Result<(), Box<dyn std::error::Error>> {
-    let mut w = deflate::Write::new(Vec::new());
+    let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
     assert_eq!(w.write(b"hello")?, 5);
     w.flush()?;
     Ok(())
 }
+#[test]
+fn higher_levels_compress_better() -> Result<(), Box<dyn std::error::Error>> {
+    let data: Vec<u8> = (0..128 * 1024).map(|i| (i % 100) as u8).collect();
+    let mut sizes = Vec::new();
+    for level in [Compression::NONE, Compression::BEST_SPEED, Compression::DEFAULT] {
+        let mut writer = deflate::Write::new(Vec::new(), level);
+        writer.write_all(&data)?;
+        writer.flush()?;
+        let compressed = writer.into_inner();
+        assert_deflate_buffer(compressed.clone(), &data)?;
+        sizes.push(compressed.len());
+    }
+    assert!(
+        sizes[0] > sizes[1] && sizes[1] > sizes[2],
+        "each level compresses better than the one before it: {sizes:?}"
+    );
+    Ok(())
+}
 #[test]
 fn big_file_small_writes() -> Result<(), Box<dyn std::error::Error>> {
-    let mut w = deflate::Write::new(Vec::new());
+    let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
     let bytes = include_bytes!(
         "../../../../gix-odb/tests/fixtures/objects/pack/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack"
     );
 #[test]
 fn big_file_a_few_big_writes() -> Result<(), Box<dyn std::error::Error>> {
-    let mut w = deflate::Write::new(Vec::new());
+    let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
     let bytes = include_bytes!(
         "../../../../gix-odb/tests/fixtures/objects/pack/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack"
     );
         FlushCompress::Sync,
         FlushCompress::Full,
     ] {
-        let mut compressor = Compress::default();
+        let mut compressor = Compress::new(Compression::BEST_SPEED);
         assert_eq!(compressor.total_in(), 0);
         assert_eq!(compressor.total_out(), 0);
 #[test]
 fn writer_clone_and_reset() -> Result<(), Box<dyn std::error::Error>> {
-    let original = deflate::Write::new(Vec::new());
+    let original = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
     let mut cloned = original.clone();
     cloned.write_all(b"clone owns a fresh compressor")?;
     cloned.flush()?;
     assert_deflate_buffer(cloned.into_inner(), b"clone owns a fresh compressor")?;
     assert!(original.into_inner().is_empty());
-    let mut writer = deflate::Write::new(Vec::new());
+    let mut writer = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
     writer.write_all(b"first")?;
     writer.flush()?;
     writer.reset();

gix-zlib/tests/zlib/stream/inflate.rs+1−1

@@ -21,7 +21,7 @@ fn errors_keep_the_underlying_cause() { · 20 unmodified lines
21	}	21	}
22		22
23	fn deflated(data: &[u8]) -> Vec<u8> {	23	fn deflated(data: &[u8]) -> Vec<u8> {
24	let mut out = deflate::Write::new(Vec::new());	24	let mut out = deflate::Write::new(Vec::new(), gix_zlib::Compression::BEST_SPEED);
25	out.write_all(data).expect("in-memory writes never fail");	25	out.write_all(data).expect("in-memory writes never fail");
26	out.flush().expect("in-memory flushes never fail");	26	out.flush().expect("in-memory flushes never fail");
27	out.into_inner()	27	out.into_inner()