feat!: make zlib compression levels configurable

Introduce a validated Compression type and require callers to choose a level when creating deflate streams. Optional serde support allows options in dependent crates to retain their serialization APIs.

Amey Pawar committed 4 months ago

Commit 6936e851 parent 9884f486 files · +82 −26Browse files

Unified · Split

6 files

Cargo.lock +2 −0
gix-zlib/Cargo.toml +4 −0
gix-zlib/src/lib.rs +34 −0
gix-zlib/src/stream/deflate.rs +14 −17
gix-zlib/tests/zlib/stream/deflate.rs +27 −8
gix-zlib/tests/zlib/stream/inflate.rs +1 −1

Cargo.lock+2−0

@@ -1572,6 +1572,7 @@ dependencies = [ · 1571 unmodified lines
1572	1572	"gix-worktree",
1573	1573	"gix-worktree-state",
1574	1574	"gix-worktree-stream",
	1575	+ "gix-zlib",
1575	1576	"insta",
1576	1577	"is_ci",
1577	1578	"nonempty",
@@ -2675,6 +2676,7 @@ name = "gix-zlib" · 1097 unmodified lines
2675	2676	version = "0.1.0"
2676	2677	dependencies = [
2677	2678	"bstr",
	2679	+ "serde",
2678	2680	"thiserror 2.0.18",
2679	2681	"zlib-rs",
2680	2682	]

gix-zlib/Cargo.toml+4−0

@@ -15,9 +15,13 @@ include = ["/src/*/", "/LICENSE-*"] · 14 unmodified lines
15	15	doctest = false
16	16	test = false
17	17
	18	+[features]
	19	+serde = ["dep:serde"]
	20	+
18	21	[dependencies]
19	22	zlib-rs = "0.6.2"
20	23	thiserror = "2.0.18"
	24	+serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
21	25
22	26	[dev-dependencies]
23	27	bstr = { version = "1.12.0", default-features = false }

gix-zlib/src/lib.rs+34−0

@@ -1,6 +1,40 @@
1	1	#![deny(missing_docs)]
2	2	//! Streaming compression and decompression utilities used by gitoxide.
3	3
	4	+/// The compression level to use for zlib-based streams, in the range from 0 (no compression)
	5	+/// to 9 (best compression, slowest).
	6	+///
	7	+/// Note that `git` maps its configured level of `-1` to the zlib default, which is level 6
	8	+/// and available as [`Compression::DEFAULT`].
	9	+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
	10	+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
	11	+pub struct Compression(i32);
	12	+
	13	+impl Compression {
	14	+ /// Do not compress at all, while still producing a valid zlib stream.
	15	+ pub const NONE: Compression = Compression(0);
	16	+ /// The fastest compression, with the lowest compression ratio, also known as level 1.
	17	+ ///
	18	+ /// This is what `git` uses for loose objects unless configured otherwise with `core.looseCompression`.
	19	+ pub const BEST_SPEED: Compression = Compression(1);
	20	+ /// The default compromise between speed and compression ratio, also known as level 6.
	21	+ ///
	22	+ /// This is what `git` uses when writing packs unless configured otherwise with `pack.compression`.
	23	+ pub const DEFAULT: Compression = Compression(6);
	24	+ /// The best compression ratio at the expense of speed, also known as level 9.
	25	+ pub const BEST: Compression = Compression(9);
	26	+
	27	+ /// Create a new instance from `level` if it is within the valid range from 0 to 9, inclusive.
	28	+ pub fn new(level: i32) -> Option<Self> {
	29	+ (0..=9).contains(&level).then_some(Compression(level))
	30	+ }
	31	+
	32	+ /// Return the compression level as integer in the range from 0 to 9, inclusive.
	33	+ pub fn level(&self) -> i32 {
	34	+ self.0
	35	+ }
	36	+}
	37	+
4	38	/// A type to hold all state needed for decompressing a ZLIB encoded stream.
5	39	pub struct Decompress(zlib_rs::Inflate);
6	40

gix-zlib/src/stream/deflate.rs+14−17

@@ -1,6 +1,6 @@
1	1	//! Compression state and a [`std::io::Write`] adapter for producing zlib streams.
2	2
3		−use crate::Status;
	3	+use crate::{Compression, Status};
4	4	use zlib_rs::DeflateError;
5	5
6	6	const BUF_SIZE: usize = 4096 * 8;
@@ -10,6 +10,7 @@ const BUF_SIZE: usize = 4096 * 8; · 3 unmodified lines
10	10	/// Be sure to call `flush()` when done to finalize the deflate stream.
11	11	pub struct Write<W> {
12	12	compressor: Compress,
	13	+ compression: Compression,
13	14	inner: W,
14	15	buf: [u8; BUF_SIZE],
15	16	}
@@ -20,7 +21,8 @@ where · 4 unmodified lines
20	21	{
21	22	fn clone(&self) -> Self {
22	23	Write {
23		− compressor: impls::new_compress(),
	24	+ compressor: impls::new_compress(self.compression),
	25	+ compression: self.compression,
24	26	inner: self.inner.clone(),
25	27	buf: self.buf,
26	28	}
@@ -30,12 +32,6 @@ where · 3 unmodified lines
30	32	/// Hold all state needed for compressing data.
31	33	pub struct Compress(zlib_rs::Deflate);
32	34
33		−impl Default for Compress {
34		− fn default() -> Self {
35		− Self::new()
36		− }
37		−}
38		−
39	35	impl Compress {
40	36	/// The number of bytes that were read from the input.
41	37	pub fn total_in(&self) -> u64 {
@@ -47,9 +43,9 @@ impl Compress { · 5 unmodified lines
47	43	self.0.total_out()
48	44	}
49	45
50		− /// Create a new instance - this allocates so should be done with care.
51		− pub fn new() -> Self {
52		− let config = zlib_rs::DeflateConfig::best_speed();
	46	+ /// Create a new instance compressing with `level` - this allocates so should be done with care.
	47	+ pub fn new(level: Compression) -> Self {
	48	+ let config = zlib_rs::DeflateConfig::new(level.level());
53	49	let header = true;
54	50	let inner = zlib_rs::Deflate::new(config.level, header, config.window_bits as u8);
55	51	Self(inner)
@@ -148,21 +144,22 @@ pub enum FlushCompress { · 92 unmodified lines
148	144	mod impls {
149	145	use std::io;
150	146
151		− use crate::Status;
152	147	use crate::stream::deflate::{self, Compress, FlushCompress};
	148	+ use crate::{Compression, Status};
153	149
154		− pub(crate) fn new_compress() -> Compress {
155		− Compress::new()
	150	+ pub(crate) fn new_compress(level: Compression) -> Compress {
	151	+ Compress::new(level)
156	152	}
157	153
158	154	impl<W> deflate::Write<W>
159	155	where
160	156	W: io::Write,
161	157	{
162		− /// Create a new instance writing compressed bytes to `inner`.
163		− pub fn new(inner: W) -> deflate::Write<W> {
	158	+ /// Create a new instance writing bytes compressed with `level` to `inner`.
	159	+ pub fn new(inner: W, level: Compression) -> deflate::Write<W> {
164	160	deflate::Write {
165		− compressor: new_compress(),
	161	+ compressor: new_compress(level),
	162	+ compression: level,
166	163	inner,
167	164	buf: [0; deflate::BUF_SIZE],
168	165	}

gix-zlib/tests/zlib/stream/deflate.rs+27−8

@@ -5,11 +5,11 @@ use std::{ · 4 unmodified lines
5	5
6	6	use bstr::ByteSlice;
7	7
8		−use gix_zlib::Decompress;
9	8	use gix_zlib::stream::deflate::{self, Compress, FlushCompress};
	9	+use gix_zlib::{Compression, Decompress};
10	10
11	11	pub(crate) fn compressed(data: &[u8]) -> Vec<u8> {
12		− let mut writer = deflate::Write::new(Vec::new());
	12	+ let mut writer = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
13	13	writer.write_all(data).expect("in-memory writes never fail");
14	14	writer.flush().expect("in-memory flushes never fail");
15	15	writer.into_inner()
@@ -63,7 +63,7 @@ fn small_file_decompress() -> Result<(), Box<dyn std::error::Error>> { · 47 unmodified lines
63	63
64	64	#[test]
65	65	fn all_at_once() -> Result<(), Box<dyn std::error::Error>> {
66		− let mut w = deflate::Write::new(Vec::new());
	66	+ let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
67	67	assert_eq!(w.write(b"hello")?, 5);
68	68	w.flush()?;
69	69
@@ -80,9 +80,28 @@ fn assert_deflate_buffer(out: Vec<u8>, expected: &[u8]) -> Result<(), Box<dyn st · 10 unmodified lines
80	80	Ok(())
81	81	}
82	82
	83	+#[test]
	84	+fn higher_levels_compress_better() -> Result<(), Box<dyn std::error::Error>> {
	85	+ let data: Vec<u8> = (0..128 * 1024).map(\|i\| (i % 100) as u8).collect();
	86	+ let mut sizes = Vec::new();
	87	+ for level in [Compression::NONE, Compression::BEST_SPEED, Compression::DEFAULT] {
	88	+ let mut writer = deflate::Write::new(Vec::new(), level);
	89	+ writer.write_all(&data)?;
	90	+ writer.flush()?;
	91	+ let compressed = writer.into_inner();
	92	+ assert_deflate_buffer(compressed.clone(), &data)?;
	93	+ sizes.push(compressed.len());
	94	+ }
	95	+ assert!(
	96	+ sizes[0] > sizes[1] && sizes[1] > sizes[2],
	97	+ "each level compresses better than the one before it: {sizes:?}"
	98	+ );
	99	+ Ok(())
	100	+}
	101	+
83	102	#[test]
84	103	fn big_file_small_writes() -> Result<(), Box<dyn std::error::Error>> {
85		− let mut w = deflate::Write::new(Vec::new());
	104	+ let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
86	105	let bytes = include_bytes!(
87	106	"../../../../gix-odb/tests/fixtures/objects/pack/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack"
88	107	);
@@ -96,7 +115,7 @@ fn big_file_small_writes() -> Result<(), Box<dyn std::error::Error>> { · 7 unmodified lines
96	115
97	116	#[test]
98	117	fn big_file_a_few_big_writes() -> Result<(), Box<dyn std::error::Error>> {
99		− let mut w = deflate::Write::new(Vec::new());
	118	+ let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
100	119	let bytes = include_bytes!(
101	120	"../../../../gix-odb/tests/fixtures/objects/pack/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack"
102	121	);
@@ -116,7 +135,7 @@ fn compressor_lifecycle_counters_and_flush_modes() -> Result<(), Box<dyn std::er · 13 unmodified lines
116	135	FlushCompress::Sync,
117	136	FlushCompress::Full,
118	137	] {
119		− let mut compressor = Compress::default();
	138	+ let mut compressor = Compress::new(Compression::BEST_SPEED);
120	139	assert_eq!(compressor.total_in(), 0);
121	140	assert_eq!(compressor.total_out(), 0);
122	141
@@ -146,14 +165,14 @@ fn compressor_lifecycle_counters_and_flush_modes() -> Result<(), Box<dyn std::er · 23 unmodified lines
146	165
147	166	#[test]
148	167	fn writer_clone_and_reset() -> Result<(), Box<dyn std::error::Error>> {
149		− let original = deflate::Write::new(Vec::new());
	168	+ let original = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
150	169	let mut cloned = original.clone();
151	170	cloned.write_all(b"clone owns a fresh compressor")?;
152	171	cloned.flush()?;
153	172	assert_deflate_buffer(cloned.into_inner(), b"clone owns a fresh compressor")?;
154	173	assert!(original.into_inner().is_empty());
155	174
156		− let mut writer = deflate::Write::new(Vec::new());
	175	+ let mut writer = deflate::Write::new(Vec::new(), Compression::BEST_SPEED);
157	176	writer.write_all(b"first")?;
158	177	writer.flush()?;
159	178	writer.reset();

gix-zlib/tests/zlib/stream/inflate.rs+1−1

@@ -21,7 +21,7 @@ fn errors_keep_the_underlying_cause() { · 20 unmodified lines
21	21	}
22	22
23	23	fn deflated(data: &[u8]) -> Vec<u8> {
24		− let mut out = deflate::Write::new(Vec::new());
	24	+ let mut out = deflate::Write::new(Vec::new(), gix_zlib::Compression::BEST_SPEED);
25	25	out.write_all(data).expect("in-memory writes never fail");
26	26	out.flush().expect("in-memory flushes never fail");
27	27	out.into_inner()