feat!: add `hash::bytes_with_header()`, also make it 32bit compatible.

That way it's possible to hash entire files as object. Previously it wasn't possible to read more than u32::MAX bytes even on 32 bit system even though we are streaming the data.

Sebastian Thiel committed 3 years ago

Commit 4c03fdb1 parent 7891fb11 file · +21 −7Browse files

Unified · Split

gix-features/src/hash.rs+21−7

@@ -96,7 +96,7 @@ pub fn hasher(kind: gix_hash::Kind) -> Sha1 { · 95 unmodified lines
96	96	#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]
97	97	pub fn bytes_of_file(
98	98	path: &std::path::Path,
99		− num_bytes_from_start: usize,
	99	+ num_bytes_from_start: u64,
100	100	kind: gix_hash::Kind,
101	101	progress: &mut dyn crate::progress::Progress,
102	102	should_interrupt: &std::sync::atomic::AtomicBool,
@@ -110,28 +110,42 @@ pub fn bytes_of_file( · 7 unmodified lines
110	110	)
111	111	}
112	112
113		−/// Similar to [`bytes_of_file`], but operates on an already open file.
	113	+/// Similar to [`bytes_of_file`], but operates on a stream of bytes.
114	114	#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]
115	115	pub fn bytes(
116	116	read: &mut dyn std::io::Read,
117		− num_bytes_from_start: usize,
	117	+ num_bytes_from_start: u64,
118	118	kind: gix_hash::Kind,
119	119	progress: &mut dyn crate::progress::Progress,
120	120	should_interrupt: &std::sync::atomic::AtomicBool,
121	121	) -> std::io::Result<gix_hash::ObjectId> {
122		− let mut hasher = hasher(kind);
	122	+ bytes_with_hasher(read, num_bytes_from_start, hasher(kind), progress, should_interrupt)
	123	+}
	124	+
	125	+/// Similar to [`bytes()`], but takes a `hasher` instead of a hash kind.
	126	+#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]
	127	+pub fn bytes_with_hasher(
	128	+ read: &mut dyn std::io::Read,
	129	+ num_bytes_from_start: u64,
	130	+ mut hasher: Sha1,
	131	+ progress: &mut dyn crate::progress::Progress,
	132	+ should_interrupt: &std::sync::atomic::AtomicBool,
	133	+) -> std::io::Result<gix_hash::ObjectId> {
123	134	let start = std::time::Instant::now();
124	135	// init progress before the possibility for failure, as convenience in case people want to recover
125		− progress.init(Some(num_bytes_from_start), crate::progress::bytes());
	136	+ progress.init(
	137	+ Some(num_bytes_from_start as prodash::progress::Step),
	138	+ crate::progress::bytes(),
	139	+ );
126	140
127	141	const BUF_SIZE: usize = u16::MAX as usize;
128	142	let mut buf = [0u8; BUF_SIZE];
129	143	let mut bytes_left = num_bytes_from_start;
130	144
131	145	while bytes_left > 0 {
132		− let out = &mut buf[..BUF_SIZE.min(bytes_left)];
	146	+ let out = &mut buf[..BUF_SIZE.min(bytes_left as usize)];
133	147	read.read_exact(out)?;
134		− bytes_left -= out.len();
	148	+ bytes_left -= out.len() as u64;
135	149	progress.inc_by(out.len());
136	150	hasher.update(out);
137	151	if should_interrupt.load(std::sync::atomic::Ordering::SeqCst) {