History

feat!: add `hash::bytes_with_header()`, also make it 32bit compatible.

That way it's possible to hash entire files as object. Previously it wasn't possible to read more than u32::MAX bytes even on 32 bit system even though we are streaming the data.

Sebastian Thiel committed 3 years ago

Commit 4c03fdb1 parent 7891fb11 file · +21 −7Browse files

Unified · Split

gix-features/src/hash.rs+21−7
@@ -96,7 +96,7 @@ pub fn hasher(kind: gix_hash::Kind) -> Sha1 { · 95 unmodified lines
96#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]96#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]
97pub fn bytes_of_file(97pub fn bytes_of_file(
98 path: &std::path::Path,98 path: &std::path::Path,
99 num_bytes_from_start: usize,99 num_bytes_from_start: u64,
100 kind: gix_hash::Kind,100 kind: gix_hash::Kind,
101 progress: &mut dyn crate::progress::Progress,101 progress: &mut dyn crate::progress::Progress,
102 should_interrupt: &std::sync::atomic::AtomicBool,102 should_interrupt: &std::sync::atomic::AtomicBool,
@@ -110,28 +110,42 @@ pub fn bytes_of_file( · 7 unmodified lines
110 )110 )
111}111}
112 112
113/// Similar to [`bytes_of_file`], but operates on an already open file.113/// Similar to [`bytes_of_file`], but operates on a stream of bytes.
114#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]114#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]
115pub fn bytes(115pub fn bytes(
116 read: &mut dyn std::io::Read,116 read: &mut dyn std::io::Read,
117 num_bytes_from_start: usize,117 num_bytes_from_start: u64,
118 kind: gix_hash::Kind,118 kind: gix_hash::Kind,
119 progress: &mut dyn crate::progress::Progress,119 progress: &mut dyn crate::progress::Progress,
120 should_interrupt: &std::sync::atomic::AtomicBool,120 should_interrupt: &std::sync::atomic::AtomicBool,
121) -> std::io::Result<gix_hash::ObjectId> {121) -> std::io::Result<gix_hash::ObjectId> {
122 let mut hasher = hasher(kind);122 bytes_with_hasher(read, num_bytes_from_start, hasher(kind), progress, should_interrupt)
123}
124
125/// Similar to [`bytes()`], but takes a `hasher` instead of a hash kind.
126#[cfg(all(feature = "progress", any(feature = "rustsha1", feature = "fast-sha1")))]
127pub fn bytes_with_hasher(
128 read: &mut dyn std::io::Read,
129 num_bytes_from_start: u64,
130 mut hasher: Sha1,
131 progress: &mut dyn crate::progress::Progress,
132 should_interrupt: &std::sync::atomic::AtomicBool,
133) -> std::io::Result<gix_hash::ObjectId> {
123 let start = std::time::Instant::now();134 let start = std::time::Instant::now();
124 // init progress before the possibility for failure, as convenience in case people want to recover135 // init progress before the possibility for failure, as convenience in case people want to recover
125 progress.init(Some(num_bytes_from_start), crate::progress::bytes());136 progress.init(
137 Some(num_bytes_from_start as prodash::progress::Step),
138 crate::progress::bytes(),
139 );
126 140
127 const BUF_SIZE: usize = u16::MAX as usize;141 const BUF_SIZE: usize = u16::MAX as usize;
128 let mut buf = [0u8; BUF_SIZE];142 let mut buf = [0u8; BUF_SIZE];
129 let mut bytes_left = num_bytes_from_start;143 let mut bytes_left = num_bytes_from_start;
130 144
131 while bytes_left > 0 {145 while bytes_left > 0 {
132 let out = &mut buf[..BUF_SIZE.min(bytes_left)];146 let out = &mut buf[..BUF_SIZE.min(bytes_left as usize)];
133 read.read_exact(out)?;147 read.read_exact(out)?;
134 bytes_left -= out.len();148 bytes_left -= out.len() as u64;
135 progress.inc_by(out.len());149 progress.inc_by(out.len());
136 hasher.update(out);150 hasher.update(out);
137 if should_interrupt.load(std::sync::atomic::Ordering::SeqCst) {151 if should_interrupt.load(std::sync::atomic::Ordering::SeqCst) {