History

feat: Add support for hashes of different size (#279)

Even though right now, there is only Sha1, in future it's easy to support other hash sizes.

Sebastian Thiel committed 4 years ago

Commit 265b8ec1 parent bf620675 files · +52 −59Browse files

Showing the changes to git-commitgraph/src (5 of 5 files). Show all

Unified · Split

5 files
  1. git-commitgraph/src/file/access.rs +12 −13
  2. git-commitgraph/src/file/commit.rs +6 −8
  3. git-commitgraph/src/file/init.rs +25 −25
  4. git-commitgraph/src/file/mod.rs +2 −4
  5. git-commitgraph/src/file/verify.rs +7 −9
git-commitgraph/src/file/access.rs+12−13
@@ -4,9 +4,7 @@ use std::{ · 3 unmodified lines
44 path::Path,
55 };
66
7−use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
8−
9−use crate::file::{self, commit::Commit, File, COMMIT_DATA_ENTRY_SIZE};
7+use crate::file::{self, commit::Commit, File, COMMIT_DATA_ENTRY_SIZE_SANS_HASH};
108
119 /// Access
1210 impl File {
@@ -30,10 +28,10 @@ impl File { · 17 unmodified lines
3028 ///
3129 /// Note that it is always conforming to the hash used in the owning repository.
3230 pub fn hash_kind(&self) -> git_hash::Kind {
33− git_hash::Kind::Sha1
31+ git_hash::Kind::from_len_in_bytes(self.hash_len).expect("hash len validated during init")
3432 }
3533
36− /// Returns 20 bytes sha1 at the given index in our list of (sorted) sha1 hashes.
34+ /// Returns an object id at the given index in our list of (sorted) hashes.
3735 /// The position ranges from 0 to self.num_commits()
3836 // copied from git-odb/src/pack/index/ext
3937 pub fn id_at(&self, pos: file::Position) -> &git_hash::oid {
@@ -47,17 +45,17 @@ impl File { · 7 unmodified lines
4745 .0
4846 .try_into()
4947 .expect("an architecture able to hold 32 bits of integer");
50− let start = self.oid_lookup_offset + (pos * SHA1_SIZE);
51− git_hash::oid::try_from(&self.data[start..start + SHA1_SIZE]).expect("20 bytes SHA1 to be alright")
48+ let start = self.oid_lookup_offset + (pos * self.hash_len);
49+ git_hash::oid::try_from(&self.data[start..][..self.hash_len]).expect("hash bytes to be alright")
5250 }
5351
5452 /// Return an iterator over all object hashes stored in the base graph.
5553 pub fn iter_base_graph_ids(&self) -> impl Iterator<Item = &git_hash::oid> {
5654 let start = self.base_graphs_list_offset.unwrap_or(0);
57− let base_graphs_list = &self.data[start..start + (SHA1_SIZE * usize::from(self.base_graph_count))];
55+ let base_graphs_list = &self.data[start..][..self.hash_len * usize::from(self.base_graph_count)];
5856 base_graphs_list
59− .chunks(SHA1_SIZE)
60− .map(|bytes| git_hash::oid::try_from(bytes).expect("20 bytes SHA1 to be alright"))
57+ .chunks(self.hash_len)
58+ .map(|bytes| git_hash::oid::try_from(bytes).expect("hash bytes to be alright"))
6159 }
6260
6361 /// return an iterator over all commits in this file.
@@ -80,7 +78,7 @@ impl File { · 16 unmodified lines
8078
8179 // Bisect using indices
8280 // TODO: Performance of V2 could possibly be better if we would be able to do a binary search
83− // on 20 byte chunks directly, but doing so requires transmuting and that is not safe, even though
81+ // on hash-sized chunks directly, but doing so requires transmuting and that is not safe, even though
8482 // it should not be if the bytes match up and the type has no destructor.
8583 while lower_bound < upper_bound {
8684 let mid = (lower_bound + upper_bound) / 2;
@@ -123,8 +121,9 @@ impl File { · 36 unmodified lines
123121 .0
124122 .try_into()
125123 .expect("an architecture able to hold 32 bits of integer");
126− let start = self.commit_data_offset + (pos * COMMIT_DATA_ENTRY_SIZE);
127− &self.data[start..start + COMMIT_DATA_ENTRY_SIZE]
124+ let entry_size = self.hash_len + COMMIT_DATA_ENTRY_SIZE_SANS_HASH;
125+ let start = self.commit_data_offset + (pos * entry_size);
126+ &self.data[start..][..entry_size]
128127 }
129128
130129 /// Returns the byte slice for this file's entire Extra Edge List (EDGE) chunk.
git-commitgraph/src/file/commit.rs+6−8
@@ -5,13 +5,11 @@ use std::{ · 4 unmodified lines
55 slice::Chunks,
66 };
77
8−use byteorder::{BigEndian, ByteOrder};
9−use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
10−
118 use crate::{
129 file::{self, File, EXTENDED_EDGES_MASK, LAST_EXTENDED_EDGE_MASK, NO_PARENT},
1310 graph,
1411 };
12+use byteorder::{BigEndian, ByteOrder};
1513
1614 /// The error used in the [`file::commit`][self] module.
1715 #[derive(thiserror::Error, Debug)]
@@ -45,11 +43,11 @@ impl<'a> Commit<'a> { · 27 unmodified lines
4543 Commit {
4644 file,
4745 pos,
48− root_tree_id: git_hash::oid::try_from(&bytes[..SHA1_SIZE]).expect("20 bytes SHA1 to be alright"),
49− parent1: ParentEdge::from_raw(BigEndian::read_u32(&bytes[SHA1_SIZE..SHA1_SIZE + 4])),
50− parent2: ParentEdge::from_raw(BigEndian::read_u32(&bytes[SHA1_SIZE + 4..SHA1_SIZE + 8])),
51− generation: BigEndian::read_u32(&bytes[SHA1_SIZE + 8..SHA1_SIZE + 12]) >> 2,
52− commit_timestamp: BigEndian::read_u64(&bytes[SHA1_SIZE + 8..SHA1_SIZE + 16]) & 0x0003_ffff_ffff,
46+ root_tree_id: git_hash::oid::try_from(&bytes[..file.hash_len]).expect("hash bytes to be alright"),
47+ parent1: ParentEdge::from_raw(BigEndian::read_u32(&bytes[file.hash_len..][..4])),
48+ parent2: ParentEdge::from_raw(BigEndian::read_u32(&bytes[file.hash_len + 4..][..4])),
49+ generation: BigEndian::read_u32(&bytes[file.hash_len + 8..][..4]) >> 2,
50+ commit_timestamp: BigEndian::read_u64(&bytes[file.hash_len + 8..][..8]) & 0x0003_ffff_ffff,
5351 }
5452 }
5553
git-commitgraph/src/file/init.rs+25−25
@@ -6,12 +6,10 @@ use std::{ · 5 unmodified lines
66 use bstr::ByteSlice;
77 use byteorder::{BigEndian, ByteOrder};
88 use filebuffer::FileBuffer;
9−use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
109
1110 use crate::file::{
12− ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, CHUNK_LOOKUP_SIZE, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE,
13− EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, OID_LOOKUP_ENTRY_SIZE,
14− SIGNATURE,
11+ ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE_SANS_HASH,
12+ EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, SIGNATURE,
1513 };
1614
1715 /// The error used in [`File::at()`].
@@ -52,11 +50,9 @@ pub enum Error { · 34 unmodified lines
5250 InvalidChunkSize { id: ChunkId, msg: String },
5351 }
5452
55−const TRAILER_LEN: usize = SHA1_SIZE;
56−const MIN_FILE_SIZE: usize = HEADER_LEN + ((MIN_CHUNKS + 1) * CHUNK_LOOKUP_SIZE) + TRAILER_LEN;
57−
58−// Required chunks: OIDF, OIDL, CDAT
59−const MIN_CHUNKS: usize = 3;
53+const MIN_FILE_SIZE: usize = HEADER_LEN
54+ + git_chunk::file::Index::size_for_entries(3 /*OIDF, OIDL, CDAT*/)
55+ + git_hash::Kind::shortest().len_in_bytes();
6056
6157 impl File {
6258 /// Try to parse the commit graph file at `path`.
@@ -96,8 +92,8 @@ impl TryFrom<&Path> for File { · 33 unmodified lines
9692 };
9793 ofs += 1;
9894
99− match data[ofs] {
100− 1 => (),
95+ let hash_kind = match data[ofs] {
96+ 1 => git_hash::Kind::Sha1,
10197 x => {
10298 return Err(Error::UnsupportedHashVersion(x));
10399 }
@@ -117,13 +113,17 @@ impl TryFrom<&Path> for File { · 13 unmodified lines
117113 let base_graphs_list_offset = chunks
118114 .validated_usize_offset_by_id(BASE_GRAPHS_LIST_CHUNK_ID, |chunk_range| {
119115 let chunk_size = chunk_range.len();
120− if chunk_size % SHA1_SIZE != 0 {
116+ if chunk_size % hash_kind.len_in_bytes() != 0 {
121117 return Err(Error::InvalidChunkSize {
122118 id: BASE_GRAPHS_LIST_CHUNK_ID,
123− msg: format!("chunk size {} is not a multiple of {}", chunk_size, SHA1_SIZE),
119+ msg: format!(
120+ "chunk size {} is not a multiple of {}",
121+ chunk_size,
122+ hash_kind.len_in_bytes()
123+ ),
124124 });
125125 }
126− let chunk_base_graph_count: u32 = (chunk_size / SHA1_SIZE)
126+ let chunk_base_graph_count: u32 = (chunk_size / hash_kind.len_in_bytes())
127127 .try_into()
128128 .expect("base graph count to fit in 32-bits");
129129 if chunk_base_graph_count != u32::from(base_graph_count) {
@@ -141,18 +141,16 @@ impl TryFrom<&Path> for File { · 11 unmodified lines
141141 chunks.validated_usize_offset_by_id(COMMIT_DATA_CHUNK_ID, |chunk_range| {
142142 let chunk_size = chunk_range.len();
143143
144− if chunk_size % COMMIT_DATA_ENTRY_SIZE != 0 {
144+ let entry_size = hash_kind.len_in_bytes() + COMMIT_DATA_ENTRY_SIZE_SANS_HASH;
145+ if chunk_size % entry_size != 0 {
145146 return Err(Error::InvalidChunkSize {
146147 id: COMMIT_DATA_CHUNK_ID,
147− msg: format!(
148− "chunk size {} is not a multiple of {}",
149− chunk_size, COMMIT_DATA_ENTRY_SIZE
150− ),
148+ msg: format!("chunk size {} is not a multiple of {}", chunk_size, entry_size),
151149 });
152150 }
153151 Ok((
154152 chunk_range.start,
155− (chunk_size / COMMIT_DATA_ENTRY_SIZE)
153+ (chunk_size / entry_size)
156154 .try_into()
157155 .expect("number of commits in CDAT chunk to fit in 32 bits"),
158156 ))
@@ -175,18 +173,19 @@ impl TryFrom<&Path> for File { · 16 unmodified lines
175173 chunks.validated_usize_offset_by_id(OID_LOOKUP_CHUNK_ID, |chunk_range| {
176174 let chunk_size = chunk_range.len();
177175
178− if chunk_size % OID_LOOKUP_ENTRY_SIZE != 0 {
176+ if chunk_size % hash_kind.len_in_bytes() != 0 {
179177 return Err(Error::InvalidChunkSize {
180178 id: OID_LOOKUP_CHUNK_ID,
181179 msg: format!(
182180 "chunk size {} is not a multiple of {}",
183− chunk_size, OID_LOOKUP_ENTRY_SIZE
181+ chunk_size,
182+ hash_kind.len_in_bytes()
184183 ),
185184 });
186185 }
187186 Ok((
188187 chunk_range.start,
189− (chunk_size / OID_LOOKUP_ENTRY_SIZE)
188+ (chunk_size / hash_kind.len_in_bytes())
190189 .try_into()
191190 .expect("number of commits in OIDL chunk to fit in 32 bits"),
192191 ))
@@ -195,10 +194,10 @@ impl TryFrom<&Path> for File { · 2 unmodified lines
195194 let extra_edges_list_range = chunks.usize_offset_by_id(EXTENDED_EDGES_LIST_CHUNK_ID).ok();
196195
197196 let trailer = &data[chunks.highest_offset() as usize..];
198− if trailer.len() != TRAILER_LEN {
197+ if trailer.len() != hash_kind.len_in_bytes() {
199198 return Err(Error::Trailer(format!(
200199 "Expected commit-graph trailer to contain {} bytes, got {}",
201− TRAILER_LEN,
200+ hash_kind.len_in_bytes(),
202201 trailer.len()
203202 )));
204203 }
@@ -236,6 +235,7 @@ impl TryFrom<&Path> for File { · 31 unmodified lines
236235 fan,
237236 oid_lookup_offset,
238237 path: path.to_owned(),
238+ hash_len: hash_kind.len_in_bytes(),
239239 })
240240 }
241241 }
git-commitgraph/src/file/mod.rs+2−4
@@ -7,7 +7,6 @@ use std::{ · 6 unmodified lines
77 };
88
99 use filebuffer::FileBuffer;
10−use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
1110
1211 pub use self::{commit::Commit, init::Error};
1312
@@ -16,11 +15,9 @@ pub mod commit; · 2 unmodified lines
1615 mod init;
1716 pub mod verify;
1817
19−const CHUNK_LOOKUP_SIZE: usize = 12;
20−const COMMIT_DATA_ENTRY_SIZE: usize = SHA1_SIZE + 16;
18+const COMMIT_DATA_ENTRY_SIZE_SANS_HASH: usize = 16;
2119 const FAN_LEN: usize = 256;
2220 const HEADER_LEN: usize = 8;
23−const OID_LOOKUP_ENTRY_SIZE: usize = SHA1_SIZE;
2421
2522 const SIGNATURE: &[u8] = b"CGPH";
2623
@@ -50,6 +47,7 @@ pub struct File { · 23 unmodified lines
5047 fan: [u32; FAN_LEN],
5148 oid_lookup_offset: usize,
5249 path: PathBuf,
50+ hash_len: usize,
5351 }
5452
5553 /// The position of a given commit within a graph file, starting at 0.
git-commitgraph/src/file/verify.rs+7−9
@@ -5,9 +5,6 @@ use std::{ · 4 unmodified lines
55 path::Path,
66 };
77
8−use bstr::ByteSlice;
9−use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
10−
118 use crate::{
129 file::{self, File},
1310 GENERATION_NUMBER_INFINITY, GENERATION_NUMBER_MAX,
@@ -68,7 +65,8 @@ pub struct Outcome { · 54 unmodified lines
6865 impl File {
6966 /// Returns the trailing checksum over the entire content of this file.
7067 pub fn checksum(&self) -> &git_hash::oid {
71− git_hash::oid::try_from(&self.data[self.data.len() - SHA1_SIZE..]).expect("file to be large enough for a hash")
68+ git_hash::oid::try_from(&self.data[self.data.len() - self.hash_len..])
69+ .expect("file to be large enough for a hash")
7270 }
7371
7472 /// Traverse all [commits][file::Commit] stored in this file and call `processor(commit) -> Result<(), Error>` on it.
@@ -83,7 +81,7 @@ impl File { · 8 unmodified lines
8381 .map_err(|(actual, expected)| Error::Mismatch { actual, expected })?;
8482 verify_split_chain_filename_hash(&self.path, self.checksum()).map_err(Error::Filename)?;
8583
86− let null_id = git_hash::oid::null_sha1();
84+ let null_id = self.hash_kind().null();
8785
8886 let mut stats = Outcome {
8987 max_generation: 0,
@@ -150,10 +148,10 @@ impl File { · 60 unmodified lines
150148 // Error type to support io::Error and Mismatch. As we only gain progress, there probably isn't much value
151149 // as these files are usually small enough to process them in less than a second, even for the large ones.
152150 // But it's possible, once a progress instance is passed.
153− let data_len_without_trailer = self.data.len() - SHA1_SIZE;
154− let mut hasher = git_features::hash::Sha1::default();
151+ let data_len_without_trailer = self.data.len() - self.hash_len;
152+ let mut hasher = git_features::hash::hasher(self.hash_kind());
155153 hasher.update(&self.data[..data_len_without_trailer]);
156− let actual = git_hash::ObjectId::new_sha1(hasher.digest());
154+ let actual = git_hash::ObjectId::from(hasher.digest().as_ref());
157155
158156 let expected = self.checksum();
159157 if actual == expected {
@@ -174,6 +172,6 @@ fn verify_split_chain_filename_hash(path: impl AsRef<Path>, expected: &git_hash: · 14 unmodified lines
174172 .and_then(|stem| stem.strip_prefix("graph-"))
175173 .map_or(Ok(()), |hex| match git_hash::ObjectId::from_hex(hex.as_bytes()) {
176174 Ok(actual) if actual == expected => Ok(()),
177− _ => Err(format!("graph-{}.graph", expected.to_sha1_hex().as_bstr())),
175+ _ => Err(format!("graph-{}.graph", expected.to_hex())),
178176 })
179177 }