History

feat: Add support for hashes of different size (#279)

Even though right now, there is only Sha1, in future it's easy to support other hash sizes.

Sebastian Thiel committed 4 years ago

Commit 265b8ec1 parent bf620675 files · +52 −59Browse files

Showing the changes to git-commitgraph/src (5 of 5 files). Show all

Unified · Split

5 files
  1. git-commitgraph/src/file/access.rs +12 −13
  2. git-commitgraph/src/file/commit.rs +6 −8
  3. git-commitgraph/src/file/init.rs +25 −25
  4. git-commitgraph/src/file/mod.rs +2 −4
  5. git-commitgraph/src/file/verify.rs +7 −9
git-commitgraph/src/file/access.rs+12−13
@@ -4,9 +4,7 @@ use std::{ · 3 unmodified lines
4 path::Path,4 path::Path,
5};5};
6 6
7use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;7use crate::file::{self, commit::Commit, File, COMMIT_DATA_ENTRY_SIZE_SANS_HASH};
8
9use crate::file::{self, commit::Commit, File, COMMIT_DATA_ENTRY_SIZE};
10 8
11/// Access9/// Access
12impl File {10impl File {
@@ -30,10 +28,10 @@ impl File { · 17 unmodified lines
30 ///28 ///
31 /// Note that it is always conforming to the hash used in the owning repository.29 /// Note that it is always conforming to the hash used in the owning repository.
32 pub fn hash_kind(&self) -> git_hash::Kind {30 pub fn hash_kind(&self) -> git_hash::Kind {
33 git_hash::Kind::Sha131 git_hash::Kind::from_len_in_bytes(self.hash_len).expect("hash len validated during init")
34 }32 }
35 33
36 /// Returns 20 bytes sha1 at the given index in our list of (sorted) sha1 hashes.34 /// Returns an object id at the given index in our list of (sorted) hashes.
37 /// The position ranges from 0 to self.num_commits()35 /// The position ranges from 0 to self.num_commits()
38 // copied from git-odb/src/pack/index/ext36 // copied from git-odb/src/pack/index/ext
39 pub fn id_at(&self, pos: file::Position) -> &git_hash::oid {37 pub fn id_at(&self, pos: file::Position) -> &git_hash::oid {
@@ -47,17 +45,17 @@ impl File { · 7 unmodified lines
47 .045 .0
48 .try_into()46 .try_into()
49 .expect("an architecture able to hold 32 bits of integer");47 .expect("an architecture able to hold 32 bits of integer");
50 let start = self.oid_lookup_offset + (pos * SHA1_SIZE);48 let start = self.oid_lookup_offset + (pos * self.hash_len);
51 git_hash::oid::try_from(&self.data[start..start + SHA1_SIZE]).expect("20 bytes SHA1 to be alright")49 git_hash::oid::try_from(&self.data[start..][..self.hash_len]).expect("hash bytes to be alright")
52 }50 }
53 51
54 /// Return an iterator over all object hashes stored in the base graph.52 /// Return an iterator over all object hashes stored in the base graph.
55 pub fn iter_base_graph_ids(&self) -> impl Iterator<Item = &git_hash::oid> {53 pub fn iter_base_graph_ids(&self) -> impl Iterator<Item = &git_hash::oid> {
56 let start = self.base_graphs_list_offset.unwrap_or(0);54 let start = self.base_graphs_list_offset.unwrap_or(0);
57 let base_graphs_list = &self.data[start..start + (SHA1_SIZE * usize::from(self.base_graph_count))];55 let base_graphs_list = &self.data[start..][..self.hash_len * usize::from(self.base_graph_count)];
58 base_graphs_list56 base_graphs_list
59 .chunks(SHA1_SIZE)57 .chunks(self.hash_len)
60 .map(|bytes| git_hash::oid::try_from(bytes).expect("20 bytes SHA1 to be alright"))58 .map(|bytes| git_hash::oid::try_from(bytes).expect("hash bytes to be alright"))
61 }59 }
62 60
63 /// return an iterator over all commits in this file.61 /// return an iterator over all commits in this file.
@@ -80,7 +78,7 @@ impl File { · 16 unmodified lines
80 78
81 // Bisect using indices79 // Bisect using indices
82 // TODO: Performance of V2 could possibly be better if we would be able to do a binary search80 // TODO: Performance of V2 could possibly be better if we would be able to do a binary search
83 // on 20 byte chunks directly, but doing so requires transmuting and that is not safe, even though81 // on hash-sized chunks directly, but doing so requires transmuting and that is not safe, even though
84 // it should not be if the bytes match up and the type has no destructor.82 // it should not be if the bytes match up and the type has no destructor.
85 while lower_bound < upper_bound {83 while lower_bound < upper_bound {
86 let mid = (lower_bound + upper_bound) / 2;84 let mid = (lower_bound + upper_bound) / 2;
@@ -123,8 +121,9 @@ impl File { · 36 unmodified lines
123 .0121 .0
124 .try_into()122 .try_into()
125 .expect("an architecture able to hold 32 bits of integer");123 .expect("an architecture able to hold 32 bits of integer");
126 let start = self.commit_data_offset + (pos * COMMIT_DATA_ENTRY_SIZE);124 let entry_size = self.hash_len + COMMIT_DATA_ENTRY_SIZE_SANS_HASH;
127 &self.data[start..start + COMMIT_DATA_ENTRY_SIZE]125 let start = self.commit_data_offset + (pos * entry_size);
126 &self.data[start..][..entry_size]
128 }127 }
129 128
130 /// Returns the byte slice for this file's entire Extra Edge List (EDGE) chunk.129 /// Returns the byte slice for this file's entire Extra Edge List (EDGE) chunk.
git-commitgraph/src/file/commit.rs+6−8
@@ -5,13 +5,11 @@ use std::{ · 4 unmodified lines
5 slice::Chunks,5 slice::Chunks,
6};6};
7 7
8use byteorder::{BigEndian, ByteOrder};
9use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
10
11use crate::{8use crate::{
12 file::{self, File, EXTENDED_EDGES_MASK, LAST_EXTENDED_EDGE_MASK, NO_PARENT},9 file::{self, File, EXTENDED_EDGES_MASK, LAST_EXTENDED_EDGE_MASK, NO_PARENT},
13 graph,10 graph,
14};11};
12use byteorder::{BigEndian, ByteOrder};
15 13
16/// The error used in the [`file::commit`][self] module.14/// The error used in the [`file::commit`][self] module.
17#[derive(thiserror::Error, Debug)]15#[derive(thiserror::Error, Debug)]
@@ -45,11 +43,11 @@ impl<'a> Commit<'a> { · 27 unmodified lines
45 Commit {43 Commit {
46 file,44 file,
47 pos,45 pos,
48 root_tree_id: git_hash::oid::try_from(&bytes[..SHA1_SIZE]).expect("20 bytes SHA1 to be alright"),46 root_tree_id: git_hash::oid::try_from(&bytes[..file.hash_len]).expect("hash bytes to be alright"),
49 parent1: ParentEdge::from_raw(BigEndian::read_u32(&bytes[SHA1_SIZE..SHA1_SIZE + 4])),47 parent1: ParentEdge::from_raw(BigEndian::read_u32(&bytes[file.hash_len..][..4])),
50 parent2: ParentEdge::from_raw(BigEndian::read_u32(&bytes[SHA1_SIZE + 4..SHA1_SIZE + 8])),48 parent2: ParentEdge::from_raw(BigEndian::read_u32(&bytes[file.hash_len + 4..][..4])),
51 generation: BigEndian::read_u32(&bytes[SHA1_SIZE + 8..SHA1_SIZE + 12]) >> 2,49 generation: BigEndian::read_u32(&bytes[file.hash_len + 8..][..4]) >> 2,
52 commit_timestamp: BigEndian::read_u64(&bytes[SHA1_SIZE + 8..SHA1_SIZE + 16]) & 0x0003_ffff_ffff,50 commit_timestamp: BigEndian::read_u64(&bytes[file.hash_len + 8..][..8]) & 0x0003_ffff_ffff,
53 }51 }
54 }52 }
55 53
git-commitgraph/src/file/init.rs+25−25
@@ -6,12 +6,10 @@ use std::{ · 5 unmodified lines
6use bstr::ByteSlice;6use bstr::ByteSlice;
7use byteorder::{BigEndian, ByteOrder};7use byteorder::{BigEndian, ByteOrder};
8use filebuffer::FileBuffer;8use filebuffer::FileBuffer;
9use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
10 9
11use crate::file::{10use crate::file::{
12 ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, CHUNK_LOOKUP_SIZE, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE,11 ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE_SANS_HASH,
13 EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, OID_LOOKUP_ENTRY_SIZE,12 EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, SIGNATURE,
14 SIGNATURE,
15};13};
16 14
17/// The error used in [`File::at()`].15/// The error used in [`File::at()`].
@@ -52,11 +50,9 @@ pub enum Error { · 34 unmodified lines
52 InvalidChunkSize { id: ChunkId, msg: String },50 InvalidChunkSize { id: ChunkId, msg: String },
53}51}
54 52
55const TRAILER_LEN: usize = SHA1_SIZE;53const MIN_FILE_SIZE: usize = HEADER_LEN
56const MIN_FILE_SIZE: usize = HEADER_LEN + ((MIN_CHUNKS + 1) * CHUNK_LOOKUP_SIZE) + TRAILER_LEN;54 + git_chunk::file::Index::size_for_entries(3 /*OIDF, OIDL, CDAT*/)
57 55 + git_hash::Kind::shortest().len_in_bytes();
58// Required chunks: OIDF, OIDL, CDAT
59const MIN_CHUNKS: usize = 3;
60 56
61impl File {57impl File {
62 /// Try to parse the commit graph file at `path`.58 /// Try to parse the commit graph file at `path`.
@@ -96,8 +92,8 @@ impl TryFrom<&Path> for File { · 33 unmodified lines
96 };92 };
97 ofs += 1;93 ofs += 1;
98 94
99 match data[ofs] {95 let hash_kind = match data[ofs] {
100 1 => (),96 1 => git_hash::Kind::Sha1,
101 x => {97 x => {
102 return Err(Error::UnsupportedHashVersion(x));98 return Err(Error::UnsupportedHashVersion(x));
103 }99 }
@@ -117,13 +113,17 @@ impl TryFrom<&Path> for File { · 13 unmodified lines
117 let base_graphs_list_offset = chunks113 let base_graphs_list_offset = chunks
118 .validated_usize_offset_by_id(BASE_GRAPHS_LIST_CHUNK_ID, |chunk_range| {114 .validated_usize_offset_by_id(BASE_GRAPHS_LIST_CHUNK_ID, |chunk_range| {
119 let chunk_size = chunk_range.len();115 let chunk_size = chunk_range.len();
120 if chunk_size % SHA1_SIZE != 0 {116 if chunk_size % hash_kind.len_in_bytes() != 0 {
121 return Err(Error::InvalidChunkSize {117 return Err(Error::InvalidChunkSize {
122 id: BASE_GRAPHS_LIST_CHUNK_ID,118 id: BASE_GRAPHS_LIST_CHUNK_ID,
123 msg: format!("chunk size {} is not a multiple of {}", chunk_size, SHA1_SIZE),119 msg: format!(
120 "chunk size {} is not a multiple of {}",
121 chunk_size,
122 hash_kind.len_in_bytes()
123 ),
124 });124 });
125 }125 }
126 let chunk_base_graph_count: u32 = (chunk_size / SHA1_SIZE)126 let chunk_base_graph_count: u32 = (chunk_size / hash_kind.len_in_bytes())
127 .try_into()127 .try_into()
128 .expect("base graph count to fit in 32-bits");128 .expect("base graph count to fit in 32-bits");
129 if chunk_base_graph_count != u32::from(base_graph_count) {129 if chunk_base_graph_count != u32::from(base_graph_count) {
@@ -141,18 +141,16 @@ impl TryFrom<&Path> for File { · 11 unmodified lines
141 chunks.validated_usize_offset_by_id(COMMIT_DATA_CHUNK_ID, |chunk_range| {141 chunks.validated_usize_offset_by_id(COMMIT_DATA_CHUNK_ID, |chunk_range| {
142 let chunk_size = chunk_range.len();142 let chunk_size = chunk_range.len();
143 143
144 if chunk_size % COMMIT_DATA_ENTRY_SIZE != 0 {144 let entry_size = hash_kind.len_in_bytes() + COMMIT_DATA_ENTRY_SIZE_SANS_HASH;
145 if chunk_size % entry_size != 0 {
145 return Err(Error::InvalidChunkSize {146 return Err(Error::InvalidChunkSize {
146 id: COMMIT_DATA_CHUNK_ID,147 id: COMMIT_DATA_CHUNK_ID,
147 msg: format!(148 msg: format!("chunk size {} is not a multiple of {}", chunk_size, entry_size),
148 "chunk size {} is not a multiple of {}",
149 chunk_size, COMMIT_DATA_ENTRY_SIZE
150 ),
151 });149 });
152 }150 }
153 Ok((151 Ok((
154 chunk_range.start,152 chunk_range.start,
155 (chunk_size / COMMIT_DATA_ENTRY_SIZE)153 (chunk_size / entry_size)
156 .try_into()154 .try_into()
157 .expect("number of commits in CDAT chunk to fit in 32 bits"),155 .expect("number of commits in CDAT chunk to fit in 32 bits"),
158 ))156 ))
@@ -175,18 +173,19 @@ impl TryFrom<&Path> for File { · 16 unmodified lines
175 chunks.validated_usize_offset_by_id(OID_LOOKUP_CHUNK_ID, |chunk_range| {173 chunks.validated_usize_offset_by_id(OID_LOOKUP_CHUNK_ID, |chunk_range| {
176 let chunk_size = chunk_range.len();174 let chunk_size = chunk_range.len();
177 175
178 if chunk_size % OID_LOOKUP_ENTRY_SIZE != 0 {176 if chunk_size % hash_kind.len_in_bytes() != 0 {
179 return Err(Error::InvalidChunkSize {177 return Err(Error::InvalidChunkSize {
180 id: OID_LOOKUP_CHUNK_ID,178 id: OID_LOOKUP_CHUNK_ID,
181 msg: format!(179 msg: format!(
182 "chunk size {} is not a multiple of {}",180 "chunk size {} is not a multiple of {}",
183 chunk_size, OID_LOOKUP_ENTRY_SIZE181 chunk_size,
182 hash_kind.len_in_bytes()
184 ),183 ),
185 });184 });
186 }185 }
187 Ok((186 Ok((
188 chunk_range.start,187 chunk_range.start,
189 (chunk_size / OID_LOOKUP_ENTRY_SIZE)188 (chunk_size / hash_kind.len_in_bytes())
190 .try_into()189 .try_into()
191 .expect("number of commits in OIDL chunk to fit in 32 bits"),190 .expect("number of commits in OIDL chunk to fit in 32 bits"),
192 ))191 ))
@@ -195,10 +194,10 @@ impl TryFrom<&Path> for File { · 2 unmodified lines
195 let extra_edges_list_range = chunks.usize_offset_by_id(EXTENDED_EDGES_LIST_CHUNK_ID).ok();194 let extra_edges_list_range = chunks.usize_offset_by_id(EXTENDED_EDGES_LIST_CHUNK_ID).ok();
196 195
197 let trailer = &data[chunks.highest_offset() as usize..];196 let trailer = &data[chunks.highest_offset() as usize..];
198 if trailer.len() != TRAILER_LEN {197 if trailer.len() != hash_kind.len_in_bytes() {
199 return Err(Error::Trailer(format!(198 return Err(Error::Trailer(format!(
200 "Expected commit-graph trailer to contain {} bytes, got {}",199 "Expected commit-graph trailer to contain {} bytes, got {}",
201 TRAILER_LEN,200 hash_kind.len_in_bytes(),
202 trailer.len()201 trailer.len()
203 )));202 )));
204 }203 }
@@ -236,6 +235,7 @@ impl TryFrom<&Path> for File { · 31 unmodified lines
236 fan,235 fan,
237 oid_lookup_offset,236 oid_lookup_offset,
238 path: path.to_owned(),237 path: path.to_owned(),
238 hash_len: hash_kind.len_in_bytes(),
239 })239 })
240 }240 }
241}241}
git-commitgraph/src/file/mod.rs+2−4
@@ -7,7 +7,6 @@ use std::{ · 6 unmodified lines
7};7};
8 8
9use filebuffer::FileBuffer;9use filebuffer::FileBuffer;
10use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
11 10
12pub use self::{commit::Commit, init::Error};11pub use self::{commit::Commit, init::Error};
13 12
@@ -16,11 +15,9 @@ pub mod commit; · 2 unmodified lines
16mod init;15mod init;
17pub mod verify;16pub mod verify;
18 17
19const CHUNK_LOOKUP_SIZE: usize = 12;18const COMMIT_DATA_ENTRY_SIZE_SANS_HASH: usize = 16;
20const COMMIT_DATA_ENTRY_SIZE: usize = SHA1_SIZE + 16;
21const FAN_LEN: usize = 256;19const FAN_LEN: usize = 256;
22const HEADER_LEN: usize = 8;20const HEADER_LEN: usize = 8;
23const OID_LOOKUP_ENTRY_SIZE: usize = SHA1_SIZE;
24 21
25const SIGNATURE: &[u8] = b"CGPH";22const SIGNATURE: &[u8] = b"CGPH";
26 23
@@ -50,6 +47,7 @@ pub struct File { · 23 unmodified lines
50 fan: [u32; FAN_LEN],47 fan: [u32; FAN_LEN],
51 oid_lookup_offset: usize,48 oid_lookup_offset: usize,
52 path: PathBuf,49 path: PathBuf,
50 hash_len: usize,
53}51}
54 52
55/// The position of a given commit within a graph file, starting at 0.53/// The position of a given commit within a graph file, starting at 0.
git-commitgraph/src/file/verify.rs+7−9
@@ -5,9 +5,6 @@ use std::{ · 4 unmodified lines
5 path::Path,5 path::Path,
6};6};
7 7
8use bstr::ByteSlice;
9use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE;
10
11use crate::{8use crate::{
12 file::{self, File},9 file::{self, File},
13 GENERATION_NUMBER_INFINITY, GENERATION_NUMBER_MAX,10 GENERATION_NUMBER_INFINITY, GENERATION_NUMBER_MAX,
@@ -68,7 +65,8 @@ pub struct Outcome { · 54 unmodified lines
68impl File {65impl File {
69 /// Returns the trailing checksum over the entire content of this file.66 /// Returns the trailing checksum over the entire content of this file.
70 pub fn checksum(&self) -> &git_hash::oid {67 pub fn checksum(&self) -> &git_hash::oid {
71 git_hash::oid::try_from(&self.data[self.data.len() - SHA1_SIZE..]).expect("file to be large enough for a hash")68 git_hash::oid::try_from(&self.data[self.data.len() - self.hash_len..])
69 .expect("file to be large enough for a hash")
72 }70 }
73 71
74 /// Traverse all [commits][file::Commit] stored in this file and call `processor(commit) -> Result<(), Error>` on it.72 /// Traverse all [commits][file::Commit] stored in this file and call `processor(commit) -> Result<(), Error>` on it.
@@ -83,7 +81,7 @@ impl File { · 8 unmodified lines
83 .map_err(|(actual, expected)| Error::Mismatch { actual, expected })?;81 .map_err(|(actual, expected)| Error::Mismatch { actual, expected })?;
84 verify_split_chain_filename_hash(&self.path, self.checksum()).map_err(Error::Filename)?;82 verify_split_chain_filename_hash(&self.path, self.checksum()).map_err(Error::Filename)?;
85 83
86 let null_id = git_hash::oid::null_sha1();84 let null_id = self.hash_kind().null();
87 85
88 let mut stats = Outcome {86 let mut stats = Outcome {
89 max_generation: 0,87 max_generation: 0,
@@ -150,10 +148,10 @@ impl File { · 60 unmodified lines
150 // Error type to support io::Error and Mismatch. As we only gain progress, there probably isn't much value148 // Error type to support io::Error and Mismatch. As we only gain progress, there probably isn't much value
151 // as these files are usually small enough to process them in less than a second, even for the large ones.149 // as these files are usually small enough to process them in less than a second, even for the large ones.
152 // But it's possible, once a progress instance is passed.150 // But it's possible, once a progress instance is passed.
153 let data_len_without_trailer = self.data.len() - SHA1_SIZE;151 let data_len_without_trailer = self.data.len() - self.hash_len;
154 let mut hasher = git_features::hash::Sha1::default();152 let mut hasher = git_features::hash::hasher(self.hash_kind());
155 hasher.update(&self.data[..data_len_without_trailer]);153 hasher.update(&self.data[..data_len_without_trailer]);
156 let actual = git_hash::ObjectId::new_sha1(hasher.digest());154 let actual = git_hash::ObjectId::from(hasher.digest().as_ref());
157 155
158 let expected = self.checksum();156 let expected = self.checksum();
159 if actual == expected {157 if actual == expected {
@@ -174,6 +172,6 @@ fn verify_split_chain_filename_hash(path: impl AsRef<Path>, expected: &git_hash: · 14 unmodified lines
174 .and_then(|stem| stem.strip_prefix("graph-"))172 .and_then(|stem| stem.strip_prefix("graph-"))
175 .map_or(Ok(()), |hex| match git_hash::ObjectId::from_hex(hex.as_bytes()) {173 .map_or(Ok(()), |hex| match git_hash::ObjectId::from_hex(hex.as_bytes()) {
176 Ok(actual) if actual == expected => Ok(()),174 Ok(actual) if actual == expected => Ok(()),
177 _ => Err(format!("graph-{}.graph", expected.to_sha1_hex().as_bstr())),175 _ => Err(format!("graph-{}.graph", expected.to_hex())),
178 })176 })
179}177}