History

First round of introducing git-chunk (#279)

Sebastian Thiel committed 4 years ago

Commit 51b991b1 parent 93280152 files · +94 −142Browse files

Showing the changes to git-commitgraph/src (2 of 2 files). Show all

Unified · Split

2 files
  1. git-commitgraph/src/file/init.rs +93 −140
  2. git-commitgraph/src/file/mod.rs +1 −2
git-commitgraph/src/file/init.rs+93−140
@@ -1,6 +1,5 @@
11 use std::{
22 convert::{TryFrom, TryInto},
3− ops::Range,
43 path::Path,
54 };
65
@@ -12,7 +11,7 @@ use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE; · 5 unmodified lines
1211 use crate::file::{
1312 ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, CHUNK_LOOKUP_SIZE, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE,
1413 EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, OID_LOOKUP_ENTRY_SIZE,
15− SENTINEL_CHUNK_ID, SIGNATURE,
14+ SIGNATURE,
1615 };
1716
1817 /// The error used in [`File::at()`].
@@ -51,6 +50,8 @@ pub enum Error { · 32 unmodified lines
5150 UnsupportedHashVersion(u8),
5251 #[error("Unsupported commit-graph file version: {0}")]
5352 UnsupportedVersion(u8),
53+ #[error(transparent)]
54+ ChunkFileDecode(#[from] git_chunk::file::decode::Error),
5455 }
5556
5657 const TRAILER_LEN: usize = SHA1_SIZE;
@@ -113,167 +114,119 @@ impl TryFrom<&Path> for File { · 56 unmodified lines
113114 let base_graph_count = data[ofs];
114115 ofs += 1;
115116
116− let chunk_lookup_end = ofs + ((usize::from(chunk_count) + 1) * CHUNK_LOOKUP_SIZE);
117− if chunk_lookup_end > data_size {
118− return Err(Error::Corrupt(format!(
119− "Commit-graph file is too small to hold {} chunks",
120− chunk_count
121− )));
122− }
117+ let chunks = git_chunk::file::Index::from_bytes(&data, ofs, chunk_count as u32)?;
123118
124119 let mut base_graphs_list_offset: Option<usize> = None;
125120 let mut commit_data_offset: Option<usize> = None;
126121 let mut commit_data_count = 0u32;
127− let mut extra_edges_list_range: Option<Range<usize>> = None;
128122 let mut fan_offset: Option<usize> = None;
129123 let mut oid_lookup_offset: Option<usize> = None;
130124 let mut oid_lookup_count = 0u32;
131125
132− let mut chunk_id: ChunkId = data[ofs..ofs + 4].try_into().expect("ChunkId to accept 4 bytes");
133− ofs += 4;
126+ if let Some(chunk_range) = chunks
127+ .offset_by_id(BASE_GRAPHS_LIST_CHUNK_ID)
128+ .ok()
129+ .and_then(git_chunk::range::into_usize)
130+ {
131+ let chunk_size = chunk_range.len();
132+ if chunk_size % SHA1_SIZE != 0 {
133+ return Err(Error::InvalidChunkSize {
134+ id: BASE_GRAPHS_LIST_CHUNK_ID,
135+ msg: format!("chunk size {} is not a multiple of {}", chunk_size, SHA1_SIZE),
136+ });
137+ }
138+ let chunk_base_graph_count: u32 = (chunk_size / SHA1_SIZE)
139+ .try_into()
140+ .expect("base graph count to fit in 32-bits");
141+ if chunk_base_graph_count != u32::from(base_graph_count) {
142+ return Err(Error::BaseGraphMismatch {
143+ from_chunk: chunk_base_graph_count,
144+ from_header: base_graph_count,
145+ });
146+ }
147+ base_graphs_list_offset = Some(chunk_range.start);
148+ }
149+ if let Some(chunk_range) = chunks
150+ .offset_by_id(COMMIT_DATA_CHUNK_ID)
151+ .ok()
152+ .and_then(git_chunk::range::into_usize)
153+ {
154+ let chunk_size = chunk_range.len();
134155
135− let mut chunk_offset: usize = BigEndian::read_u64(&data[ofs..ofs + 8])
136− .try_into()
137− .expect("an offset small enough to fit a usize");
138− if chunk_offset < chunk_lookup_end {
139− return Err(Error::Corrupt(format!(
140− "Commit-graph chunk 0 has invalid offset {} (must be at least {})",
141− chunk_offset, chunk_lookup_end
142− )));
156+ if chunk_size % COMMIT_DATA_ENTRY_SIZE != 0 {
157+ return Err(Error::InvalidChunkSize {
158+ id: COMMIT_DATA_CHUNK_ID,
159+ msg: format!(
160+ "chunk size {} is not a multiple of {}",
161+ chunk_size, COMMIT_DATA_ENTRY_SIZE
162+ ),
163+ });
164+ }
165+ commit_data_offset = Some(chunk_range.start);
166+ commit_data_count = (chunk_size / COMMIT_DATA_ENTRY_SIZE)
167+ .try_into()
168+ .expect("number of commits in CDAT chunk to fit in 32 bits");
143169 }
144− ofs += 8;
145170
146− for _ in 0..chunk_count {
147− let next_chunk_id: ChunkId = data[ofs..ofs + 4].try_into().expect("ChunkId to accept 4 bytes");
148− ofs += 4;
171+ let extra_edges_list_range = chunks
172+ .offset_by_id(EXTENDED_EDGES_LIST_CHUNK_ID)
173+ .ok()
174+ .and_then(git_chunk::range::into_usize);
149175
150− let next_chunk_offset: usize = BigEndian::read_u64(&data[ofs..ofs + 8])
151− .try_into()
152− .expect("an offset small enough to fit a usize");
153− ofs += 8;
176+ if let Some(chunk_range) = chunks
177+ .offset_by_id(OID_FAN_CHUNK_ID)
178+ .ok()
179+ .and_then(git_chunk::range::into_usize)
180+ {
181+ let chunk_size = chunk_range.len();
154182
155− let chunk_size: usize =
156− next_chunk_offset
157− .checked_sub(chunk_offset)
158− .ok_or_else(|| Error::InvalidChunkSize {
159− id: chunk_id,
160− msg: "size is negative".to_string(),
161− })?;
162− if next_chunk_offset >= data_size {
183+ let expected_size = 4 * FAN_LEN;
184+ if chunk_size != expected_size {
163185 return Err(Error::InvalidChunkSize {
164− id: chunk_id,
165− msg: "chunk extends beyond end of file".to_string(),
186+ id: OID_FAN_CHUNK_ID,
187+ msg: format!("expected chunk length {}, got {}", expected_size, chunk_size),
166188 });
167189 }
190+ fan_offset = Some(chunk_range.start);
191+ }
168192
169− match chunk_id {
170− BASE_GRAPHS_LIST_CHUNK_ID => {
171− if base_graphs_list_offset.is_some() {
172− return Err(Error::DuplicateChunk(chunk_id));
173− }
174− if chunk_size % SHA1_SIZE != 0 {
175− return Err(Error::InvalidChunkSize {
176− id: chunk_id,
177− msg: format!("chunk size {} is not a multiple of {}", chunk_size, SHA1_SIZE),
178− });
179− }
180− let chunk_base_graph_count: u32 = (chunk_size / SHA1_SIZE)
181− .try_into()
182− .expect("base graph count to fit in 32-bits");
183− if chunk_base_graph_count != u32::from(base_graph_count) {
184− return Err(Error::BaseGraphMismatch {
185− from_chunk: chunk_base_graph_count,
186− from_header: base_graph_count,
187− });
188− }
189− base_graphs_list_offset = Some(chunk_offset);
190− }
191− COMMIT_DATA_CHUNK_ID => {
192− if commit_data_offset.is_some() {
193− return Err(Error::DuplicateChunk(chunk_id));
194− }
195− if chunk_size % COMMIT_DATA_ENTRY_SIZE != 0 {
196− return Err(Error::InvalidChunkSize {
197− id: chunk_id,
198− msg: format!(
199− "chunk size {} is not a multiple of {}",
200− chunk_size, COMMIT_DATA_ENTRY_SIZE
201− ),
202− });
203− }
204− commit_data_offset = Some(chunk_offset);
205− commit_data_count = (chunk_size / COMMIT_DATA_ENTRY_SIZE)
206− .try_into()
207− .expect("number of commits in CDAT chunk to fit in 32 bits");
208− }
209− EXTENDED_EDGES_LIST_CHUNK_ID => {
210− if extra_edges_list_range.is_some() {
211− return Err(Error::DuplicateChunk(chunk_id));
212− }
193+ if let Some(chunk_range) = chunks
194+ .offset_by_id(OID_LOOKUP_CHUNK_ID)
195+ .ok()
196+ .and_then(git_chunk::range::into_usize)
197+ {
198+ let chunk_size = chunk_range.len();
213199
214− extra_edges_list_range = Some(Range {
215− start: chunk_offset,
216− end: next_chunk_offset,
217− })
218− }
219− OID_FAN_CHUNK_ID => {
220− if fan_offset.is_some() {
221− return Err(Error::DuplicateChunk(chunk_id));
222− }
223− let expected_size = 4 * FAN_LEN;
224− if chunk_size != expected_size {
225− return Err(Error::InvalidChunkSize {
226− id: chunk_id,
227− msg: format!("expected chunk length {}, got {}", expected_size, chunk_size),
228− });
229− }
230− fan_offset = Some(chunk_offset);
231− }
232− OID_LOOKUP_CHUNK_ID => {
233− if oid_lookup_offset.is_some() {
234− return Err(Error::DuplicateChunk(chunk_id));
235− }
236− if chunk_size % OID_LOOKUP_ENTRY_SIZE != 0 {
237− return Err(Error::InvalidChunkSize {
238− id: chunk_id,
239− msg: format!(
240− "chunk size {} is not a multiple of {}",
241− chunk_size, OID_LOOKUP_ENTRY_SIZE
242− ),
243− });
244− }
245− oid_lookup_offset = Some(chunk_offset);
246− oid_lookup_count = (chunk_size / OID_LOOKUP_ENTRY_SIZE)
247− .try_into()
248− .expect("number of commits in OIDL chunk to fit in 32 bits");
249− // TODO(ST): Figure out how to handle this. Don't know what to do with the commented code.
250− // git allows extra garbage in the extra edges list chunk?
251− // if oid_lookup_count > 0 {
252− // let last_edge = &data[next_chunk_offset - 4..next_chunk_offset];
253− // let last_edge = BigEndian::read_u32(last_edge);
254− // if let ExtraEdge::Internal(_) = ExtraEdge::from_raw(last_edge) {
255− // return Err(Error::ExtraEdgesListOverflow);
256− // }
257− // }
258− }
259− _ => {}
200+ if chunk_size % OID_LOOKUP_ENTRY_SIZE != 0 {
201+ return Err(Error::InvalidChunkSize {
202+ id: OID_FAN_CHUNK_ID,
203+ msg: format!(
204+ "chunk size {} is not a multiple of {}",
205+ chunk_size, OID_LOOKUP_ENTRY_SIZE
206+ ),
207+ });
260208 }
261−
262− chunk_id = next_chunk_id;
263− chunk_offset = next_chunk_offset;
264− }
265− if chunk_id != SENTINEL_CHUNK_ID {
266− return Err(Error::Corrupt(format!(
267− "Commit-graph file has invalid last chunk ID: {:?}",
268− chunk_id.as_bstr()
269− )));
209+ oid_lookup_offset = Some(chunk_range.start);
210+ oid_lookup_count = (chunk_size / OID_LOOKUP_ENTRY_SIZE)
211+ .try_into()
212+ .expect("number of commits in OIDL chunk to fit in 32 bits");
213+ // TODO(ST): Figure out how to handle this. Don't know what to do with the commented code.
214+ // git allows extra garbage in the extra edges list chunk?
215+ // if oid_lookup_count > 0 {
216+ // let last_edge = &data[next_chunk_offset - 4..next_chunk_offset];
217+ // let last_edge = BigEndian::read_u32(last_edge);
218+ // if let ExtraEdge::Internal(_) = ExtraEdge::from_raw(last_edge) {
219+ // return Err(Error::ExtraEdgesListOverflow);
220+ // }
221+ // }
270222 }
271223
272− let actual_trailer_len = data_size.saturating_sub(chunk_offset);
273− if actual_trailer_len != TRAILER_LEN {
224+ let trailer = &data[chunks.highest_offset() as usize..];
225+ if trailer.len() != TRAILER_LEN {
274226 return Err(Error::Trailer(format!(
275227 "Expected commit-graph trailer to contain {} bytes, got {}",
276− TRAILER_LEN, actual_trailer_len
228+ TRAILER_LEN,
229+ trailer.len()
277230 )));
278231 }
279232
git-commitgraph/src/file/mod.rs+1−2
@@ -24,13 +24,12 @@ const OID_LOOKUP_ENTRY_SIZE: usize = SHA1_SIZE; · 23 unmodified lines
2424
2525 const SIGNATURE: &[u8] = b"CGPH";
2626
27−type ChunkId = [u8; 4];
27+type ChunkId = git_chunk::Id;
2828 const BASE_GRAPHS_LIST_CHUNK_ID: ChunkId = *b"BASE";
2929 const COMMIT_DATA_CHUNK_ID: ChunkId = *b"CDAT";
3030 const EXTENDED_EDGES_LIST_CHUNK_ID: ChunkId = *b"EDGE";
3131 const OID_FAN_CHUNK_ID: ChunkId = *b"OIDF";
3232 const OID_LOOKUP_CHUNK_ID: ChunkId = *b"OIDL";
33−const SENTINEL_CHUNK_ID: ChunkId = [0u8; 4];
3433
3534 // Note that git's commit-graph-format.txt as of v2.28.0 gives an incorrect value 0x0700_0000 for
3635 // NO_PARENT. Fixed in https://github.com/git/git/commit/4d515253afcef985e94400adbfed7044959f9121 .