History

First round of introducing git-chunk (#279)

Sebastian Thiel committed 4 years ago

Commit 51b991b1 parent 93280152 files · +94 −142Browse files

Showing the changes to git-commitgraph/src (2 of 2 files). Show all

Unified · Split

2 files
  1. git-commitgraph/src/file/init.rs +93 −140
  2. git-commitgraph/src/file/mod.rs +1 −2
git-commitgraph/src/file/init.rs+93−140
@@ -1,6 +1,5 @@
1use std::{1use std::{
2 convert::{TryFrom, TryInto},2 convert::{TryFrom, TryInto},
3 ops::Range,
4 path::Path,3 path::Path,
5};4};
6 5
@@ -12,7 +11,7 @@ use git_hash::SIZE_OF_SHA1_DIGEST as SHA1_SIZE; · 5 unmodified lines
12use crate::file::{11use crate::file::{
13 ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, CHUNK_LOOKUP_SIZE, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE,12 ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, CHUNK_LOOKUP_SIZE, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE,
14 EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, OID_LOOKUP_ENTRY_SIZE,13 EXTENDED_EDGES_LIST_CHUNK_ID, FAN_LEN, HEADER_LEN, OID_FAN_CHUNK_ID, OID_LOOKUP_CHUNK_ID, OID_LOOKUP_ENTRY_SIZE,
15 SENTINEL_CHUNK_ID, SIGNATURE,14 SIGNATURE,
16};15};
17 16
18/// The error used in [`File::at()`].17/// The error used in [`File::at()`].
@@ -51,6 +50,8 @@ pub enum Error { · 32 unmodified lines
51 UnsupportedHashVersion(u8),50 UnsupportedHashVersion(u8),
52 #[error("Unsupported commit-graph file version: {0}")]51 #[error("Unsupported commit-graph file version: {0}")]
53 UnsupportedVersion(u8),52 UnsupportedVersion(u8),
53 #[error(transparent)]
54 ChunkFileDecode(#[from] git_chunk::file::decode::Error),
54}55}
55 56
56const TRAILER_LEN: usize = SHA1_SIZE;57const TRAILER_LEN: usize = SHA1_SIZE;
@@ -113,167 +114,119 @@ impl TryFrom<&Path> for File { · 56 unmodified lines
113 let base_graph_count = data[ofs];114 let base_graph_count = data[ofs];
114 ofs += 1;115 ofs += 1;
115 116
116 let chunk_lookup_end = ofs + ((usize::from(chunk_count) + 1) * CHUNK_LOOKUP_SIZE);117 let chunks = git_chunk::file::Index::from_bytes(&data, ofs, chunk_count as u32)?;
117 if chunk_lookup_end > data_size {
118 return Err(Error::Corrupt(format!(
119 "Commit-graph file is too small to hold {} chunks",
120 chunk_count
121 )));
122 }
123 118
124 let mut base_graphs_list_offset: Option<usize> = None;119 let mut base_graphs_list_offset: Option<usize> = None;
125 let mut commit_data_offset: Option<usize> = None;120 let mut commit_data_offset: Option<usize> = None;
126 let mut commit_data_count = 0u32;121 let mut commit_data_count = 0u32;
127 let mut extra_edges_list_range: Option<Range<usize>> = None;
128 let mut fan_offset: Option<usize> = None;122 let mut fan_offset: Option<usize> = None;
129 let mut oid_lookup_offset: Option<usize> = None;123 let mut oid_lookup_offset: Option<usize> = None;
130 let mut oid_lookup_count = 0u32;124 let mut oid_lookup_count = 0u32;
131 125
132 let mut chunk_id: ChunkId = data[ofs..ofs + 4].try_into().expect("ChunkId to accept 4 bytes");126 if let Some(chunk_range) = chunks
133 ofs += 4;127 .offset_by_id(BASE_GRAPHS_LIST_CHUNK_ID)
128 .ok()
129 .and_then(git_chunk::range::into_usize)
130 {
131 let chunk_size = chunk_range.len();
132 if chunk_size % SHA1_SIZE != 0 {
133 return Err(Error::InvalidChunkSize {
134 id: BASE_GRAPHS_LIST_CHUNK_ID,
135 msg: format!("chunk size {} is not a multiple of {}", chunk_size, SHA1_SIZE),
136 });
137 }
138 let chunk_base_graph_count: u32 = (chunk_size / SHA1_SIZE)
139 .try_into()
140 .expect("base graph count to fit in 32-bits");
141 if chunk_base_graph_count != u32::from(base_graph_count) {
142 return Err(Error::BaseGraphMismatch {
143 from_chunk: chunk_base_graph_count,
144 from_header: base_graph_count,
145 });
146 }
147 base_graphs_list_offset = Some(chunk_range.start);
148 }
149 if let Some(chunk_range) = chunks
150 .offset_by_id(COMMIT_DATA_CHUNK_ID)
151 .ok()
152 .and_then(git_chunk::range::into_usize)
153 {
154 let chunk_size = chunk_range.len();
134 155
135 let mut chunk_offset: usize = BigEndian::read_u64(&data[ofs..ofs + 8])156 if chunk_size % COMMIT_DATA_ENTRY_SIZE != 0 {
136 .try_into()157 return Err(Error::InvalidChunkSize {
137 .expect("an offset small enough to fit a usize");158 id: COMMIT_DATA_CHUNK_ID,
138 if chunk_offset < chunk_lookup_end {159 msg: format!(
139 return Err(Error::Corrupt(format!(160 "chunk size {} is not a multiple of {}",
140 "Commit-graph chunk 0 has invalid offset {} (must be at least {})",161 chunk_size, COMMIT_DATA_ENTRY_SIZE
141 chunk_offset, chunk_lookup_end162 ),
142 )));163 });
164 }
165 commit_data_offset = Some(chunk_range.start);
166 commit_data_count = (chunk_size / COMMIT_DATA_ENTRY_SIZE)
167 .try_into()
168 .expect("number of commits in CDAT chunk to fit in 32 bits");
143 }169 }
144 ofs += 8;
145 170
146 for _ in 0..chunk_count {171 let extra_edges_list_range = chunks
147 let next_chunk_id: ChunkId = data[ofs..ofs + 4].try_into().expect("ChunkId to accept 4 bytes");172 .offset_by_id(EXTENDED_EDGES_LIST_CHUNK_ID)
148 ofs += 4;173 .ok()
174 .and_then(git_chunk::range::into_usize);
149 175
150 let next_chunk_offset: usize = BigEndian::read_u64(&data[ofs..ofs + 8])176 if let Some(chunk_range) = chunks
151 .try_into()177 .offset_by_id(OID_FAN_CHUNK_ID)
152 .expect("an offset small enough to fit a usize");178 .ok()
153 ofs += 8;179 .and_then(git_chunk::range::into_usize)
180 {
181 let chunk_size = chunk_range.len();
154 182
155 let chunk_size: usize =183 let expected_size = 4 * FAN_LEN;
156 next_chunk_offset184 if chunk_size != expected_size {
157 .checked_sub(chunk_offset)
158 .ok_or_else(|| Error::InvalidChunkSize {
159 id: chunk_id,
160 msg: "size is negative".to_string(),
161 })?;
162 if next_chunk_offset >= data_size {
163 return Err(Error::InvalidChunkSize {185 return Err(Error::InvalidChunkSize {
164 id: chunk_id,186 id: OID_FAN_CHUNK_ID,
165 msg: "chunk extends beyond end of file".to_string(),187 msg: format!("expected chunk length {}, got {}", expected_size, chunk_size),
166 });188 });
167 }189 }
190 fan_offset = Some(chunk_range.start);
191 }
168 192
169 match chunk_id {193 if let Some(chunk_range) = chunks
170 BASE_GRAPHS_LIST_CHUNK_ID => {194 .offset_by_id(OID_LOOKUP_CHUNK_ID)
171 if base_graphs_list_offset.is_some() {195 .ok()
172 return Err(Error::DuplicateChunk(chunk_id));196 .and_then(git_chunk::range::into_usize)
173 }197 {
174 if chunk_size % SHA1_SIZE != 0 {198 let chunk_size = chunk_range.len();
175 return Err(Error::InvalidChunkSize {
176 id: chunk_id,
177 msg: format!("chunk size {} is not a multiple of {}", chunk_size, SHA1_SIZE),
178 });
179 }
180 let chunk_base_graph_count: u32 = (chunk_size / SHA1_SIZE)
181 .try_into()
182 .expect("base graph count to fit in 32-bits");
183 if chunk_base_graph_count != u32::from(base_graph_count) {
184 return Err(Error::BaseGraphMismatch {
185 from_chunk: chunk_base_graph_count,
186 from_header: base_graph_count,
187 });
188 }
189 base_graphs_list_offset = Some(chunk_offset);
190 }
191 COMMIT_DATA_CHUNK_ID => {
192 if commit_data_offset.is_some() {
193 return Err(Error::DuplicateChunk(chunk_id));
194 }
195 if chunk_size % COMMIT_DATA_ENTRY_SIZE != 0 {
196 return Err(Error::InvalidChunkSize {
197 id: chunk_id,
198 msg: format!(
199 "chunk size {} is not a multiple of {}",
200 chunk_size, COMMIT_DATA_ENTRY_SIZE
201 ),
202 });
203 }
204 commit_data_offset = Some(chunk_offset);
205 commit_data_count = (chunk_size / COMMIT_DATA_ENTRY_SIZE)
206 .try_into()
207 .expect("number of commits in CDAT chunk to fit in 32 bits");
208 }
209 EXTENDED_EDGES_LIST_CHUNK_ID => {
210 if extra_edges_list_range.is_some() {
211 return Err(Error::DuplicateChunk(chunk_id));
212 }
213 199
214 extra_edges_list_range = Some(Range {200 if chunk_size % OID_LOOKUP_ENTRY_SIZE != 0 {
215 start: chunk_offset,201 return Err(Error::InvalidChunkSize {
216 end: next_chunk_offset,202 id: OID_FAN_CHUNK_ID,
217 })203 msg: format!(
218 }204 "chunk size {} is not a multiple of {}",
219 OID_FAN_CHUNK_ID => {205 chunk_size, OID_LOOKUP_ENTRY_SIZE
220 if fan_offset.is_some() {206 ),
221 return Err(Error::DuplicateChunk(chunk_id));207 });
222 }
223 let expected_size = 4 * FAN_LEN;
224 if chunk_size != expected_size {
225 return Err(Error::InvalidChunkSize {
226 id: chunk_id,
227 msg: format!("expected chunk length {}, got {}", expected_size, chunk_size),
228 });
229 }
230 fan_offset = Some(chunk_offset);
231 }
232 OID_LOOKUP_CHUNK_ID => {
233 if oid_lookup_offset.is_some() {
234 return Err(Error::DuplicateChunk(chunk_id));
235 }
236 if chunk_size % OID_LOOKUP_ENTRY_SIZE != 0 {
237 return Err(Error::InvalidChunkSize {
238 id: chunk_id,
239 msg: format!(
240 "chunk size {} is not a multiple of {}",
241 chunk_size, OID_LOOKUP_ENTRY_SIZE
242 ),
243 });
244 }
245 oid_lookup_offset = Some(chunk_offset);
246 oid_lookup_count = (chunk_size / OID_LOOKUP_ENTRY_SIZE)
247 .try_into()
248 .expect("number of commits in OIDL chunk to fit in 32 bits");
249 // TODO(ST): Figure out how to handle this. Don't know what to do with the commented code.
250 // git allows extra garbage in the extra edges list chunk?
251 // if oid_lookup_count > 0 {
252 // let last_edge = &data[next_chunk_offset - 4..next_chunk_offset];
253 // let last_edge = BigEndian::read_u32(last_edge);
254 // if let ExtraEdge::Internal(_) = ExtraEdge::from_raw(last_edge) {
255 // return Err(Error::ExtraEdgesListOverflow);
256 // }
257 // }
258 }
259 _ => {}
260 }208 }
261 209 oid_lookup_offset = Some(chunk_range.start);
262 chunk_id = next_chunk_id;210 oid_lookup_count = (chunk_size / OID_LOOKUP_ENTRY_SIZE)
263 chunk_offset = next_chunk_offset;211 .try_into()
264 }212 .expect("number of commits in OIDL chunk to fit in 32 bits");
265 if chunk_id != SENTINEL_CHUNK_ID {213 // TODO(ST): Figure out how to handle this. Don't know what to do with the commented code.
266 return Err(Error::Corrupt(format!(214 // git allows extra garbage in the extra edges list chunk?
267 "Commit-graph file has invalid last chunk ID: {:?}",215 // if oid_lookup_count > 0 {
268 chunk_id.as_bstr()216 // let last_edge = &data[next_chunk_offset - 4..next_chunk_offset];
269 )));217 // let last_edge = BigEndian::read_u32(last_edge);
218 // if let ExtraEdge::Internal(_) = ExtraEdge::from_raw(last_edge) {
219 // return Err(Error::ExtraEdgesListOverflow);
220 // }
221 // }
270 }222 }
271 223
272 let actual_trailer_len = data_size.saturating_sub(chunk_offset);224 let trailer = &data[chunks.highest_offset() as usize..];
273 if actual_trailer_len != TRAILER_LEN {225 if trailer.len() != TRAILER_LEN {
274 return Err(Error::Trailer(format!(226 return Err(Error::Trailer(format!(
275 "Expected commit-graph trailer to contain {} bytes, got {}",227 "Expected commit-graph trailer to contain {} bytes, got {}",
276 TRAILER_LEN, actual_trailer_len228 TRAILER_LEN,
229 trailer.len()
277 )));230 )));
278 }231 }
279 232
git-commitgraph/src/file/mod.rs+1−2
@@ -24,13 +24,12 @@ const OID_LOOKUP_ENTRY_SIZE: usize = SHA1_SIZE; · 23 unmodified lines
24 24
25const SIGNATURE: &[u8] = b"CGPH";25const SIGNATURE: &[u8] = b"CGPH";
26 26
27type ChunkId = [u8; 4];27type ChunkId = git_chunk::Id;
28const BASE_GRAPHS_LIST_CHUNK_ID: ChunkId = *b"BASE";28const BASE_GRAPHS_LIST_CHUNK_ID: ChunkId = *b"BASE";
29const COMMIT_DATA_CHUNK_ID: ChunkId = *b"CDAT";29const COMMIT_DATA_CHUNK_ID: ChunkId = *b"CDAT";
30const EXTENDED_EDGES_LIST_CHUNK_ID: ChunkId = *b"EDGE";30const EXTENDED_EDGES_LIST_CHUNK_ID: ChunkId = *b"EDGE";
31const OID_FAN_CHUNK_ID: ChunkId = *b"OIDF";31const OID_FAN_CHUNK_ID: ChunkId = *b"OIDF";
32const OID_LOOKUP_CHUNK_ID: ChunkId = *b"OIDL";32const OID_LOOKUP_CHUNK_ID: ChunkId = *b"OIDL";
33const SENTINEL_CHUNK_ID: ChunkId = [0u8; 4];
34 33
35// Note that git's commit-graph-format.txt as of v2.28.0 gives an incorrect value 0x0700_0000 for34// Note that git's commit-graph-format.txt as of v2.28.0 gives an incorrect value 0x0700_0000 for
36// NO_PARENT. Fixed in https://github.com/git/git/commit/4d515253afcef985e94400adbfed7044959f9121 .35// NO_PARENT. Fixed in https://github.com/git/git/commit/4d515253afcef985e94400adbfed7044959f9121 .