History

[commitgraph] Implement basic commit-graph file verification.

Missing features: 1. It operates on commit-graph files only, so it doesn't verify that commit-graph data matches `git-odb` data. 2. No progress reporting or parallelization. This shouldn't be needed until until we need to check against `git-odb` data. Example output for Linux repo: ``` $ time ./target/release/gixp commit-graph-verify -s ~/src/linux/.git/objects/info number of commits with the given number of parents 0: 4 1: 878988 2: 67800 3: 652 4: 408 5: 382 6: 454 7: 95 8: 65 9: 47 10: 25 11: 26 12: 14 13: 4 14: 3 18: 1 19: 1 20: 1 21: 1 24: 1 27: 1 30: 1 32: 1 66: 1 ->: 948976 longest path length between two commits: 160521 real 0m0.196s user 0m0.180s sys 0m0.016s ```

Conor Davis committed 6 years ago

Commit 25711131 parent 701f33c19 files · +459 −21Browse files

Showing the changes to git-commitgraph/src/graph (3 of 19 files). Show all

Unified · Split

3 files
  1. git-commitgraph/src/graph/access.rs +7 −5
  2. git-commitgraph/src/graph/mod.rs +1 −0
  3. git-commitgraph/src/graph/verify.rs +158 −0
git-commitgraph/src/graph/access.rs+7−5
@@ -42,7 +42,7 @@ impl Graph { · 41 unmodified lines
4242
4343 /// Access fundamentals
4444 impl Graph {
45− fn lookup_by_id(&self, id: borrowed::Id<'_>) -> Option<LookupByIdResult<'_>> {
45+ pub(crate) fn lookup_by_id(&self, id: borrowed::Id<'_>) -> Option<LookupByIdResult<'_>> {
4646 let mut current_file_start = 0;
4747 for file in &self.files {
4848 if let Some(lex_pos) = file.lookup(id) {
@@ -57,14 +57,15 @@ impl Graph { · 8 unmodified lines
5757 None
5858 }
5959
60− fn lookup_by_pos(&self, pos: graph::Position) -> LookupByPositionResult<'_> {
60+ pub(crate) fn lookup_by_pos(&self, pos: graph::Position) -> LookupByPositionResult<'_> {
6161 let mut remaining = pos.0;
62− for file in &self.files {
62+ for (file_index, file) in self.files.iter().enumerate() {
6363 match remaining.checked_sub(file.num_commits()) {
6464 Some(v) => remaining = v,
6565 None => {
6666 return LookupByPositionResult {
6767 file,
68+ file_index,
6869 pos: file::Position(remaining),
6970 }
7071 }
@@ -75,14 +76,15 @@ impl Graph { · 4 unmodified lines
7576 }
7677
7778 #[derive(Clone)]
78−struct LookupByIdResult<'a> {
79+pub(crate) struct LookupByIdResult<'a> {
7980 pub file: &'a File,
8081 pub graph_pos: graph::Position,
8182 pub file_pos: file::Position,
8283 }
8384
8485 #[derive(Clone)]
85−struct LookupByPositionResult<'a> {
86+pub(crate) struct LookupByPositionResult<'a> {
8687 pub file: &'a File,
88+ pub file_index: usize,
8789 pub pos: file::Position,
8890 }
git-commitgraph/src/graph/mod.rs+1−0
@@ -1,6 +1,7 @@
11 //! Operations on a complete commit graph.
22 mod access;
33 mod init;
4+pub mod verify;
45
56 use crate::file::File;
67 use std::fmt;
git-commitgraph/src/graph/verify.rs+158−0
@@ -0,0 +1,158 @@
1+use crate::{
2+ file::{self, commit},
3+ graph, Graph, GENERATION_NUMBER_MAX,
4+};
5+use git_object::owned;
6+use std::cmp::{max, min};
7+use std::collections::BTreeMap;
8+use std::convert::TryInto;
9+use std::path::PathBuf;
10+
11+#[derive(thiserror::Error, Debug)]
12+pub enum Error<E: std::error::Error + 'static> {
13+ #[error("'{}' should have {expected} base graphs, but claims {actual} base graphs", .path.display())]
14+ BaseGraphCount { actual: u8, expected: u8, path: PathBuf },
15+ #[error("'{}' base graph at index {index} should have ID {expected} but is {actual}", .path.display())]
16+ BaseGraphId {
17+ actual: owned::Id,
18+ expected: owned::Id,
19+ index: u8,
20+ path: PathBuf,
21+ },
22+ #[error(transparent)]
23+ Commit(#[from] commit::Error),
24+ #[error("{}: {err}", .path.display())]
25+ File { err: file::verify::Error, path: PathBuf },
26+ #[error("Commit {id}'s generation should be {expected} but is {actual}")]
27+ Generation { actual: u32, expected: u32, id: owned::Id },
28+ #[error(
29+ "Commit {id} has parent position {parent_pos} that is out of range (should be in range 0-{max_valid_pos})"
30+ )]
31+ ParentOutOfRange {
32+ id: owned::Id,
33+ max_valid_pos: graph::Position,
34+ parent_pos: graph::Position,
35+ },
36+ #[error("{0}")]
37+ Processor(#[source] E),
38+ #[error("Commit-graph should be composed of at most 256 files but actually contains {0} files")]
39+ TooManyFiles(usize),
40+}
41+
42+#[derive(Clone, Debug, Eq, PartialEq)]
43+#[cfg_attr(feature = "serde1", derive(serde::Deserialize, serde::Serialize))]
44+pub struct Outcome {
45+ pub longest_path_length: Option<u32>,
46+ pub num_commits: u32,
47+ pub parent_counts: BTreeMap<u32, u32>,
48+}
49+
50+impl Graph {
51+ pub fn verify_integrity<E>(
52+ &self,
53+ mut processor: impl FnMut(&file::Commit<'_>) -> Result<(), E>,
54+ ) -> Result<Outcome, Error<E>>
55+ where
56+ E: std::error::Error + 'static,
57+ {
58+ if self.files.len() > 256 {
59+ // A file in a split chain can only have up to 255 base files.
60+ return Err(Error::TooManyFiles(self.files.len()));
61+ }
62+
63+ let mut stats = Outcome {
64+ longest_path_length: None,
65+ num_commits: 0,
66+ parent_counts: BTreeMap::new(),
67+ };
68+ let mut max_generation = 0u32;
69+
70+ // TODO: Detect duplicate commit IDs across different files. Not sure how to do this without
71+ // a separate loop, e.g. self.iter_sorted_ids().
72+
73+ let mut file_start_pos = graph::Position(0);
74+ for (file_index, file) in self.files.iter().enumerate() {
75+ if usize::from(file.base_graph_count()) != file_index {
76+ return Err(Error::BaseGraphCount {
77+ actual: file.base_graph_count(),
78+ expected: file_index
79+ .try_into()
80+ .expect("files.len() check to protect against this"),
81+ path: file.path().to_owned(),
82+ });
83+ }
84+
85+ for (base_graph_index, (expected, actual)) in self.files[..file_index]
86+ .iter()
87+ .map(|base_file| base_file.checksum())
88+ .zip(file.iter_base_graph_ids())
89+ .enumerate()
90+ {
91+ if actual != expected {
92+ return Err(Error::BaseGraphId {
93+ actual: actual.into(),
94+ expected: expected.into(),
95+ index: base_graph_index
96+ .try_into()
97+ .expect("files.len() check to protect against this"),
98+ path: file.path().to_owned(),
99+ });
100+ }
101+ }
102+
103+ let next_file_start_pos = graph::Position(file_start_pos.0 + file.num_commits());
104+ let file_stats = file
105+ .traverse(|commit| {
106+ let mut max_parent_generation = 0u32;
107+ for parent_pos in commit.iter_parents() {
108+ let parent_pos = parent_pos.map_err(Error::Commit)?;
109+ if parent_pos >= next_file_start_pos {
110+ return Err(Error::ParentOutOfRange {
111+ parent_pos,
112+ id: commit.id().into(),
113+ max_valid_pos: graph::Position(next_file_start_pos.0 - 1),
114+ });
115+ }
116+ let parent = self.commit_at(parent_pos);
117+ max_parent_generation = max(max_parent_generation, parent.generation());
118+ }
119+
120+ // If the max parent generation is GENERATION_NUMBER_MAX, then this commit's
121+ // generation should be GENERATION_NUMBER_MAX too.
122+ let expected_generation = min(max_parent_generation + 1, GENERATION_NUMBER_MAX);
123+ if commit.generation() != expected_generation {
124+ return Err(Error::Generation {
125+ actual: commit.generation(),
126+ expected: expected_generation,
127+ id: commit.id().into(),
128+ });
129+ }
130+
131+ processor(commit).map_err(Error::Processor)?;
132+
133+ Ok(())
134+ })
135+ .map_err(|e| match e {
136+ file::verify::EitherError::Internal(err) => Error::File {
137+ err,
138+ path: file.path().to_owned(),
139+ },
140+ file::verify::EitherError::Processor(e) => e,
141+ })?;
142+
143+ max_generation = max(max_generation, file_stats.max_generation);
144+ stats.num_commits += file_stats.num_commits;
145+ for (key, value) in file_stats.parent_counts.into_iter() {
146+ *stats.parent_counts.entry(key).or_insert(0) += value;
147+ }
148+ file_start_pos = next_file_start_pos;
149+ }
150+
151+ stats.longest_path_length = if max_generation < GENERATION_NUMBER_MAX {
152+ Some(max_generation.saturating_sub(1))
153+ } else {
154+ None
155+ };
156+ Ok(stats)
157+ }
158+}