History

[commitgraph] Implement basic commit-graph file verification.

Missing features: 1. It operates on commit-graph files only, so it doesn't verify that commit-graph data matches `git-odb` data. 2. No progress reporting or parallelization. This shouldn't be needed until until we need to check against `git-odb` data. Example output for Linux repo: ``` $ time ./target/release/gixp commit-graph-verify -s ~/src/linux/.git/objects/info number of commits with the given number of parents 0: 4 1: 878988 2: 67800 3: 652 4: 408 5: 382 6: 454 7: 95 8: 65 9: 47 10: 25 11: 26 12: 14 13: 4 14: 3 18: 1 19: 1 20: 1 21: 1 24: 1 27: 1 30: 1 32: 1 66: 1 ->: 948976 longest path length between two commits: 160521 real 0m0.196s user 0m0.180s sys 0m0.016s ```

Conor Davis committed 6 years ago

Commit 25711131 parent 701f33c19 files · +459 −21Browse files

Showing the changes to git-commitgraph/src/graph (3 of 19 files). Show all

Unified · Split

3 files
  1. git-commitgraph/src/graph/access.rs +7 −5
  2. git-commitgraph/src/graph/mod.rs +1 −0
  3. git-commitgraph/src/graph/verify.rs +158 −0
git-commitgraph/src/graph/access.rs+7−5
@@ -42,7 +42,7 @@ impl Graph { · 41 unmodified lines
42 42
43/// Access fundamentals43/// Access fundamentals
44impl Graph {44impl Graph {
45 fn lookup_by_id(&self, id: borrowed::Id<'_>) -> Option<LookupByIdResult<'_>> {45 pub(crate) fn lookup_by_id(&self, id: borrowed::Id<'_>) -> Option<LookupByIdResult<'_>> {
46 let mut current_file_start = 0;46 let mut current_file_start = 0;
47 for file in &self.files {47 for file in &self.files {
48 if let Some(lex_pos) = file.lookup(id) {48 if let Some(lex_pos) = file.lookup(id) {
@@ -57,14 +57,15 @@ impl Graph { · 8 unmodified lines
57 None57 None
58 }58 }
59 59
60 fn lookup_by_pos(&self, pos: graph::Position) -> LookupByPositionResult<'_> {60 pub(crate) fn lookup_by_pos(&self, pos: graph::Position) -> LookupByPositionResult<'_> {
61 let mut remaining = pos.0;61 let mut remaining = pos.0;
62 for file in &self.files {62 for (file_index, file) in self.files.iter().enumerate() {
63 match remaining.checked_sub(file.num_commits()) {63 match remaining.checked_sub(file.num_commits()) {
64 Some(v) => remaining = v,64 Some(v) => remaining = v,
65 None => {65 None => {
66 return LookupByPositionResult {66 return LookupByPositionResult {
67 file,67 file,
68 file_index,
68 pos: file::Position(remaining),69 pos: file::Position(remaining),
69 }70 }
70 }71 }
@@ -75,14 +76,15 @@ impl Graph { · 4 unmodified lines
75}76}
76 77
77#[derive(Clone)]78#[derive(Clone)]
78struct LookupByIdResult<'a> {79pub(crate) struct LookupByIdResult<'a> {
79 pub file: &'a File,80 pub file: &'a File,
80 pub graph_pos: graph::Position,81 pub graph_pos: graph::Position,
81 pub file_pos: file::Position,82 pub file_pos: file::Position,
82}83}
83 84
84#[derive(Clone)]85#[derive(Clone)]
85struct LookupByPositionResult<'a> {86pub(crate) struct LookupByPositionResult<'a> {
86 pub file: &'a File,87 pub file: &'a File,
88 pub file_index: usize,
87 pub pos: file::Position,89 pub pos: file::Position,
88}90}
git-commitgraph/src/graph/mod.rs+1−0
@@ -1,6 +1,7 @@
1//! Operations on a complete commit graph.1//! Operations on a complete commit graph.
2mod access;2mod access;
3mod init;3mod init;
4pub mod verify;
4 5
5use crate::file::File;6use crate::file::File;
6use std::fmt;7use std::fmt;
git-commitgraph/src/graph/verify.rs+158−0
@@ -0,0 +1,158 @@
1use crate::{
2 file::{self, commit},
3 graph, Graph, GENERATION_NUMBER_MAX,
4};
5use git_object::owned;
6use std::cmp::{max, min};
7use std::collections::BTreeMap;
8use std::convert::TryInto;
9use std::path::PathBuf;
10
11#[derive(thiserror::Error, Debug)]
12pub enum Error<E: std::error::Error + 'static> {
13 #[error("'{}' should have {expected} base graphs, but claims {actual} base graphs", .path.display())]
14 BaseGraphCount { actual: u8, expected: u8, path: PathBuf },
15 #[error("'{}' base graph at index {index} should have ID {expected} but is {actual}", .path.display())]
16 BaseGraphId {
17 actual: owned::Id,
18 expected: owned::Id,
19 index: u8,
20 path: PathBuf,
21 },
22 #[error(transparent)]
23 Commit(#[from] commit::Error),
24 #[error("{}: {err}", .path.display())]
25 File { err: file::verify::Error, path: PathBuf },
26 #[error("Commit {id}'s generation should be {expected} but is {actual}")]
27 Generation { actual: u32, expected: u32, id: owned::Id },
28 #[error(
29 "Commit {id} has parent position {parent_pos} that is out of range (should be in range 0-{max_valid_pos})"
30 )]
31 ParentOutOfRange {
32 id: owned::Id,
33 max_valid_pos: graph::Position,
34 parent_pos: graph::Position,
35 },
36 #[error("{0}")]
37 Processor(#[source] E),
38 #[error("Commit-graph should be composed of at most 256 files but actually contains {0} files")]
39 TooManyFiles(usize),
40}
41
42#[derive(Clone, Debug, Eq, PartialEq)]
43#[cfg_attr(feature = "serde1", derive(serde::Deserialize, serde::Serialize))]
44pub struct Outcome {
45 pub longest_path_length: Option<u32>,
46 pub num_commits: u32,
47 pub parent_counts: BTreeMap<u32, u32>,
48}
49
50impl Graph {
51 pub fn verify_integrity<E>(
52 &self,
53 mut processor: impl FnMut(&file::Commit<'_>) -> Result<(), E>,
54 ) -> Result<Outcome, Error<E>>
55 where
56 E: std::error::Error + 'static,
57 {
58 if self.files.len() > 256 {
59 // A file in a split chain can only have up to 255 base files.
60 return Err(Error::TooManyFiles(self.files.len()));
61 }
62
63 let mut stats = Outcome {
64 longest_path_length: None,
65 num_commits: 0,
66 parent_counts: BTreeMap::new(),
67 };
68 let mut max_generation = 0u32;
69
70 // TODO: Detect duplicate commit IDs across different files. Not sure how to do this without
71 // a separate loop, e.g. self.iter_sorted_ids().
72
73 let mut file_start_pos = graph::Position(0);
74 for (file_index, file) in self.files.iter().enumerate() {
75 if usize::from(file.base_graph_count()) != file_index {
76 return Err(Error::BaseGraphCount {
77 actual: file.base_graph_count(),
78 expected: file_index
79 .try_into()
80 .expect("files.len() check to protect against this"),
81 path: file.path().to_owned(),
82 });
83 }
84
85 for (base_graph_index, (expected, actual)) in self.files[..file_index]
86 .iter()
87 .map(|base_file| base_file.checksum())
88 .zip(file.iter_base_graph_ids())
89 .enumerate()
90 {
91 if actual != expected {
92 return Err(Error::BaseGraphId {
93 actual: actual.into(),
94 expected: expected.into(),
95 index: base_graph_index
96 .try_into()
97 .expect("files.len() check to protect against this"),
98 path: file.path().to_owned(),
99 });
100 }
101 }
102
103 let next_file_start_pos = graph::Position(file_start_pos.0 + file.num_commits());
104 let file_stats = file
105 .traverse(|commit| {
106 let mut max_parent_generation = 0u32;
107 for parent_pos in commit.iter_parents() {
108 let parent_pos = parent_pos.map_err(Error::Commit)?;
109 if parent_pos >= next_file_start_pos {
110 return Err(Error::ParentOutOfRange {
111 parent_pos,
112 id: commit.id().into(),
113 max_valid_pos: graph::Position(next_file_start_pos.0 - 1),
114 });
115 }
116 let parent = self.commit_at(parent_pos);
117 max_parent_generation = max(max_parent_generation, parent.generation());
118 }
119
120 // If the max parent generation is GENERATION_NUMBER_MAX, then this commit's
121 // generation should be GENERATION_NUMBER_MAX too.
122 let expected_generation = min(max_parent_generation + 1, GENERATION_NUMBER_MAX);
123 if commit.generation() != expected_generation {
124 return Err(Error::Generation {
125 actual: commit.generation(),
126 expected: expected_generation,
127 id: commit.id().into(),
128 });
129 }
130
131 processor(commit).map_err(Error::Processor)?;
132
133 Ok(())
134 })
135 .map_err(|e| match e {
136 file::verify::EitherError::Internal(err) => Error::File {
137 err,
138 path: file.path().to_owned(),
139 },
140 file::verify::EitherError::Processor(e) => e,
141 })?;
142
143 max_generation = max(max_generation, file_stats.max_generation);
144 stats.num_commits += file_stats.num_commits;
145 for (key, value) in file_stats.parent_counts.into_iter() {
146 *stats.parent_counts.entry(key).or_insert(0) += value;
147 }
148 file_start_pos = next_file_start_pos;
149 }
150
151 stats.longest_path_length = if max_generation < GENERATION_NUMBER_MAX {
152 Some(max_generation.saturating_sub(1))
153 } else {
154 None
155 };
156 Ok(stats)
157 }
158}