[commitgraph] implement basic, low-level read API

This is just the first pass at ensuring we can read commit-graph files correctly. It does not support anything interesting like reachability, merge-base, or ahead-behind queries.

Conor Davis committed 6 years ago

Commit d1f0e9c1 parent 1b42b3153 files · +1429 −-23Browse files

Showing the changes to git-commitgraph/src (8 of 53 files). Show all

Unified · Split

8 files

git-commitgraph/src/graph/access.rs +83 −0
git-commitgraph/src/graph/init.rs +104 −0
git-commitgraph/src/graph/mod.rs +23 −0
git-commitgraph/src/graph_file/access.rs +128 −0
git-commitgraph/src/graph_file/commit_data.rs +255 −0
git-commitgraph/src/graph_file/init.rs +312 −0
git-commitgraph/src/graph_file/mod.rs +53 −0
git-commitgraph/src/lib.rs +9 −0

git-commitgraph/src/graph/access.rs+83−0

@@ -0,0 +1,83 @@
	1	+use crate::graph::GraphPosition;
	2	+use crate::graph_file::LexPosition;
	3	+use crate::{CommitData, Graph, GraphFile};
	4	+use git_object::borrowed;
	5	+
	6	+impl Graph {
	7	+ pub fn commit_at(&self, pos: GraphPosition) -> CommitData {
	8	+ let r = self.lookup_by_pos(pos);
	9	+ r.file.commit_at(r.lex_pos)
	10	+ }
	11	+
	12	+ pub fn commit_by_id(&self, id: borrowed::Id) -> Option<CommitData> {
	13	+ let r = self.lookup_by_id(id)?;
	14	+ Some(r.file.commit_at(r.lex_pos))
	15	+ }
	16	+
	17	+ pub fn id_at(&self, pos: GraphPosition) -> borrowed::Id {
	18	+ let r = self.lookup_by_pos(pos);
	19	+ r.file.id_at(r.lex_pos)
	20	+ }
	21	+
	22	+ /// Iterate over commits in unsorted order.
	23	+ pub fn iter_commits(&self) -> impl Iterator<Item = CommitData> {
	24	+ self.files.iter().flat_map(\|file\| file.iter_commits())
	25	+ }
	26	+
	27	+ /// Iterate over commit IDs in unsorted order.
	28	+ pub fn iter_ids(&self) -> impl Iterator<Item = borrowed::Id> {
	29	+ self.files.iter().flat_map(\|file\| file.iter_ids())
	30	+ }
	31	+
	32	+ pub fn lookup(&self, id: borrowed::Id) -> Option<GraphPosition> {
	33	+ Some(self.lookup_by_id(id)?.graph_pos)
	34	+ }
	35	+
	36	+ pub fn num_commits(&self) -> u32 {
	37	+ self.files.iter().map(\|f\| f.num_commits()).sum()
	38	+ }
	39	+}
	40	+
	41	+impl Graph {
	42	+ fn lookup_by_id(&self, id: borrowed::Id) -> Option<LookupByIdResult> {
	43	+ let mut current_file_start = 0;
	44	+ for file in self.files.iter() {
	45	+ if let Some(lex_pos) = file.lookup(id) {
	46	+ return Some(LookupByIdResult {
	47	+ file,
	48	+ lex_pos,
	49	+ graph_pos: GraphPosition(current_file_start + lex_pos.0),
	50	+ });
	51	+ }
	52	+ current_file_start += file.num_commits();
	53	+ }
	54	+ None
	55	+ }
	56	+
	57	+ fn lookup_by_pos(&self, pos: GraphPosition) -> LookupByPositionResult {
	58	+ let mut remaining = pos.0;
	59	+ for file in self.files.iter() {
	60	+ match remaining.checked_sub(file.num_commits()) {
	61	+ Some(v) => remaining = v,
	62	+ None => {
	63	+ return LookupByPositionResult {
	64	+ file,
	65	+ lex_pos: LexPosition(remaining),
	66	+ }
	67	+ }
	68	+ }
	69	+ }
	70	+ panic!("graph position too large: {}", pos.0);
	71	+ }
	72	+}
	73	+
	74	+struct LookupByIdResult<'a> {
	75	+ pub file: &'a GraphFile,
	76	+ pub graph_pos: GraphPosition,
	77	+ pub lex_pos: LexPosition,
	78	+}
	79	+
	80	+struct LookupByPositionResult<'a> {
	81	+ pub file: &'a GraphFile,
	82	+ pub lex_pos: LexPosition,
	83	+}

git-commitgraph/src/graph/init.rs+104−0

@@ -0,0 +1,104 @@
	1	+use crate::{graph_file, Graph, GraphFile, MAX_COMMITS};
	2	+use git_object::HashKind;
	3	+use quick_error::quick_error;
	4	+use std::io::{BufRead, BufReader};
	5	+use std::path::{Path, PathBuf};
	6	+
	7	+quick_error! {
	8	+ #[derive(Debug)]
	9	+ pub enum Error {
	10	+ Corrupt(msg: String, path: PathBuf) {
	11	+ display("{}", msg)
	12	+ }
	13	+ GraphFile(err: graph_file::Error, path: PathBuf) {
	14	+ display("{}", path.display())
	15	+ source(err)
	16	+ }
	17	+ HashVersionMismatch(path1: PathBuf, hash1: HashKind, path2: PathBuf, hash2: HashKind) {
	18	+ display(
	19	+ "Commit-graph files mismatch: '{}' uses hash {:?}, but '{}' uses hash {:?}",
	20	+ path1.display(),
	21	+ hash1,
	22	+ path2.display(),
	23	+ hash2,
	24	+ )
	25	+ }
	26	+ Io(err: std::io::Error, path: PathBuf) {
	27	+ display("Could not open commit-graph graph_file at '{}'", path.display())
	28	+ source(err)
	29	+ }
	30	+ TooManyCommits(num_commits: u64) {
	31	+ display(
	32	+ "Commit-graph files contain {} commits altogether, but only {} commits are allowed",
	33	+ num_commits,
	34	+ MAX_COMMITS,
	35	+ )
	36	+ }
	37	+ // This might actually be legal...
	38	+ VersionMismatch(path1: PathBuf, version1: graph_file::Kind, path2: PathBuf, version2: graph_file::Kind) {
	39	+ display(
	40	+ "Commit-graph files mismatch: '{}' is version {:?}, but '{}' is version {:?}",
	41	+ path1.display(),
	42	+ version1,
	43	+ path2.display(),
	44	+ version2,
	45	+ )
	46	+ }
	47	+ }
	48	+}
	49	+
	50	+impl Graph {
	51	+ pub fn from_object_dir(object_dir: impl AsRef<Path>) -> Result<Self, Error> {
	52	+ Self::from_single_file(object_dir.as_ref()).or_else(\|_\| Self::from_split_chain(object_dir.as_ref()))
	53	+ }
	54	+
	55	+ pub fn from_single_file(object_dir: impl AsRef<Path>) -> Result<Self, Error> {
	56	+ let single_graph_file = object_dir.as_ref().join("info").join("commit-graph");
	57	+ let file = GraphFile::at(&single_graph_file).map_err(\|e\| Error::GraphFile(e, single_graph_file.clone()))?;
	58	+ Self::new(vec![file])
	59	+ }
	60	+
	61	+ pub fn from_split_chain(object_dir: impl AsRef<Path>) -> Result<Self, Error> {
	62	+ let commit_graphs_dir = object_dir.as_ref().join("info").join("commit-graphs");
	63	+ let chain_file_path = commit_graphs_dir.join("commit-graph-chain");
	64	+ let chain_file = std::fs::File::open(&chain_file_path).map_err(\|e\| Error::Io(e, chain_file_path.clone()))?;
	65	+ let mut files: Vec<GraphFile> = Vec::new();
	66	+ for line in BufReader::new(chain_file).lines() {
	67	+ let line = line.map_err(\|e\| Error::Io(e, chain_file_path.clone()))?;
	68	+ let graph_filename = format!("graph-{}.graph", line);
	69	+ let graph_file_path = commit_graphs_dir.join(graph_filename);
	70	+ files.push(GraphFile::at(&graph_file_path).map_err(\|e\| Error::GraphFile(e, graph_file_path.clone()))?);
	71	+ }
	72	+ Self::new(files)
	73	+ }
	74	+
	75	+ pub fn new(files: Vec<GraphFile>) -> Result<Self, Error> {
	76	+ let num_commits: u64 = files.iter().map(\|f\| f.num_commits() as u64).sum();
	77	+ if num_commits > MAX_COMMITS as u64 {
	78	+ return Err(Error::TooManyCommits(num_commits));
	79	+ }
	80	+
	81	+ for window in files.windows(2) {
	82	+ let f1 = &window[0];
	83	+ let f2 = &window[1];
	84	+ if f1.kind() != f2.kind() {
	85	+ return Err(Error::VersionMismatch(
	86	+ f1.path().to_owned(),
	87	+ f1.kind(),
	88	+ f2.path().to_owned(),
	89	+ f2.kind(),
	90	+ ));
	91	+ }
	92	+ if f1.hash_kind() != f2.hash_kind() {
	93	+ return Err(Error::HashVersionMismatch(
	94	+ f1.path().to_owned(),
	95	+ f1.hash_kind(),
	96	+ f2.path().to_owned(),
	97	+ f2.hash_kind(),
	98	+ ));
	99	+ }
	100	+ }
	101	+
	102	+ Ok(Self { files })
	103	+ }
	104	+}

git-commitgraph/src/graph/mod.rs+23−0

@@ -0,0 +1,23 @@
	1	+mod access;
	2	+mod init;
	3	+
	4	+use crate::GraphFile;
	5	+use std::fmt::{Display, Formatter};
	6	+
	7	+/// A complete commit graph.
	8	+///
	9	+/// The data in the commit graph may come from a monolithic `objects/info/commit-graph` file, or it
	10	+/// may come from one or more `objects/info/commit-graphs/graph-*.graph` files. These files are
	11	+/// generated via `git commit-graph write ...` commands.
	12	+pub struct Graph {
	13	+ files: Vec<GraphFile>,
	14	+}
	15	+
	16	+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
	17	+pub struct GraphPosition(pub u32);
	18	+
	19	+impl Display for GraphPosition {
	20	+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
	21	+ self.0.fmt(f)
	22	+ }
	23	+}

git-commitgraph/src/graph_file/access.rs+128−0

@@ -0,0 +1,128 @@
	1	+use crate::graph_file::{Kind, LexPosition, COMMIT_DATA_ENTRY_SIZE};
	2	+use crate::{CommitData, GraphFile};
	3	+use git_object::{borrowed, HashKind, SHA1_SIZE};
	4	+use std::convert::{TryFrom, TryInto};
	5	+use std::fmt::{Debug, Formatter};
	6	+use std::path::Path;
	7	+
	8	+impl GraphFile {
	9	+ /// Returns the commit data for the commit located at the given lex position.
	10	+ ///
	11	+ /// `pos` must range from 0 to self.num_commits().
	12	+ ///
	13	+ /// # Panics
	14	+ ///
	15	+ /// Panics if `pos` is out of bounds.
	16	+ pub fn commit_at(&self, pos: LexPosition) -> CommitData {
	17	+ CommitData::new(self, pos)
	18	+ }
	19	+
	20	+ pub fn hash_kind(&self) -> HashKind {
	21	+ HashKind::Sha1
	22	+ }
	23	+
	24	+ // copied from git-odb/src/pack/index/access.rs
	25	+ /// Returns 20 bytes sha1 at the given index in our list of (sorted) sha1 hashes.
	26	+ /// The position ranges from 0 to self.num_commits()
	27	+ pub fn id_at(&self, pos: LexPosition) -> borrowed::Id {
	28	+ assert!(
	29	+ pos.0 < self.num_commits(),
	30	+ "expected lex position less than {}, got {}",
	31	+ self.num_commits(),
	32	+ pos.0
	33	+ );
	34	+ let pos: usize = pos
	35	+ .0
	36	+ .try_into()
	37	+ .expect("an architecture able to hold 32 bits of integer");
	38	+ let start = self.oid_lookup_offset + (pos * SHA1_SIZE);
	39	+ borrowed::Id::try_from(&self.data[start..start + SHA1_SIZE]).expect("20 bytes SHA1 to be alright")
	40	+ }
	41	+
	42	+ pub fn iter_base_graph_ids(&self) -> impl Iterator<Item = borrowed::Id> {
	43	+ let base_graphs_list = match self.base_graphs_list_offset {
	44	+ Some(v) => &self.data[v..v + (SHA1_SIZE * self.base_graph_count as usize)],
	45	+ None => &[],
	46	+ };
	47	+ base_graphs_list
	48	+ .chunks_exact(SHA1_SIZE)
	49	+ .map(\|bytes\| borrowed::Id::try_from(bytes).expect("20 bytes SHA1 to be alright"))
	50	+ }
	51	+
	52	+ pub fn iter_commits(&self) -> impl Iterator<Item = CommitData> {
	53	+ (0..self.num_commits()).map(move \|i\| self.commit_at(LexPosition(i)))
	54	+ }
	55	+
	56	+ pub fn iter_ids(&self) -> impl Iterator<Item = borrowed::Id> {
	57	+ (0..self.num_commits()).map(move \|i\| self.id_at(LexPosition(i)))
	58	+ }
	59	+
	60	+ pub fn kind(&self) -> Kind {
	61	+ Kind::V1
	62	+ }
	63	+
	64	+ // copied from git-odb/src/pack/index/access.rs
	65	+ pub fn lookup(&self, id: borrowed::Id) -> Option<LexPosition> {
	66	+ let first_byte = id.first_byte() as usize;
	67	+ let mut upper_bound = self.fan[first_byte];
	68	+ let mut lower_bound = if first_byte != 0 { self.fan[first_byte - 1] } else { 0 };
	69	+
	70	+ // Bisect using indices
	71	+ // TODO: Performance of V2 could possibly be better if we would be able to do a binary search
	72	+ // on 20 byte chunks directly, but doing so requires transmuting and that is unsafe, even though
	73	+ // it should not be if the bytes match up and the type has no destructor.
	74	+ while lower_bound < upper_bound {
	75	+ let mid = (lower_bound + upper_bound) / 2;
	76	+ let mid_sha = self.id_at(LexPosition(mid));
	77	+
	78	+ use std::cmp::Ordering::*;
	79	+ match id.cmp(&mid_sha) {
	80	+ Less => upper_bound = mid,
	81	+ Equal => return Some(LexPosition(mid)),
	82	+ Greater => lower_bound = mid + 1,
	83	+ }
	84	+ }
	85	+ None
	86	+ }
	87	+
	88	+ /// Returns the number of commits in this graph file.
	89	+ ///
	90	+ /// The maximum valid `LexPosition` that can be used with this file is one less than
	91	+ /// `num_commits()`.
	92	+ pub fn num_commits(&self) -> u32 {
	93	+ self.fan[255]
	94	+ }
	95	+
	96	+ pub fn path(&self) -> &Path {
	97	+ &self.path
	98	+ }
	99	+}
	100	+
	101	+impl GraphFile {
	102	+ /// Returns the byte slice for the given commit in this file's Commit Data (CDAT) chunk.
	103	+ pub(crate) fn commit_data_bytes(&self, pos: LexPosition) -> &[u8] {
	104	+ assert!(
	105	+ pos.0 < self.num_commits(),
	106	+ "expected lex position less than {}, got {}",
	107	+ self.num_commits(),
	108	+ pos.0
	109	+ );
	110	+ let pos: usize = pos
	111	+ .0
	112	+ .try_into()
	113	+ .expect("an architecture able to hold 32 bits of integer");
	114	+ let start = self.commit_data_offset + (pos * COMMIT_DATA_ENTRY_SIZE);
	115	+ &self.data[start..start + COMMIT_DATA_ENTRY_SIZE]
	116	+ }
	117	+
	118	+ /// Returns the byte slice for this file's entire Extra Edge List (EDGE) chunk.
	119	+ pub(crate) fn extra_edges_data(&self) -> Option<&[u8]> {
	120	+ Some(&self.data[self.extra_edges_list_range.clone()?])
	121	+ }
	122	+}
	123	+
	124	+impl Debug for GraphFile {
	125	+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
	126	+ write!(f, r#"GraphFile("{:?}")"#, self.path.display())
	127	+ }
	128	+}

git-commitgraph/src/graph_file/commit_data.rs+255−0

@@ -0,0 +1,255 @@
	1	+use crate::graph::GraphPosition;
	2	+use crate::graph_file::LexPosition;
	3	+use crate::GraphFile;
	4	+use byteorder::{BigEndian, ByteOrder};
	5	+use git_object::{borrowed, owned, SHA1_SIZE};
	6	+use quick_error::quick_error;
	7	+use std::convert::{TryFrom, TryInto};
	8	+use std::fmt::{Debug, Formatter};
	9	+use std::slice::Chunks;
	10	+
	11	+quick_error! {
	12	+ #[derive(Debug)]
	13	+ pub enum Error {
	14	+ ExtraEdgesListOverflow(commit: owned::Id) {
	15	+ display(
	16	+ "commit {}'s extra edges overflows the commit-graph file's extra edges list",
	17	+ commit,
	18	+ )
	19	+ }
	20	+ FirstParentIsExtraEdgeIndex(commit: owned::Id) {
	21	+ display(
	22	+ "commit {}'s first parent is an extra edge index, which is invalid",
	23	+ commit,
	24	+ )
	25	+ }
	26	+ MissingExtraEdgesList(commit: owned::Id) {
	27	+ display(
	28	+ "commit {} has extra edges, but commit-graph file has no extra edges list",
	29	+ commit,
	30	+ )
	31	+ }
	32	+ SecondParentWithoutFirstParent(commit: owned::Id) {
	33	+ display("commit {} has a second parent but not a first parent", commit)
	34	+ }
	35	+ }
	36	+}
	37	+
	38	+// Note that git's commit-graph-format.txt as of v2.28.0 gives an incorrect value 0x0700_0000 for
	39	+// NO_PARENT.
	40	+const NO_PARENT: u32 = 0x7000_0000;
	41	+const EXTENDED_EDGES_MASK: u32 = 0x8000_0000;
	42	+
	43	+pub struct CommitData<'a> {
	44	+ file: &'a GraphFile,
	45	+ lex_pos: LexPosition,
	46	+ // We can parse the below fields lazily if needed.
	47	+ commit_timestamp: u64,
	48	+ generation: u32,
	49	+ parent1: ParentEdge,
	50	+ parent2: ParentEdge,
	51	+ root_tree_id: borrowed::Id<'a>,
	52	+}
	53	+
	54	+impl<'a> CommitData<'a> {
	55	+ pub(crate) fn new(file: &'a GraphFile, pos: LexPosition) -> Self {
	56	+ let bytes = file.commit_data_bytes(pos);
	57	+ CommitData {
	58	+ file,
	59	+ lex_pos: pos,
	60	+ root_tree_id: borrowed::Id::try_from(&bytes[..SHA1_SIZE]).expect("20 bytes SHA1 to be alright"),
	61	+ parent1: ParentEdge::from_raw(BigEndian::read_u32(&bytes[SHA1_SIZE..SHA1_SIZE + 4])),
	62	+ parent2: ParentEdge::from_raw(BigEndian::read_u32(&bytes[SHA1_SIZE + 4..SHA1_SIZE + 8])),
	63	+ generation: BigEndian::read_u32(&bytes[SHA1_SIZE + 8..SHA1_SIZE + 12]) >> 2,
	64	+ commit_timestamp: BigEndian::read_u64(&bytes[SHA1_SIZE + 8..SHA1_SIZE + 16]) & 0x0003_ffff_ffff,
	65	+ }
	66	+ }
	67	+
	68	+ /// Returns the committer timestamp of this commit.
	69	+ ///
	70	+ /// The value is the number of seconds since 1970-01-01 00:00:00 UTC.
	71	+ pub fn committer_timestamp(&self) -> u64 {
	72	+ self.commit_timestamp
	73	+ }
	74	+
	75	+ /// Returns the generation number of this commit.
	76	+ ///
	77	+ /// Commits without parents have generation number 1. Commits with parents have a generation
	78	+ /// number that is the max of their parents' generation numbers + 1.
	79	+ pub fn generation(&self) -> u32 {
	80	+ self.generation
	81	+ }
	82	+
	83	+ pub fn iter_parent_indices(&'a self) -> impl Iterator<Item = Result<GraphPosition, Error>> + 'a {
	84	+ // I didn't find a combinator approach that a) was as strict as ParentIterator, b) supported
	85	+ // fuse-after-first-error behavior, and b) was significantly shorter or more understandable
	86	+ // than ParentIterator. So here we are.
	87	+ ParentIterator {
	88	+ commit_data: self,
	89	+ state: ParentIteratorState::First,
	90	+ }
	91	+ }
	92	+
	93	+ pub fn id(&self) -> borrowed::Id {
	94	+ self.file.id_at(self.lex_pos)
	95	+ }
	96	+
	97	+ pub fn parent1(&self) -> Result<Option<GraphPosition>, Error> {
	98	+ self.iter_parent_indices().next().transpose()
	99	+ }
	100	+
	101	+ pub fn root_tree_id(&self) -> borrowed::Id {
	102	+ self.root_tree_id
	103	+ }
	104	+}
	105	+
	106	+impl<'a> Debug for CommitData<'a> {
	107	+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
	108	+ write!(
	109	+ f,
	110	+ "CommitData {{ id: {}, lex_pos: {}, generation: {}, root_tree_oid: {}, parent1: {:?}, parent2: {:?} }}",
	111	+ self.id(),
	112	+ self.lex_pos,
	113	+ self.generation(),
	114	+ self.root_tree_id(),
	115	+ self.parent1,
	116	+ self.parent2,
	117	+ )
	118	+ }
	119	+}
	120	+
	121	+impl<'a> Eq for CommitData<'a> {}
	122	+
	123	+impl<'a> PartialEq for CommitData<'a> {
	124	+ fn eq(&self, other: &Self) -> bool {
	125	+ self.file as const GraphFile == other.file as const GraphFile && self.lex_pos == other.lex_pos
	126	+ }
	127	+}
	128	+
	129	+pub struct ParentIterator<'a> {
	130	+ commit_data: &'a CommitData<'a>,
	131	+ state: ParentIteratorState<'a>,
	132	+}
	133	+
	134	+impl<'a> Iterator for ParentIterator<'a> {
	135	+ type Item = Result<GraphPosition, Error>;
	136	+
	137	+ fn next(&mut self) -> Option<Self::Item> {
	138	+ let state = std::mem::replace(&mut self.state, ParentIteratorState::Exhausted);
	139	+ match state {
	140	+ ParentIteratorState::First => match self.commit_data.parent1 {
	141	+ ParentEdge::None => match self.commit_data.parent2 {
	142	+ ParentEdge::None => None,
	143	+ _ => Some(Err(Error::SecondParentWithoutFirstParent(self.commit_data.id().into()))),
	144	+ },
	145	+ ParentEdge::GraphPosition(pos) => {
	146	+ self.state = ParentIteratorState::Second;
	147	+ Some(Ok(pos))
	148	+ }
	149	+ ParentEdge::ExtraEdgeIndex(_) => {
	150	+ Some(Err(Error::FirstParentIsExtraEdgeIndex(self.commit_data.id().into())))
	151	+ }
	152	+ },
	153	+ ParentIteratorState::Second => match self.commit_data.parent2 {
	154	+ ParentEdge::None => None,
	155	+ ParentEdge::GraphPosition(pos) => Some(Ok(pos)),
	156	+ ParentEdge::ExtraEdgeIndex(extra_edge_index) => {
	157	+ if let Some(extra_edges_list) = self.commit_data.file.extra_edges_data() {
	158	+ let start_offset: usize = extra_edge_index
	159	+ .try_into()
	160	+ .expect("an architecture able to hold 32 bits of integer");
	161	+ let start_offset = start_offset
	162	+ .checked_mul(4)
	163	+ .expect("an extended edge index small enough to fit in usize");
	164	+ if let Some(tail) = extra_edges_list.get(start_offset..) {
	165	+ self.state = ParentIteratorState::Extra(tail.chunks(4));
	166	+ // This recursive call is what blocks me from replacing ParentIterator
	167	+ // with a std::iter::from_fn closure.
	168	+ self.next()
	169	+ } else {
	170	+ Some(Err(Error::ExtraEdgesListOverflow(self.commit_data.id().into())))
	171	+ }
	172	+ } else {
	173	+ Some(Err(Error::MissingExtraEdgesList(self.commit_data.id().into())))
	174	+ }
	175	+ }
	176	+ },
	177	+ ParentIteratorState::Extra(mut chunks) => {
	178	+ if let Some(chunk) = chunks.next() {
	179	+ let extra_edge = BigEndian::read_u32(chunk);
	180	+ match ExtraEdge::from_raw(extra_edge) {
	181	+ ExtraEdge::Internal(pos) => {
	182	+ self.state = ParentIteratorState::Extra(chunks);
	183	+ Some(Ok(pos))
	184	+ }
	185	+ ExtraEdge::Last(pos) => Some(Ok(pos)),
	186	+ }
	187	+ } else {
	188	+ Some(Err(Error::ExtraEdgesListOverflow(self.commit_data.id().into())))
	189	+ }
	190	+ }
	191	+ ParentIteratorState::Exhausted => None,
	192	+ }
	193	+ }
	194	+
	195	+ fn size_hint(&self) -> (usize, Option<usize>) {
	196	+ match (&self.state, self.commit_data.parent1, self.commit_data.parent2) {
	197	+ (ParentIteratorState::First, ParentEdge::None, ParentEdge::None) => (0, Some(0)),
	198	+ (ParentIteratorState::First, ParentEdge::None, _) => (1, Some(1)),
	199	+ (ParentIteratorState::First, ParentEdge::GraphPosition(_), ParentEdge::None) => (1, Some(1)),
	200	+ (ParentIteratorState::First, ParentEdge::GraphPosition(_), ParentEdge::GraphPosition(_)) => (2, Some(2)),
	201	+ (ParentIteratorState::First, ParentEdge::GraphPosition(_), ParentEdge::ExtraEdgeIndex(_)) => (3, None),
	202	+ (ParentIteratorState::First, ParentEdge::ExtraEdgeIndex(_), _) => (1, Some(1)),
	203	+ (ParentIteratorState::Second, _, ParentEdge::None) => (0, Some(0)),
	204	+ (ParentIteratorState::Second, _, ParentEdge::GraphPosition(_)) => (1, Some(1)),
	205	+ (ParentIteratorState::Second, _, ParentEdge::ExtraEdgeIndex(_)) => (2, None),
	206	+ (ParentIteratorState::Extra(_), _, _) => (1, None),
	207	+ (ParentIteratorState::Exhausted, _, _) => (0, Some(0)),
	208	+ }
	209	+ }
	210	+}
	211	+
	212	+#[derive(Debug)]
	213	+enum ParentIteratorState<'a> {
	214	+ First,
	215	+ Second,
	216	+ Extra(Chunks<'a, u8>),
	217	+ Exhausted,
	218	+}
	219	+
	220	+#[derive(Clone, Copy, Debug)]
	221	+enum ParentEdge {
	222	+ None,
	223	+ GraphPosition(GraphPosition),
	224	+ ExtraEdgeIndex(u32),
	225	+}
	226	+
	227	+impl ParentEdge {
	228	+ pub fn from_raw(raw: u32) -> ParentEdge {
	229	+ if raw == NO_PARENT {
	230	+ return ParentEdge::None;
	231	+ }
	232	+ if raw & EXTENDED_EDGES_MASK != 0 {
	233	+ ParentEdge::ExtraEdgeIndex(raw & !EXTENDED_EDGES_MASK)
	234	+ } else {
	235	+ ParentEdge::GraphPosition(GraphPosition(raw))
	236	+ }
	237	+ }
	238	+}
	239	+
	240	+const LAST_EXTENDED_EDGE_MASK: u32 = 0x8000_0000;
	241	+
	242	+enum ExtraEdge {
	243	+ Internal(GraphPosition),
	244	+ Last(GraphPosition),
	245	+}
	246	+
	247	+impl ExtraEdge {
	248	+ pub fn from_raw(raw: u32) -> Self {
	249	+ if raw & LAST_EXTENDED_EDGE_MASK != 0 {
	250	+ Self::Last(GraphPosition(raw & !LAST_EXTENDED_EDGE_MASK))
	251	+ } else {
	252	+ Self::Internal(GraphPosition(raw))
	253	+ }
	254	+ }
	255	+}

git-commitgraph/src/graph_file/init.rs+312−0

@@ -0,0 +1,312 @@
	1	+use crate::graph_file::{GraphFile, COMMIT_DATA_ENTRY_SIZE, FAN_LEN, SIGNATURE};
	2	+use bstr::ByteSlice;
	3	+use byteorder::{BigEndian, ByteOrder};
	4	+use filebuffer::FileBuffer;
	5	+use git_object::SHA1_SIZE;
	6	+use quick_error::quick_error;
	7	+use std::convert::{TryFrom, TryInto};
	8	+use std::ops::Range;
	9	+use std::path::Path;
	10	+
	11	+type ChunkId = [u8; 4];
	12	+
	13	+quick_error! {
	14	+ #[derive(Debug)]
	15	+ pub enum Error {
	16	+ BaseGraphMismatch(from_header: u8, from_chunk: u32) {
	17	+ display(
	18	+ "Commit-graph {} chunk contains {} base graphs, but commit-graph graph_file header claims {} base graphs",
	19	+ BASE_GRAPHS_LIST_CHUNK_ID.as_bstr(),
	20	+ from_chunk,
	21	+ from_header,
	22	+ )
	23	+ }
	24	+ CommitCountMismatch(chunk1_id: ChunkId, chunk1_commits: u32, chunk2_id: ChunkId, chunk2_commits: u32) {
	25	+ display(
	26	+ "Commit-graph {:?} chunk contains {} commits, but {:?} chunk contains {} commits",
	27	+ chunk1_id.as_bstr(),
	28	+ chunk1_commits,
	29	+ chunk2_id.as_bstr(),
	30	+ chunk2_commits,
	31	+ )
	32	+ }
	33	+ Corrupt(msg: String) {
	34	+ display("{}", msg)
	35	+ }
	36	+ DuplicateChunk(id: ChunkId) {
	37	+ display("Commit-graph graph_file contains multiple {:?} chunks", id.as_bstr())
	38	+ }
	39	+ // This error case is disabled, as git allows extra garbage in the extra edges list.
	40	+ // ExtraEdgesOverflow {
	41	+ // display("The last entry in commit-graph's extended edges list does is not marked as being terminal")
	42	+ // }
	43	+ InvalidChunkSize(id: ChunkId, msg: String) {
	44	+ display("Commit-graph chunk {:?} has invalid size: {}", id.as_bstr(), msg)
	45	+ }
	46	+ Io(err: std::io::Error, path: std::path::PathBuf) {
	47	+ display("Could not open commit-graph graph_file at '{}'", path.display())
	48	+ source(err)
	49	+ }
	50	+ MissingChunk(id: ChunkId) {
	51	+ display("Missing required chunk {:?}", id.as_bstr())
	52	+ }
	53	+ UnsupportedHashVersion(version: u8) {
	54	+ display("Commit-graph graph_file uses unsupported hash version: {}", version)
	55	+ }
	56	+ UnsupportedVersion(version: u8) {
	57	+ display("Unsupported commit-graph graph_file version: {}", version)
	58	+ }
	59	+ }
	60	+}
	61	+
	62	+const CHUNK_LOOKUP_SIZE: usize = 12;
	63	+const HEADER_LEN: usize = 8;
	64	+const MIN_FILE_SIZE: usize = HEADER_LEN + ((MIN_CHUNKS + 1) * CHUNK_LOOKUP_SIZE);
	65	+const OID_LOOKUP_ENTRY_SIZE: usize = SHA1_SIZE;
	66	+
	67	+// Required chunks: OIDF, OIDL, CDAT
	68	+const MIN_CHUNKS: usize = 3;
	69	+const BASE_GRAPHS_LIST_CHUNK_ID: ChunkId = *b"BASE";
	70	+const COMMIT_DATA_CHUNK_ID: ChunkId = *b"CDAT";
	71	+const EXTENDED_EDGES_LIST_CHUNK_ID: ChunkId = *b"EDGE";
	72	+const OID_FAN_CHUNK_ID: ChunkId = *b"OIDF";
	73	+const OID_LOOKUP_CHUNK_ID: ChunkId = *b"OIDL";
	74	+const SENTINEL_CHUNK_ID: ChunkId = [0u8; 4];
	75	+
	76	+impl GraphFile {
	77	+ pub fn at(path: impl AsRef<Path>) -> Result<GraphFile, Error> {
	78	+ Self::try_from(path.as_ref())
	79	+ }
	80	+}
	81	+
	82	+impl TryFrom<&Path> for GraphFile {
	83	+ type Error = Error;
	84	+
	85	+ fn try_from(path: &Path) -> Result<Self, Self::Error> {
	86	+ let data = FileBuffer::open(path).map_err(\|e\| Error::Io(e, path.to_owned()))?;
	87	+ let data_size = data.len();
	88	+ if data_size < MIN_FILE_SIZE {
	89	+ return Err(Error::Corrupt(
	90	+ "Commit-graph graph_file too small even for an empty graph".to_owned(),
	91	+ ));
	92	+ }
	93	+
	94	+ let mut ofs = 0;
	95	+ if &data[ofs..ofs + SIGNATURE.len()] != SIGNATURE {
	96	+ return Err(Error::Corrupt(
	97	+ "Commit-graph graph_file does not start with expected signature".to_owned(),
	98	+ ));
	99	+ }
	100	+ ofs += SIGNATURE.len();
	101	+
	102	+ match data[ofs] {
	103	+ 1 => (),
	104	+ x => {
	105	+ return Err(Error::UnsupportedVersion(x));
	106	+ }
	107	+ };
	108	+ ofs += 1;
	109	+
	110	+ match data[ofs] {
	111	+ 1 => (),
	112	+ x => {
	113	+ return Err(Error::UnsupportedHashVersion(x));
	114	+ }
	115	+ };
	116	+ ofs += 1;
	117	+
	118	+ let chunk_count = data[ofs];
	119	+ // Can assert chunk_count >= MIN_CHUNKS here, but later OIDF+OIDL+CDAT presence checks make
	120	+ // it redundant.
	121	+ ofs += 1;
	122	+
	123	+ let base_graph_count = data[ofs];
	124	+ ofs += 1;
	125	+
	126	+ let chunk_lookup_end = ofs + ((chunk_count as usize + 1) * CHUNK_LOOKUP_SIZE);
	127	+ if chunk_lookup_end > data_size {
	128	+ return Err(Error::Corrupt(format!(
	129	+ "Commit-graph graph_file is too small to hold {} chunks",
	130	+ chunk_count
	131	+ )));
	132	+ }
	133	+
	134	+ let mut base_graphs_list_offset: Option<usize> = None;
	135	+ let mut commit_data_offset: Option<usize> = None;
	136	+ let mut commit_data_count = 0u32;
	137	+ let mut extended_edges_list_range: Option<Range<usize>> = None;
	138	+ let mut fan_offset: Option<usize> = None;
	139	+ let mut oid_lookup_offset: Option<usize> = None;
	140	+ let mut oid_lookup_count = 0u32;
	141	+
	142	+ let mut chunk_id: ChunkId = data[ofs..ofs + 4].try_into().expect("ChunkId to accept 4 bytes");
	143	+ ofs += 4;
	144	+
	145	+ let mut chunk_offset: usize = BigEndian::read_u64(&data[ofs..ofs + 8])
	146	+ .try_into()
	147	+ .expect("an offset small enough to fit a usize");
	148	+ if chunk_offset < chunk_lookup_end {
	149	+ return Err(Error::Corrupt(format!(
	150	+ "Commit-graph chunk 0 has invalid offset {} (must be at least {})",
	151	+ chunk_offset, chunk_lookup_end
	152	+ )));
	153	+ }
	154	+ ofs += 8;
	155	+
	156	+ for _ in 0..chunk_count {
	157	+ let next_chunk_id: ChunkId = data[ofs..ofs + 4].try_into().expect("ChunkId to accept 4 bytes");
	158	+ ofs += 4;
	159	+
	160	+ let next_chunk_offset: usize = BigEndian::read_u64(&data[ofs..ofs + 8])
	161	+ .try_into()
	162	+ .expect("an offset small enough to fit a usize");
	163	+ ofs += 8;
	164	+
	165	+ let chunk_size: usize = next_chunk_offset
	166	+ .checked_sub(chunk_offset)
	167	+ .ok_or_else(\|\| Error::InvalidChunkSize(chunk_id, "size is negative".to_string()))?;
	168	+ if next_chunk_offset >= data_size {
	169	+ return Err(Error::InvalidChunkSize(
	170	+ chunk_id,
	171	+ "chunk extends beyond end of graph_file".to_string(),
	172	+ ));
	173	+ }
	174	+
	175	+ match chunk_id {
	176	+ BASE_GRAPHS_LIST_CHUNK_ID => {
	177	+ if base_graphs_list_offset.is_some() {
	178	+ return Err(Error::DuplicateChunk(chunk_id));
	179	+ }
	180	+ if chunk_size % SHA1_SIZE != 0 {
	181	+ return Err(Error::InvalidChunkSize(
	182	+ chunk_id,
	183	+ format!("chunk size {} is not a multiple of {}", chunk_size, SHA1_SIZE),
	184	+ ));
	185	+ }
	186	+ let chunk_base_graph_count = (chunk_size / SHA1_SIZE) as u32;
	187	+ if chunk_base_graph_count != base_graph_count as u32 {
	188	+ return Err(Error::BaseGraphMismatch(base_graph_count, chunk_base_graph_count));
	189	+ }
	190	+ base_graphs_list_offset = Some(chunk_offset);
	191	+ }
	192	+ COMMIT_DATA_CHUNK_ID => {
	193	+ if commit_data_offset.is_some() {
	194	+ return Err(Error::DuplicateChunk(chunk_id));
	195	+ }
	196	+ if chunk_size % COMMIT_DATA_ENTRY_SIZE != 0 {
	197	+ return Err(Error::InvalidChunkSize(
	198	+ chunk_id,
	199	+ format!(
	200	+ "chunk size {} is not a multiple of {}",
	201	+ chunk_size, COMMIT_DATA_ENTRY_SIZE
	202	+ ),
	203	+ ));
	204	+ }
	205	+ commit_data_offset = Some(chunk_offset);
	206	+ commit_data_count = (chunk_size / COMMIT_DATA_ENTRY_SIZE) as u32;
	207	+ }
	208	+ EXTENDED_EDGES_LIST_CHUNK_ID => {
	209	+ if extended_edges_list_range.is_some() {
	210	+ return Err(Error::DuplicateChunk(chunk_id));
	211	+ }
	212	+
	213	+ extended_edges_list_range = Some(Range {
	214	+ start: chunk_offset,
	215	+ end: next_chunk_offset,
	216	+ })
	217	+ }
	218	+ OID_FAN_CHUNK_ID => {
	219	+ if fan_offset.is_some() {
	220	+ return Err(Error::DuplicateChunk(chunk_id));
	221	+ }
	222	+ let expected_size = 4 * FAN_LEN;
	223	+ if chunk_size != expected_size {
	224	+ return Err(Error::InvalidChunkSize(
	225	+ chunk_id,
	226	+ format!("expected chunk length {}, got {}", expected_size, chunk_size),
	227	+ ));
	228	+ }
	229	+ fan_offset = Some(chunk_offset);
	230	+ }
	231	+ OID_LOOKUP_CHUNK_ID => {
	232	+ if oid_lookup_offset.is_some() {
	233	+ return Err(Error::DuplicateChunk(chunk_id));
	234	+ }
	235	+ if chunk_size % OID_LOOKUP_ENTRY_SIZE != 0 {
	236	+ return Err(Error::InvalidChunkSize(
	237	+ chunk_id,
	238	+ format!(
	239	+ "chunk size {} is not a multiple of {}",
	240	+ chunk_size, OID_LOOKUP_ENTRY_SIZE
	241	+ ),
	242	+ ));
	243	+ }
	244	+ oid_lookup_offset = Some(chunk_offset);
	245	+ oid_lookup_count = (chunk_size / OID_LOOKUP_ENTRY_SIZE) as u32;
	246	+ // git allows extra garbage in the extra edges list chunk?
	247	+ // if oid_lookup_count > 0 {
	248	+ // let last_edge = &data[next_chunk_offset - 4..next_chunk_offset];
	249	+ // let last_edge = BigEndian::read_u32(last_edge);
	250	+ // if let ExtraEdge::Internal(_) = ExtraEdge::from_raw(last_edge) {
	251	+ // return Err(Error::ExtraEdgesListOverflow);
	252	+ // }
	253	+ // }
	254	+ }
	255	+ _ => {}
	256	+ }
	257	+
	258	+ chunk_id = next_chunk_id;
	259	+ chunk_offset = next_chunk_offset;
	260	+ }
	261	+ if chunk_id != SENTINEL_CHUNK_ID {
	262	+ return Err(Error::Corrupt(format!(
	263	+ "Commit-graph graph_file has invalid last chunk ID: {:?}",
	264	+ chunk_id.as_bstr()
	265	+ )));
	266	+ }
	267	+
	268	+ let fan_offset = fan_offset.ok_or_else(\|\| Error::MissingChunk(OID_FAN_CHUNK_ID))?;
	269	+ let oid_lookup_offset = oid_lookup_offset.ok_or_else(\|\| Error::MissingChunk(OID_LOOKUP_CHUNK_ID))?;
	270	+ let commit_data_offset = commit_data_offset.ok_or_else(\|\| Error::MissingChunk(COMMIT_DATA_CHUNK_ID))?;
	271	+ if base_graph_count > 0 && base_graphs_list_offset == None {
	272	+ return Err(Error::MissingChunk(BASE_GRAPHS_LIST_CHUNK_ID));
	273	+ }
	274	+
	275	+ let (fan, _) = read_fan(&data[fan_offset..]);
	276	+ if oid_lookup_count != fan[255] {
	277	+ return Err(Error::CommitCountMismatch(
	278	+ OID_FAN_CHUNK_ID,
	279	+ fan[255],
	280	+ OID_LOOKUP_CHUNK_ID,
	281	+ oid_lookup_count,
	282	+ ));
	283	+ }
	284	+ if commit_data_count != fan[255] {
	285	+ return Err(Error::CommitCountMismatch(
	286	+ OID_FAN_CHUNK_ID,
	287	+ fan[255],
	288	+ COMMIT_DATA_CHUNK_ID,
	289	+ commit_data_count,
	290	+ ));
	291	+ }
	292	+ Ok(GraphFile {
	293	+ base_graph_count,
	294	+ base_graphs_list_offset,
	295	+ commit_data_offset,
	296	+ data,
	297	+ extra_edges_list_range: extended_edges_list_range,
	298	+ fan,
	299	+ oid_lookup_offset,
	300	+ path: path.to_owned(),
	301	+ })
	302	+ }
	303	+}
	304	+
	305	+// Copied from git-odb/pack/index/init.rs
	306	+fn read_fan(d: &[u8]) -> ([u32; FAN_LEN], usize) {
	307	+ let mut fan = [0; FAN_LEN];
	308	+ for (c, f) in d.chunks(4).zip(fan.iter_mut()) {
	309	+ *f = BigEndian::read_u32(c);
	310	+ }
	311	+ (fan, FAN_LEN * 4)
	312	+}

git-commitgraph/src/graph_file/mod.rs+53−0

@@ -0,0 +1,53 @@
	1	+use filebuffer::FileBuffer;
	2	+use git_object::SHA1_SIZE;
	3	+use std::ops::Range;
	4	+use std::path::PathBuf;
	5	+
	6	+mod access;
	7	+pub mod commit_data;
	8	+mod init;
	9	+
	10	+pub use init::Error;
	11	+use std::fmt::{Display, Formatter};
	12	+
	13	+const COMMIT_DATA_ENTRY_SIZE: usize = SHA1_SIZE + 16;
	14	+const FAN_LEN: usize = 256;
	15	+const SIGNATURE: &[u8] = b"CGPH";
	16	+
	17	+/// A single commit-graph file.
	18	+///
	19	+/// All operations on a `GraphFile` are local to that graph file. Since a commit graph can span
	20	+/// multiple files, all interesting graph operations belong on `Graph`.
	21	+pub struct GraphFile {
	22	+ base_graph_count: u8,
	23	+ base_graphs_list_offset: Option<usize>,
	24	+ commit_data_offset: usize,
	25	+ data: FileBuffer,
	26	+ extra_edges_list_range: Option<Range<usize>>,
	27	+ fan: [u32; FAN_LEN],
	28	+ oid_lookup_offset: usize,
	29	+ path: PathBuf,
	30	+}
	31	+
	32	+/// The position of a given commit within a graph file, starting at 0.
	33	+///
	34	+/// Commits within a graph file are sorted in lexicographical order by OID; a commit's lex position
	35	+/// is its position in this ordering. If a commit graph spans multiple files, each file's commits
	36	+/// start at lex position 0, so lex position is unique across a single file but is not unique across
	37	+/// the whole commit graph. Each commit also has a graph position (`GraphPosition`), which is unique
	38	+/// across the whole commit graph. In order to avoid accidentally mixing lex positions with graph
	39	+/// positions, distinct types are used for each.
	40	+#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
	41	+pub struct LexPosition(pub u32);
	42	+
	43	+impl Display for LexPosition {
	44	+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
	45	+ self.0.fmt(f)
	46	+ }
	47	+}
	48	+
	49	+/// The version of a commit graph file.
	50	+#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
	51	+pub enum Kind {
	52	+ V1,
	53	+}

git-commitgraph/src/lib.rs+9−0

@@ -1 +1,10 @@
1	1	#![forbid(unsafe_code)]
	2	+
	3	+pub mod graph;
	4	+pub mod graph_file;
	5	+
	6	+pub use graph::{Graph, GraphPosition};
	7	+pub use graph_file::commit_data::CommitData;
	8	+pub use graph_file::{GraphFile, LexPosition};
	9	+
	10	+pub const MAX_COMMITS: u32 = (1 << 30) + (1 << 29) + (1 << 28) - 1;