feat: `ein tool query` - a git analytics engine.

A tool to build and efficiently maintain a database of information contained in a git repository, preferably the kind of information that is expensive to obtain, in order to facilitate queries that would be prohibitive without an accelerating data structure.

Sebastian Thiel committed 3 years ago

Commit f8cc6231 parent 1d3d22d19 files · +1062 −130Browse files

Unified · Split

19 files

Cargo.lock +62 −2
Cargo.toml +6 −3
gitoxide-core/Cargo.toml +7 −3
gitoxide-core/src/hours/mod.rs +3 −4
gitoxide-core/src/lib.rs +2 −0
gitoxide-core/src/query/db.rs +68 −0
gitoxide-core/src/query/engine/command.rs +183 −0
gitoxide-core/src/query/engine/mod.rs +11 −0
gitoxide-core/src/query/engine/update.rs +501 −0
gitoxide-core/src/query/mod.rs +23 −0
src/plumbing/options/mod.rs +1 −24
src/porcelain/main.rs +46 −9
src/porcelain/options.rs +127 −81
src/shared.rs +16 −1
tests/journey/ein.sh +2 −0
tests/snapshots/panic-behaviour/expected-failure +1 −1
tests/snapshots/panic-behaviour/expected-failure-in-thread +1 −1
tests/snapshots/panic-behaviour/expected-failure-in-thread-with-progress +1 −1
tests/snapshots/porcelain/tool/no-args-failure +1 −0

Cargo.lock+62−2

@@ -8,6 +8,17 @@ version = "1.0.2" · 7 unmodified lines
8	8	source = "registry+https://github.com/rust-lang/crates.io-index"
9	9	checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
10	10
	11	+[[package]]
	12	+name = "ahash"
	13	+version = "0.7.6"
	14	+source = "registry+https://github.com/rust-lang/crates.io-index"
	15	+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
	16	+dependencies = [
	17	+ "getrandom",
	18	+ "once_cell",
	19	+ "version_check",
	20	+]
	21	+
11	22	[[package]]
12	23	name = "ahash"
13	24	version = "0.8.3"
@@ -988,6 +999,18 @@ dependencies = [ · 974 unmodified lines
988	999	"regex",
989	1000	]
990	1001
	1002	+[[package]]
	1003	+name = "fallible-iterator"
	1004	+version = "0.2.0"
	1005	+source = "registry+https://github.com/rust-lang/crates.io-index"
	1006	+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
	1007	+
	1008	+[[package]]
	1009	+name = "fallible-streaming-iterator"
	1010	+version = "0.1.9"
	1011	+source = "registry+https://github.com/rust-lang/crates.io-index"
	1012	+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
	1013	+
991	1014	[[package]]
992	1015	name = "fastrand"
993	1016	version = "1.9.0"
@@ -1256,7 +1279,7 @@ dependencies = [ · 262 unmodified lines
1256	1279	"itertools",
1257	1280	"jwalk",
1258	1281	"mime_guess",
1259		− "num_cpus",
	1282	+ "rusqlite",
1260	1283	"serde",
1261	1284	"serde_json",
1262	1285	"tempfile",
@@ -2370,6 +2393,9 @@ name = "hashbrown" · 1107 unmodified lines
2370	2393	version = "0.12.3"
2371	2394	source = "registry+https://github.com/rust-lang/crates.io-index"
2372	2395	checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
	2396	+dependencies = [
	2397	+ "ahash 0.7.6",
	2398	+]
2373	2399
2374	2400	[[package]]
2375	2401	name = "hashbrown"
@@ -2377,6 +2403,15 @@ version = "0.13.2" · 1 unmodified line
2377	2403	source = "registry+https://github.com/rust-lang/crates.io-index"
2378	2404	checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
2379	2405
	2406	+[[package]]
	2407	+name = "hashlink"
	2408	+version = "0.8.1"
	2409	+source = "registry+https://github.com/rust-lang/crates.io-index"
	2410	+checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"
	2411	+dependencies = [
	2412	+ "hashbrown 0.12.3",
	2413	+]
	2414	+
2380	2415	[[package]]
2381	2416	name = "heck"
2382	2417	version = "0.4.1"
@@ -2559,7 +2594,7 @@ version = "0.1.5" · 176 unmodified lines
2559	2594	source = "registry+https://github.com/rust-lang/crates.io-index"
2560	2595	checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8"
2561	2596	dependencies = [
2562		− "ahash",
	2597	+ "ahash 0.8.3",
2563	2598	"hashbrown 0.12.3",
2564	2599	]
2565	2600
@@ -2728,6 +2763,17 @@ dependencies = [ · 162 unmodified lines
2728	2763	"pkg-config",
2729	2764	]
2730	2765
	2766	+[[package]]
	2767	+name = "libsqlite3-sys"
	2768	+version = "0.25.2"
	2769	+source = "registry+https://github.com/rust-lang/crates.io-index"
	2770	+checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
	2771	+dependencies = [
	2772	+ "cc",
	2773	+ "pkg-config",
	2774	+ "vcpkg",
	2775	+]
	2776	+
2731	2777	[[package]]
2732	2778	name = "libz-ng-sys"
2733	2779	version = "1.1.8"
@@ -3517,6 +3563,20 @@ dependencies = [ · 783 unmodified lines
3517	3563	"winapi",
3518	3564	]
3519	3565
	3566	+[[package]]
	3567	+name = "rusqlite"
	3568	+version = "0.28.0"
	3569	+source = "registry+https://github.com/rust-lang/crates.io-index"
	3570	+checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
	3571	+dependencies = [
	3572	+ "bitflags",
	3573	+ "fallible-iterator",
	3574	+ "fallible-streaming-iterator",
	3575	+ "hashlink",
	3576	+ "libsqlite3-sys",
	3577	+ "smallvec",
	3578	+]
	3579	+
3520	3580	[[package]]
3521	3581	name = "rustc-hash"
3522	3582	version = "1.1.0"

Cargo.toml+6−3

@@ -43,14 +43,14 @@ prodash-render-line-crossterm = ["prodash-render-line", "prodash/render-line-cro · 42 unmodified lines
43	43	#! These combine common choices of the above features to represent typical builds
44	44
45	45	## fast + prodash-render-tui-crossterm + prodash-render-line-crossterm + http + gitoxide-core-tools + client-networking
46		−max = ["fast", "pretty-cli", "http-client-curl", "gitoxide-core-tools", "gitoxide-core-blocking-client", "prodash-render-line", "prodash-render-tui", "prodash/render-line-autoconfigure", "gix/regex" ]
	46	+max = ["fast", "pretty-cli", "http-client-curl", "gitoxide-core-tools-query", "gitoxide-core-tools", "gitoxide-core-blocking-client", "prodash-render-line", "prodash-render-tui", "prodash/render-line-autoconfigure", "gix/regex" ]
47	47
48	48	## fast + prodash-render-line-crossterm + gitoxide-core-tools + client-networking.
49		−lean = ["fast", "pretty-cli", "http-client-curl", "gitoxide-core-tools", "gitoxide-core-blocking-client", "prodash-render-line" ]
	49	+lean = ["fast", "pretty-cli", "http-client-curl", "gitoxide-core-tools-query", "gitoxide-core-tools", "gitoxide-core-blocking-client", "prodash-render-line" ]
50	50	## fast* + prodash-render-line-crossterm + gitoxide-core-tools + client-async-networking.
51	51	## Due to async client-networking not being implemented for most transports, this one supports only the 'git' transport.
52	52	## It uses, however, a fully asynchronous networking implementation which can serve a real-world example on how to implement custom async transports.
53		−lean-async = ["fast", "pretty-cli", "gitoxide-core-tools", "gitoxide-core-async-client", "prodash-render-line"]
	53	+lean-async = ["fast", "pretty-cli", "gitoxide-core-tools", "gitoxide-core-tools-query", "gitoxide-core-async-client", "prodash-render-line"]
54	54
55	55	## As small as it can possibly be, no threading, no fast sha1, line progress only, rust based zlib implementation.
56	56	## no networking, local operations only.
@@ -64,6 +64,9 @@ max-pure = ["pretty-cli", "gix-features/rustsha1", "gix-features/zlib-rust-backe · 7 unmodified lines
64	64	## A way to enable all `gitoxide-core` tools found in `gix tools`
65	65	gitoxide-core-tools = ["gitoxide-core/organize", "gitoxide-core/estimate-hours"]
66	66
	67	+## A program to perform analytics on a git repository, using an auto-maintained sqlite database
	68	+gitoxide-core-tools-query = ["gitoxide-core-tools", "gitoxide-core/query"]
	69	+
67	70	#! #### Mutually Exclusive Networking
68	71	#! If both are set a compile error is triggered. This also means that `cargo … --all-features` will fail.
69	72

gitoxide-core/Cargo.toml+7−3

@@ -16,9 +16,11 @@ default = [] · 15 unmodified lines
16	16
17	17	#! ### Tools
18	18	## Discover all git repositories within a directory. Particularly useful with [skim](https://github.com/lotabout/skim).
19		−organize = ["gix-url", "jwalk"]
	19	+organize = ["dep:gix-url", "dep:jwalk"]
20	20	## Derive the amount of time invested into a git repository akin to [git-hours](https://github.com/kimmobrunfeldt/git-hours).
21		−estimate-hours = ["itertools", "fs-err", "num_cpus", "crossbeam-channel", "mime_guess"]
	21	+estimate-hours = ["dep:itertools", "dep:fs-err", "dep:crossbeam-channel", "dep:mime_guess"]
	22	+## Gather information about repositories and store it in a database for easy querying.
	23	+query = ["dep:rusqlite"]
22	24
23	25	#! ### Mutually Exclusive Networking
24	26	#! If both are set, _blocking-client_ will take precedence, allowing `--all-features` to be used.
@@ -62,10 +64,12 @@ jwalk = { version = "0.8.0", optional = true } · 37 unmodified lines
62	64	# for 'hours'
63	65	itertools = { version = "0.10.1", optional = true }
64	66	fs-err = { version = "2.6.0", optional = true }
65		−num_cpus = { version = "1.13.1", optional = true }
66	67	crossbeam-channel = { version = "0.5.6", optional = true }
67	68	mime_guess = { version = "2.0.4", optional = true }
68	69
	70	+# for 'query'
	71	+rusqlite = { version = "0.28.0", optional = true, features = ["bundled"] }
	72	+
69	73	document-features = { version = "0.2.0", optional = true }
70	74
71	75	[package.metadata.docs.rs]

gitoxide-core/src/hours/mod.rs+3−4

@@ -57,10 +57,7 @@ where · 56 unmodified lines
57	57	let commit_id = repo.rev_parse_single(rev_spec)?.detach();
58	58	let mut string_heap = BTreeSet::<&'static [u8]>::new();
59	59	let needs_stats = file_stats \|\| line_stats;
60		− let threads = {
61		− let t = threads.unwrap_or_else(num_cpus::get);
62		− (t == 0).then(num_cpus::get_physical).unwrap_or(t)
63		− };
	60	+ let threads = gix::features::parallel::num_threads(threads);
64	61
65	62	let (commit_authors, stats, is_shallow, skipped_merge_commits) = {
66	63	let stat_progress = needs_stats.then(\|\| progress.add_child("extract stats")).map(\|mut p\| {
@@ -236,6 +233,8 @@ where · 169 unmodified lines
236	233	(true, true) => {
237	234	files.modified += 1;
238	235	if line_stats {
	236	+ // TODO: replace this with proper git-attributes - this isn't
	237	+ // really working, can't see shell scripts for example.
239	238	let is_text_file = mime_guess::from_path(
240	239	gix::path::from_bstr(change.location)
241	240	.as_ref(),

gitoxide-core/src/lib.rs+2−0

@@ -73,6 +73,8 @@ pub mod mailmap; · 72 unmodified lines
73	73	#[cfg(feature = "organize")]
74	74	pub mod organize;
75	75	pub mod pack;
	76	+#[cfg(feature = "query")]
	77	+pub mod query;
76	78	pub mod repository;
77	79
78	80	#[cfg(all(feature = "async-client", feature = "blocking-client"))]

gitoxide-core/src/query/db.rs+68−0

@@ -0,0 +1,68 @@
	1	+use anyhow::Context;
	2	+use rusqlite::{params, OptionalExtension};
	3	+
	4	+/// A version to be incremented whenever the database layout is changed, to refresh it automatically.
	5	+const VERSION: usize = 1;
	6	+
	7	+pub fn create(path: impl AsRef<std::path::Path>) -> anyhow::Result<rusqlite::Connection> {
	8	+ let path = path.as_ref();
	9	+ let mut con = rusqlite::Connection::open(path)?;
	10	+ let meta_table = r#"
	11	+ CREATE TABLE if not exists meta(
	12	+ version int
	13	+ )"#;
	14	+ con.execute_batch(meta_table)?;
	15	+ let version: Option<usize> = con.query_row("SELECT version FROM meta", [], \|r\| r.get(0)).optional()?;
	16	+ match version {
	17	+ None => {
	18	+ con.execute("INSERT into meta(version) values(?)", params![VERSION])?;
	19	+ }
	20	+ Some(version) if version != VERSION => match con.close() {
	21	+ Ok(()) => {
	22	+ std::fs::remove_file(path)
	23	+ .with_context(\|\| format!("Failed to remove incompatible database file at {path:?}"))?;
	24	+ con = rusqlite::Connection::open(path)?;
	25	+ con.execute_batch(meta_table)?;
	26	+ con.execute("INSERT into meta(version) values(?)", params![VERSION])?;
	27	+ }
	28	+ Err((_, err)) => return Err(err.into()),
	29	+ },
	30	+ _ => {}
	31	+ }
	32	+ con.execute_batch(
	33	+ r#"
	34	+ CREATE TABLE if not exists commits(
	35	+ hash blob(20) NOT NULL PRIMARY KEY
	36	+ )
	37	+ "#,
	38	+ )?;
	39	+ // Files are stored as paths which also have an id for referencing purposes
	40	+ con.execute_batch(
	41	+ r#"
	42	+ CREATE TABLE if not exists files(
	43	+ file_id integer NOT NULL PRIMARY KEY,
	44	+ file_path text UNIQUE
	45	+ )
	46	+ "#,
	47	+ )?;
	48	+ con.execute_batch(
	49	+ r#"
	50	+ CREATE TABLE if not exists commit_file(
	51	+ hash blob(20),
	52	+ file_id text,
	53	+ has_diff boolean NOT NULL,
	54	+ lines_added integer NOT NULL,
	55	+ lines_removed integer NOT NULL,
	56	+ lines_before integer NOT NULL,
	57	+ lines_after integer NOT NULL,
	58	+ mode integer,
	59	+ source_file_id integer,
	60	+ FOREIGN KEY (hash) REFERENCES commits (hash),
	61	+ FOREIGN KEY (file_id) REFERENCES files (file_id),
	62	+ PRIMARY KEY (hash, file_id)
	63	+ )
	64	+ "#,
	65	+ )?;
	66	+
	67	+ Ok(con)
	68	+}

gitoxide-core/src/query/engine/command.rs+183−0

@@ -0,0 +1,183 @@
	1	+use crate::query;
	2	+use crate::query::engine::update::FileMode;
	3	+use crate::query::Command;
	4	+use anyhow::Context;
	5	+use gix::bstr::ByteSlice;
	6	+use gix::Progress;
	7	+use rusqlite::{params, OptionalExtension};
	8	+use std::collections::HashMap;
	9	+
	10	+impl query::Engine {
	11	+ pub fn run(
	12	+ &self,
	13	+ cmd: Command,
	14	+ mut out: impl std::io::Write,
	15	+ mut progress: impl gix::Progress,
	16	+ ) -> anyhow::Result<()> {
	17	+ match cmd {
	18	+ Command::TracePath { mut spec } => {
	19	+ if let Some(prefix) = self.repo.prefix() {
	20	+ spec.apply_prefix(&prefix?);
	21	+ };
	22	+ let relpath = spec.items().next().expect("spec has at least one item");
	23	+ let file_id: usize = self
	24	+ .con
	25	+ .query_row(
	26	+ "SELECT file_id FROM files WHERE file_path = ?",
	27	+ params![relpath.to_str_lossy()],
	28	+ \|r\| r.get(0),
	29	+ )
	30	+ .optional()?
	31	+ .context("Path not found anywhere in recorded history")?;
	32	+
	33	+ let mut by_file_id = self
	34	+ .con
	35	+ .prepare("SELECT hash, mode, source_file_id, has_diff, lines_added, lines_removed from commit_file where file_id = ? order by mode")?;
	36	+ let mut path_by_id = self.con.prepare("SELECT file_path from files where file_id = ?")?;
	37	+ let mut seen = HashMap::<usize, String>::new();
	38	+ seen.insert(file_id, relpath.to_string());
	39	+
	40	+ let mut stack = vec![file_id];
	41	+ let mut info = Vec::new();
	42	+ let start = std::time::Instant::now();
	43	+ let mut progress = progress.add_child("run sql query");
	44	+ progress.init(None, gix::progress::count("round"));
	45	+ while let Some(file_id) = stack.pop() {
	46	+ let rows = by_file_id.query_map([file_id], \|r\| {
	47	+ Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?))
	48	+ })?;
	49	+ progress.inc();
	50	+ for row in rows {
	51	+ let (hash, mode, source_file_id, has_diff, lines_added, lines_removed): (
	52	+ [u8; 20],
	53	+ usize,
	54	+ Option<usize>,
	55	+ bool,
	56	+ usize,
	57	+ usize,
	58	+ ) = row?;
	59	+ let id = gix::ObjectId::from(hash);
	60	+ let mode = FileMode::from_usize(mode).context("invalid file mode")?;
	61	+ info.push(trace_path::Info {
	62	+ id,
	63	+ file_id,
	64	+ mode,
	65	+ diff: has_diff.then_some(trace_path::Diff {
	66	+ lines_added,
	67	+ lines_removed,
	68	+ }),
	69	+ source_file_id,
	70	+ });
	71	+ if let Some(source_id) = source_file_id {
	72	+ if let std::collections::hash_map::Entry::Vacant(e) = seen.entry(source_id) {
	73	+ stack.push(source_id);
	74	+ e.insert(path_by_id.query_row([source_id], \|r\| r.get(0))?);
	75	+ }
	76	+ }
	77	+ }
	78	+ }
	79	+
	80	+ info.sort_by(\|a, b\| a.id.cmp(&b.id));
	81	+ let max_diff_lines = info
	82	+ .iter()
	83	+ .map(\|i\| i.diff.map_or(0, \|d\| d.lines_removed + d.lines_added))
	84	+ .max()
	85	+ .unwrap_or_default();
	86	+ let mut found = 0;
	87	+ progress.show_throughput(start);
	88	+ let start = std::time::Instant::now();
	89	+ progress.init(Some(self.commits.len()), gix::progress::count("commits"));
	90	+ progress.set_name("associate info");
	91	+ for info in self
	92	+ .commits
	93	+ .iter()
	94	+ .inspect(\|_\| progress.inc())
	95	+ .filter_map(\|c\| info.binary_search_by(\|i\| i.id.cmp(c)).ok().map(\|idx\| &info[idx]))
	96	+ {
	97	+ found += 1;
	98	+ info.write_to(&mut out, &self.repo, &seen, max_diff_lines)?;
	99	+ }
	100	+ let missing = info.len() - found;
	101	+ progress.show_throughput(start);
	102	+ if missing > 0 {
	103	+ writeln!(
	104	+ out,
	105	+ "{missing} file(s) were found in history that are not reachable from HEAD"
	106	+ )?;
	107	+ }
	108	+ Ok(())
	109	+ }
	110	+ }
	111	+ }
	112	+}
	113	+
	114	+mod trace_path {
	115	+ use crate::query::engine::update::FileMode;
	116	+ use gix::prelude::ObjectIdExt;
	117	+ use std::collections::HashMap;
	118	+
	119	+ #[derive(Debug, Default, Copy, Clone)]
	120	+ pub struct Diff {
	121	+ pub lines_added: usize,
	122	+ pub lines_removed: usize,
	123	+ }
	124	+
	125	+ impl Diff {
	126	+ fn format(&self, max_diff_lines: usize) -> String {
	127	+ const NUM_CHARS: f32 = 10.0;
	128	+ let mut buf = String::with_capacity(NUM_CHARS as usize);
	129	+ if max_diff_lines != 0 {
	130	+ let num_plus = ((self.lines_added as f32 / max_diff_lines as f32) * NUM_CHARS).ceil() as usize;
	131	+ let num_minus = ((self.lines_removed as f32 / max_diff_lines as f32) * NUM_CHARS) as usize;
	132	+ buf.extend((0..num_plus).map(\|_\| '+'));
	133	+ buf.extend((0..num_minus).map(\|_\| '-'));
	134	+ }
	135	+ buf.extend((buf.len()..NUM_CHARS as usize).map(\|_\| ' '));
	136	+ buf
	137	+ }
	138	+ }
	139	+
	140	+ #[derive(Debug)]
	141	+ pub struct Info {
	142	+ pub id: gix::ObjectId,
	143	+ pub file_id: usize,
	144	+ pub mode: FileMode,
	145	+ pub diff: Option<Diff>,
	146	+ pub source_file_id: Option<usize>,
	147	+ }
	148	+
	149	+ impl Info {
	150	+ pub fn write_to(
	151	+ &self,
	152	+ mut out: impl std::io::Write,
	153	+ repo: &gix::Repository,
	154	+ path_by_id: &HashMap<usize, String>,
	155	+ max_diff_lines: usize,
	156	+ ) -> std::io::Result<()> {
	157	+ let id = self.id.attach(repo);
	158	+ match self.source_file_id {
	159	+ Some(source_id) => {
	160	+ writeln!(
	161	+ out,
	162	+ "{}\|{} {} {} ➡ {}",
	163	+ self.diff.unwrap_or_default().format(max_diff_lines),
	164	+ id.shorten_or_id(),
	165	+ self.mode.as_str(),
	166	+ path_by_id[&source_id],
	167	+ path_by_id[&self.file_id],
	168	+ )
	169	+ }
	170	+ None => {
	171	+ writeln!(
	172	+ out,
	173	+ "{}\|{} {} {}",
	174	+ self.diff.unwrap_or_default().format(max_diff_lines),
	175	+ id.shorten_or_id(),
	176	+ self.mode.as_str(),
	177	+ path_by_id[&self.file_id]
	178	+ )
	179	+ }
	180	+ }
	181	+ }
	182	+ }
	183	+}

gitoxide-core/src/query/engine/mod.rs+11−0

@@ -0,0 +1,11 @@
	1	+pub enum Command {
	2	+ TracePath {
	3	+ /// The repo-relative path to the file to trace
	4	+ spec: gix::path::Spec,
	5	+ },
	6	+}
	7	+
	8	+pub(crate) mod update;
	9	+pub use update::update;
	10	+
	11	+mod command;

gitoxide-core/src/query/engine/update.rs+501−0

@@ -0,0 +1,501 @@
	1	+use crate::query::Options;
	2	+use anyhow::{anyhow, bail};
	3	+use gix::bstr::{BStr, BString, ByteSlice};
	4	+use gix::features::progress;
	5	+use gix::object::tree::diff::rewrites::CopySource;
	6	+use gix::odb::FindExt;
	7	+use gix::parallel::{InOrderIter, SequenceId};
	8	+use gix::prelude::ObjectIdExt;
	9	+use gix::Progress;
	10	+use rusqlite::{params, Statement, Transaction};
	11	+use std::convert::Infallible;
	12	+use std::sync::atomic::{AtomicUsize, Ordering};
	13	+use std::time::Instant;
	14	+
	15	+pub fn update(
	16	+ repo: &gix::Repository,
	17	+ con: &mut rusqlite::Connection,
	18	+ progress: &mut impl gix::Progress,
	19	+ Options {
	20	+ object_cache_size_mb,
	21	+ find_copies_harder,
	22	+ threads,
	23	+ }: Options,
	24	+) -> anyhow::Result<Vec<gix::ObjectId>> {
	25	+ let commit_id = repo.head_id()?.detach();
	26	+ let threads = gix::features::parallel::num_threads(threads);
	27	+
	28	+ let mut stat_progress = {
	29	+ let mut p = progress.add_child("extract stats");
	30	+ p.init(None, progress::count("commits"));
	31	+ p
	32	+ };
	33	+ let stat_counter = stat_progress.counter().expect("shared counter available");
	34	+
	35	+ let mut db_progress = {
	36	+ let mut p = progress.add_child("db cache");
	37	+ p.init(None, progress::count("events"));
	38	+ p
	39	+ };
	40	+ let commit_counter = db_progress.counter().expect("shared counter available");
	41	+
	42	+ let mut change_progress = {
	43	+ let mut p = progress.add_child("find changes");
	44	+ p.init(None, progress::count("modified files"));
	45	+ p
	46	+ };
	47	+ let change_counter = change_progress.counter().expect("shared counter available");
	48	+
	49	+ let mut lines_progress = {
	50	+ let mut p = progress.add_child("find changes");
	51	+ p.init(None, progress::count("diff lines"));
	52	+ p
	53	+ };
	54	+ let lines_counter = lines_progress.counter().expect("shared counter available");
	55	+
	56	+ let mut traverse_progress = progress.add_child("traverse commit graph");
	57	+ traverse_progress.init(None, progress::count("commits"));
	58	+
	59	+ let out = std::thread::scope(\|scope\| -> anyhow::Result<_> {
	60	+ struct Commit {
	61	+ id: gix::hash::ObjectId,
	62	+ }
	63	+ struct CommitDiffStats {
	64	+ /// The id of the commit which was diffed with its predecessor
	65	+ id: gix::hash::ObjectId,
	66	+ changes: Vec<FileChange>,
	67	+ }
	68	+ let start = Instant::now();
	69	+ let (tx_commits, rx_commits) = std::sync::mpsc::channel::<Commit>();
	70	+ let (tx_stats, rx_stats) = std::sync::mpsc::channel::<Result<(SequenceId, Vec<CommitDiffStats>), Infallible>>();
	71	+
	72	+ let mut known_commits =
	73	+ Vec::with_capacity(con.query_row("SELECT COUNT(hash) from commits", [], \|r\| r.get::<_, usize>(0))?);
	74	+ for item in con.prepare("SELECT hash from commits")?.query_map([], \|r\| {
	75	+ Ok(gix::ObjectId::try_from(r.get_ref(0)?.as_bytes()?)
	76	+ .unwrap_or_else(\|_\| gix::ObjectId::null(gix::hash::Kind::Sha1)))
	77	+ })? {
	78	+ known_commits.push(item?);
	79	+ }
	80	+ known_commits.sort();
	81	+
	82	+ let db_thread = scope.spawn({
	83	+ move \|\| -> anyhow::Result<()> {
	84	+ let trans = con.transaction()?;
	85	+ {
	86	+ let Updates {
	87	+ mut new_commit,
	88	+ mut insert_commit_file,
	89	+ mut insert_commit_file_with_source,
	90	+ mut insert_file_path,
	91	+ } = Updates::new(&trans)?;
	92	+ for Commit { id } in rx_commits {
	93	+ new_commit.execute(params![id.as_bytes()])?;
	94	+ commit_counter.fetch_add(1, Ordering::Relaxed);
	95	+ }
	96	+ for stats in InOrderIter::from(rx_stats.into_iter()) {
	97	+ for CommitDiffStats { id, changes } in stats.expect("infallible") {
	98	+ new_commit.execute(params![id.as_bytes()])?;
	99	+ for change in changes {
	100	+ insert_file_path.execute(params![change.relpath.to_str_lossy()])?;
	101	+ let (has_diff, lines) = change.lines.map(\|l\| (true, l)).unwrap_or_default();
	102	+ if let Some(source_relpath) = change.source_relpath {
	103	+ insert_file_path.execute(params![source_relpath.to_str_lossy()])?;
	104	+ insert_commit_file_with_source.execute(params![
	105	+ id.as_bytes(),
	106	+ change.relpath.to_str_lossy(),
	107	+ has_diff,
	108	+ lines.added,
	109	+ lines.removed,
	110	+ lines.before,
	111	+ lines.after,
	112	+ change.mode as usize,
	113	+ source_relpath.to_str_lossy(),
	114	+ ])?;
	115	+ } else {
	116	+ insert_commit_file.execute(params![
	117	+ id.as_bytes(),
	118	+ change.relpath.to_str_lossy(),
	119	+ has_diff,
	120	+ lines.added,
	121	+ lines.removed,
	122	+ lines.before,
	123	+ lines.after,
	124	+ change.mode as usize,
	125	+ ])?;
	126	+ }
	127	+ }
	128	+ commit_counter.fetch_add(1, Ordering::Relaxed);
	129	+ }
	130	+ }
	131	+ }
	132	+ trans.commit()?;
	133	+ Ok(())
	134	+ }
	135	+ });
	136	+
	137	+ let rewrites = {
	138	+ let mut r =
	139	+ gix::object::tree::diff::Rewrites::try_from_config(&repo.config_snapshot(), true)?.unwrap_or_default();
	140	+ r.copies = Some(gix::object::tree::diff::rewrites::Copies {
	141	+ source: if find_copies_harder {
	142	+ CopySource::FromSetOfModifiedFilesAndSourceTree
	143	+ } else {
	144	+ CopySource::FromSetOfModifiedFiles
	145	+ },
	146	+ percentage: None,
	147	+ });
	148	+ r
	149	+ };
	150	+ let (tx_tree_ids, stat_threads) = {
	151	+ let (tx, rx) =
	152	+ crossbeam_channel::unbounded::<(SequenceId, Vec<(Option<gix::hash::ObjectId>, gix::hash::ObjectId)>)>();
	153	+ let stat_workers = (0..threads)
	154	+ .map(\|_\| {
	155	+ scope.spawn({
	156	+ let stat_counter = stat_counter.clone();
	157	+ let change_counter = change_counter.clone();
	158	+ let lines_counter = lines_counter.clone();
	159	+ let tx_stats = tx_stats.clone();
	160	+ let mut repo = repo.clone();
	161	+ repo.object_cache_size_if_unset((object_cache_size_mb * 1024 * 1024) / threads);
	162	+ let rx = rx.clone();
	163	+ move \|\| -> anyhow::Result<()> {
	164	+ for (chunk_id, chunk) in rx {
	165	+ let mut out_chunk = Vec::with_capacity(chunk.len());
	166	+ for (parent_commit, commit) in chunk {
	167	+ stat_counter.fetch_add(1, Ordering::SeqCst);
	168	+ if gix::interrupt::is_triggered() {
	169	+ return Ok(());
	170	+ }
	171	+ let mut out = Vec::new();
	172	+ let from = match parent_commit {
	173	+ Some(id) => {
	174	+ match repo.find_object(id).ok().and_then(\|c\| c.peel_to_tree().ok()) {
	175	+ Some(tree) => tree,
	176	+ None => continue,
	177	+ }
	178	+ }
	179	+ None => repo.empty_tree(),
	180	+ };
	181	+ let to = match repo.find_object(commit).ok().and_then(\|c\| c.peel_to_tree().ok()) {
	182	+ Some(c) => c,
	183	+ None => continue,
	184	+ };
	185	+ from.changes()?
	186	+ .track_path()
	187	+ .track_rewrites(Some(rewrites))
	188	+ .for_each_to_obtain_tree(&to, \|change\| {
	189	+ use gix::object::tree::diff::change::Event::*;
	190	+ change_counter.fetch_add(1, Ordering::SeqCst);
	191	+ match change.event {
	192	+ Addition { entry_mode, id } => {
	193	+ if entry_mode.is_blob_or_symlink() {
	194	+ add_lines(&mut out, change.location, &lines_counter, id);
	195	+ }
	196	+ }
	197	+ Modification {
	198	+ entry_mode,
	199	+ previous_entry_mode,
	200	+ id,
	201	+ previous_id,
	202	+ } => match (previous_entry_mode.is_blob(), entry_mode.is_blob()) {
	203	+ (false, false) => {}
	204	+ (false, true) => {
	205	+ add_lines(&mut out, change.location, &lines_counter, id);
	206	+ }
	207	+ (true, false) => {
	208	+ add_lines(
	209	+ &mut out,
	210	+ change.location,
	211	+ &lines_counter,
	212	+ previous_id,
	213	+ );
	214	+ }
	215	+ (true, true) => {
	216	+ // TODO: use git attributes here to know if it's a binary file or not.
	217	+ if let Some(Ok(diff)) = change.event.diff() {
	218	+ let mut nl = 0;
	219	+ let tokens = diff.line_tokens();
	220	+ let counts = gix::diff::blob::diff(
	221	+ diff.algo,
	222	+ &tokens,
	223	+ gix::diff::blob::sink::Counter::default(),
	224	+ );
	225	+ nl += counts.insertions as usize + counts.removals as usize;
	226	+ let lines = LineStats {
	227	+ added: counts.insertions as usize,
	228	+ removed: counts.removals as usize,
	229	+ before: tokens.before.len(),
	230	+ after: tokens.after.len(),
	231	+ };
	232	+ lines_counter.fetch_add(nl, Ordering::SeqCst);
	233	+ out.push(FileChange {
	234	+ relpath: change.location.to_owned(),
	235	+ mode: FileMode::Modified,
	236	+ source_relpath: None,
	237	+ lines: Some(lines),
	238	+ });
	239	+ }
	240	+ }
	241	+ },
	242	+ Deletion { entry_mode, id } => {
	243	+ if entry_mode.is_blob_or_symlink() {
	244	+ remove_lines(&mut out, change.location, &lines_counter, id);
	245	+ }
	246	+ }
	247	+ Rewrite {
	248	+ source_location,
	249	+ diff,
	250	+ copy,
	251	+ ..
	252	+ } => {
	253	+ out.push(FileChange {
	254	+ relpath: change.location.to_owned(),
	255	+ source_relpath: Some(source_location.to_owned()),
	256	+ mode: if copy { FileMode::Copy } else { FileMode::Rename },
	257	+ lines: diff.map(\|d\| LineStats {
	258	+ added: d.insertions as usize,
	259	+ removed: d.removals as usize,
	260	+ before: d.before as usize,
	261	+ after: d.after as usize,
	262	+ }),
	263	+ });
	264	+ }
	265	+ }
	266	+ Ok::<_, Infallible>(Default::default())
	267	+ })?;
	268	+ out_chunk.push(CommitDiffStats {
	269	+ id: commit,
	270	+ changes: out,
	271	+ });
	272	+ }
	273	+ if tx_stats.send(Ok((chunk_id, out_chunk))).is_err() {
	274	+ break;
	275	+ }
	276	+ }
	277	+ Ok(())
	278	+ }
	279	+ })
	280	+ })
	281	+ .collect::<Vec<_>>();
	282	+ (tx, stat_workers)
	283	+ };
	284	+ drop(tx_stats);
	285	+
	286	+ let mut skipped_merge_commits = 0;
	287	+ const CHUNK_SIZE: usize = 50;
	288	+ let mut chunk = Vec::with_capacity(CHUNK_SIZE);
	289	+ let mut chunk_id: SequenceId = 0;
	290	+ let commit_iter = gix::interrupt::Iter::new(
	291	+ commit_id.ancestors(\|oid, buf\| -> Result<_, gix::object::find::existing::Error> {
	292	+ let obj = repo.objects.find(oid, buf)?;
	293	+ traverse_progress.inc();
	294	+ if known_commits.binary_search(&oid.to_owned()).is_err() {
	295	+ let res = {
	296	+ let mut parents = gix::objs::CommitRefIter::from_bytes(obj.data).parent_ids();
	297	+ let res = parents.next().map(\|first_parent\| (Some(first_parent), oid.to_owned()));
	298	+ match parents.next() {
	299	+ Some(_) => {
	300	+ skipped_merge_commits += 1;
	301	+ None
	302	+ }
	303	+ None => res,
	304	+ }
	305	+ };
	306	+ if let Some((first_parent, commit)) = res {
	307	+ if chunk.len() == CHUNK_SIZE {
	308	+ tx_tree_ids
	309	+ .send((chunk_id, std::mem::replace(&mut chunk, Vec::with_capacity(CHUNK_SIZE))))
	310	+ .ok();
	311	+ chunk_id += 1;
	312	+ }
	313	+ chunk.push((first_parent, commit));
	314	+ } else {
	315	+ tx_commits.send(Commit { id: oid.to_owned() }).ok();
	316	+ }
	317	+ }
	318	+ Ok(gix::objs::CommitRefIter::from_bytes(obj.data))
	319	+ }),
	320	+ \|\| anyhow!("Cancelled by user"),
	321	+ );
	322	+ let mut commits = Vec::new();
	323	+ for c in commit_iter {
	324	+ match c? {
	325	+ Ok(c) => {
	326	+ commits.push(c);
	327	+ }
	328	+ Err(gix::traverse::commit::ancestors::Error::FindExisting { .. }) => {
	329	+ eprintln!("shallow repository - commit history is truncated");
	330	+ break;
	331	+ }
	332	+ Err(err) => return Err(err.into()),
	333	+ };
	334	+ }
	335	+ drop(tx_commits);
	336	+ tx_tree_ids.send((chunk_id, chunk)).ok();
	337	+ drop(tx_tree_ids);
	338	+ let saw_new_commits = !commits.is_empty();
	339	+ if saw_new_commits {
	340	+ traverse_progress.show_throughput(start);
	341	+ }
	342	+ drop(traverse_progress);
	343	+
	344	+ let stat_max = Some(commits.len());
	345	+ stat_progress.set_max(stat_max);
	346	+ db_progress.set_max(stat_max);
	347	+ for handle in stat_threads {
	348	+ handle.join().expect("no panic")?;
	349	+ if gix::interrupt::is_triggered() {
	350	+ bail!("Cancelled by user");
	351	+ }
	352	+ }
	353	+ if saw_new_commits {
	354	+ stat_progress.show_throughput(start);
	355	+ change_progress.show_throughput(start);
	356	+ lines_progress.show_throughput(start);
	357	+ }
	358	+
	359	+ db_thread.join().expect("no panic")?;
	360	+ if saw_new_commits {
	361	+ db_progress.show_throughput(start);
	362	+ } else {
	363	+ db_progress.info("up to date");
	364	+ }
	365	+ Ok(commits)
	366	+ })?;
	367	+
	368	+ Ok(out)
	369	+}
	370	+
	371	+fn add_lines(out: &mut Vec<FileChange>, path: &BStr, lines_counter: &AtomicUsize, id: gix::Id<'_>) {
	372	+ if let Ok(blob) = id.object() {
	373	+ let nl = blob.data.lines_with_terminator().count();
	374	+ let mut lines = LineStats::default();
	375	+ lines.added += nl;
	376	+ lines.after = nl;
	377	+ lines_counter.fetch_add(nl, Ordering::SeqCst);
	378	+ out.push(FileChange {
	379	+ relpath: path.to_owned(),
	380	+ mode: FileMode::Added,
	381	+ source_relpath: None,
	382	+ lines: Some(lines),
	383	+ });
	384	+ }
	385	+}
	386	+
	387	+fn remove_lines(out: &mut Vec<FileChange>, path: &BStr, lines_counter: &AtomicUsize, id: gix::Id<'_>) {
	388	+ if let Ok(blob) = id.object() {
	389	+ let mut lines = LineStats::default();
	390	+ let nl = blob.data.lines_with_terminator().count();
	391	+ lines.removed += nl;
	392	+ lines.before = nl;
	393	+ lines_counter.fetch_add(nl, Ordering::SeqCst);
	394	+ out.push(FileChange {
	395	+ relpath: path.to_owned(),
	396	+ mode: FileMode::Removed,
	397	+ source_relpath: None,
	398	+ lines: Some(lines),
	399	+ })
	400	+ }
	401	+}
	402	+
	403	+#[derive(Debug, Copy, Clone)]
	404	+pub enum FileMode {
	405	+ Added = 1,
	406	+ Removed = 2,
	407	+ Modified = 3,
	408	+ Rename = 4,
	409	+ Copy = 5,
	410	+}
	411	+
	412	+impl FileMode {
	413	+ pub fn as_str(&self) -> &'static str {
	414	+ use FileMode::*;
	415	+ match self {
	416	+ Added => "+",
	417	+ Removed => "-",
	418	+ Modified => "Δ",
	419	+ Rename => "➡",
	420	+ Copy => "⏸",
	421	+ }
	422	+ }
	423	+ pub fn from_usize(mode: usize) -> Option<Self> {
	424	+ use FileMode::*;
	425	+ match mode {
	426	+ 1 => Added,
	427	+ 2 => Removed,
	428	+ 3 => Modified,
	429	+ 4 => Rename,
	430	+ 5 => Copy,
	431	+ _ => return None,
	432	+ }
	433	+ .into()
	434	+ }
	435	+}
	436	+
	437	+#[derive(Debug)]
	438	+struct FileChange {
	439	+ relpath: BString,
	440	+ mode: FileMode,
	441	+ source_relpath: Option<BString>,
	442	+ lines: Option<LineStats>,
	443	+}
	444	+
	445	+/// Line statistics for a particular commit.
	446	+#[derive(Debug, Default, Copy, Clone)]
	447	+struct LineStats {
	448	+ /// amount of added lines
	449	+ added: usize,
	450	+ /// amount of removed lines
	451	+ removed: usize,
	452	+ /// the amount of lines before the change.
	453	+ before: usize,
	454	+ /// the amount of lines after the change.
	455	+ after: usize,
	456	+}
	457	+
	458	+struct Updates<'a> {
	459	+ new_commit: Statement<'a>,
	460	+ insert_commit_file: Statement<'a>,
	461	+ insert_commit_file_with_source: Statement<'a>,
	462	+ insert_file_path: Statement<'a>,
	463	+}
	464	+
	465	+impl<'a> Updates<'a> {
	466	+ fn new(trans: &'a Transaction<'_>) -> rusqlite::Result<Self> {
	467	+ let new_commit = trans.prepare(
	468	+ r#"INSERT INTO
	469	+ commits(hash)
	470	+ VALUES(?)"#,
	471	+ )?;
	472	+ let insert_commit_file = trans.prepare(
	473	+ r#"
	474	+ INSERT INTO
	475	+ commit_file(hash, file_id, has_diff, lines_added, lines_removed, lines_before, lines_after, mode)
	476	+ VALUES(?, (SELECT files.file_id FROM files WHERE files.file_path = ?), ?, ?, ?, ?, ?, ?)
	477	+ "#,
	478	+ )?;
	479	+ let insert_commit_file_with_source = trans.prepare(
	480	+ r#"
	481	+ INSERT INTO
	482	+ commit_file(hash, file_id, has_diff, lines_added, lines_removed, lines_before, lines_after, mode, source_file_id)
	483	+ VALUES(?, (SELECT files.file_id FROM files WHERE files.file_path = ?), ?, ?, ?, ?, ?, ?, (SELECT files.file_id FROM files WHERE files.file_path = ?))
	484	+ "#,
	485	+ )?;
	486	+
	487	+ let insert_file_path = trans.prepare(
	488	+ r#"
	489	+ INSERT OR IGNORE INTO
	490	+ files(file_path)
	491	+ VALUES(?)
	492	+ "#,
	493	+ )?;
	494	+ Ok(Updates {
	495	+ new_commit,
	496	+ insert_commit_file,
	497	+ insert_commit_file_with_source,
	498	+ insert_file_path,
	499	+ })
	500	+ }
	501	+}

gitoxide-core/src/query/mod.rs+23−0

@@ -0,0 +1,23 @@
	1	+pub struct Engine {
	2	+ repo: gix::Repository,
	3	+ con: rusqlite::Connection,
	4	+ commits: Vec<gix::ObjectId>,
	5	+}
	6	+
	7	+pub struct Options {
	8	+ pub object_cache_size_mb: usize,
	9	+ pub find_copies_harder: bool,
	10	+ pub threads: Option<usize>,
	11	+}
	12	+
	13	+mod db;
	14	+
	15	+mod engine;
	16	+pub use engine::Command;
	17	+
	18	+pub fn prepare(repo_dir: &std::path::Path, mut progress: impl gix::Progress, opts: Options) -> anyhow::Result<Engine> {
	19	+ let repo = gix::discover(repo_dir)?;
	20	+ let mut con = db::create(repo.git_dir().join("ein.query"))?;
	21	+ let commits = engine::update(&repo, &mut con, &mut progress, opts)?;
	22	+ Ok(Engine { repo, con, commits })
	23	+}

src/plumbing/options/mod.rs+1−24

@@ -355,7 +355,7 @@ pub mod revision { · 354 unmodified lines
355	355	pub mod exclude {
356	356	use std::ffi::OsString;
357	357
358		− use super::AsPathSpec;
	358	+ use crate::shared::AsPathSpec;
359	359
360	360	#[derive(Debug, clap::Subcommand)]
361	361	pub enum Subcommands {
@@ -402,26 +402,3 @@ pub mod index { · 40 unmodified lines
402	402
403	403	///
404	404	pub mod free;
405		−
406		−mod clap_util {
407		− use std::ffi::OsStr;
408		−
409		− use clap::{
410		− builder::{OsStringValueParser, TypedValueParser},
411		− Arg, Command, Error,
412		− };
413		−
414		− #[derive(Clone)]
415		− pub struct AsPathSpec;
416		−
417		− impl TypedValueParser for AsPathSpec {
418		− type Value = gix::path::Spec;
419		−
420		− fn parse_ref(&self, cmd: &Command, arg: Option<&Arg>, value: &OsStr) -> Result<Self::Value, Error> {
421		− OsStringValueParser::new()
422		− .try_map(\|arg\| gix::path::Spec::try_from(arg.as_os_str()))
423		− .parse_ref(cmd, arg, value)
424		− }
425		− }
426		−}
427		−use clap_util::AsPathSpec;

src/porcelain/main.rs+46−9

@@ -26,6 +26,8 @@ pub fn main() -> Result<()> { · 25 unmodified lines
26	26	})?;
27	27	let verbose = !args.quiet;
28	28	let progress = args.progress;
	29	+ #[cfg(feature = "gitoxide-core-tools")]
	30	+ let threads = args.threads;
29	31	let progress_keep_open = args.progress_keep_open;
30	32
31	33	match args.cmd {
@@ -41,16 +43,51 @@ pub fn main() -> Result<()> { · 9 unmodified lines
41	43	Subcommands::Init { directory } => core::repository::init(directory).map(\|_\| ()),
42	44	#[cfg(feature = "gitoxide-core-tools")]
43	45	Subcommands::Tool(tool) => match tool {
44		− crate::porcelain::options::ToolCommands::EstimateHours(crate::porcelain::options::EstimateHours {
45		− working_dir,
46		− rev_spec,
47		− no_bots,
48		− threads,
49		− file_stats,
50		− line_stats,
51		− show_pii,
52		− omit_unify_identities,
	46	+ #[cfg(feature = "gitoxide-core-tools-query")]
	47	+ crate::porcelain::options::ToolCommands::Query(crate::porcelain::options::tools::Query {
	48	+ object_cache_size_mb,
	49	+ find_copies_harder,
	50	+ repo_dir,
	51	+ cmd,
53	52	}) => {
	53	+ use gitoxide_core::query;
	54	+ prepare_and_run(
	55	+ "query",
	56	+ verbose,
	57	+ progress,
	58	+ progress_keep_open,
	59	+ crate::shared::STANDARD_RANGE,
	60	+ move \|mut progress, out, err\| {
	61	+ let engine = query::prepare(
	62	+ &repo_dir,
	63	+ &mut progress,
	64	+ query::Options {
	65	+ object_cache_size_mb,
	66	+ find_copies_harder,
	67	+ threads,
	68	+ },
	69	+ )?;
	70	+ match cmd {
	71	+ None => writeln!(err, "Choose a command for the query engine")?,
	72	+ Some(crate::porcelain::options::tools::query::Command::TracePath { path }) => {
	73	+ engine.run(query::Command::TracePath { spec: path }, out, progress)?;
	74	+ }
	75	+ }
	76	+ Ok(())
	77	+ },
	78	+ )
	79	+ }
	80	+ crate::porcelain::options::ToolCommands::EstimateHours(
	81	+ crate::porcelain::options::tools::EstimateHours {
	82	+ working_dir,
	83	+ rev_spec,
	84	+ no_bots,
	85	+ file_stats,
	86	+ line_stats,
	87	+ show_pii,
	88	+ omit_unify_identities,
	89	+ },
	90	+ ) => {
54	91	use gitoxide_core::hours;
55	92	prepare_and_run(
56	93	"estimate-hours",

src/porcelain/options.rs+127−81

@@ -1,7 +1,5 @@
1	1	use std::path::PathBuf;
2	2
3		−use gix::bstr::BString;
4		−
5	3	#[derive(Debug, clap::Parser)]
6	4	#[clap(about = "The rusty git", version = clap::crate_version!())]
7	5	#[clap(subcommand_required = true)]
@@ -13,6 +11,9 @@ pub struct Args { · 5 unmodified lines
13	11	/// Bring up a terminal user interface displaying progress visually
14	12	#[clap(long, conflicts_with("quiet"))]
15	13	pub progress: bool,
	14	+ /// The amount of threads to use. If unset, use all cores, if 0 use al physical cores.
	15	+ #[clap(short = 't', long)]
	16	+ pub threads: Option<usize>,
16	17
17	18	/// The progress TUI will stay up even though the work is already completed.
18	19	///
@@ -78,91 +79,136 @@ pub enum ToolCommands { · 59 unmodified lines
78	79	/// Defaults to the current working directory.
79	80	destination_directory: Option<PathBuf>,
80	81	},
81		− EstimateHours(EstimateHours),
	82	+ #[cfg(feature = "gitoxide-core-tools-query")]
	83	+ Query(tools::Query),
	84	+ EstimateHours(tools::EstimateHours),
82	85	}
83	86
84		−#[derive(Debug, clap::Parser)]
85		−#[clap(
86		− about = "Estimate hours worked based on a commit history",
87		− long_about = "See https://github.com/kimmobrunfeldt/git-hours#how-it-works for details",
88		− version = clap::crate_version!(),
89		− visible_alias = "h",
90		− visible_alias = "hours")
91		−]
92		−pub struct EstimateHours {
93		− /// The directory containing a '.git/' folder.
94		− #[clap(value_parser = validator::IsRepo)]
95		− #[clap(default_value = ".")]
96		− pub working_dir: PathBuf,
97		− /// The name of the revision as spec, like 'HEAD' or 'main' at which to start iterating the commit graph.
98		− #[clap(default_value("HEAD"), value_parser = crate::shared::AsBString)]
99		− pub rev_spec: BString,
100		− /// Ignore github bots which match the `[bot]` search string.
101		− #[clap(short = 'b', long)]
102		− pub no_bots: bool,
103		− /// Collect additional information about file modifications, additions and deletions (without rename tracking).
104		− #[clap(short = 'f', long)]
105		− pub file_stats: bool,
106		− /// Collect additional information about lines added and deleted (without rename tracking).
107		− ///
108		− /// Note that this implies the work to be done for file-stats, so it should be set as well.
109		− #[clap(short = 'l', long)]
110		− pub line_stats: bool,
111		− /// The amount of threads to use. If unset, use all cores, if 0 use al physical cores.
112		− #[clap(short = 't', long)]
113		− pub threads: Option<usize>,
114		− /// Show personally identifiable information before the summary. Includes names and email addresses.
115		− #[clap(short = 'p', long)]
116		− pub show_pii: bool,
117		− /// Omit unifying identities by name and email which can lead to the same author appear multiple times
118		− /// due to using different names or email addresses.
119		− #[clap(short = 'i', long)]
120		− pub omit_unify_identities: bool,
121		−}
	87	+#[cfg(feature = "gitoxide-core-tools")]
	88	+pub mod tools {
	89	+ use gix::bstr::BString;
	90	+ use std::path::PathBuf;
	91	+
	92	+ #[cfg(feature = "gitoxide-core-tools-query")]
	93	+ #[derive(Debug, clap::Parser)]
	94	+ #[command(
	95	+ about = "a database accelerated engine to extract information and query it",
	96	+ version = clap::crate_version!(),
	97	+ visible_alias = "q"
	98	+ )]
	99	+ pub struct Query {
	100	+ /// The total amount of object cache memory in MB. Bigger repos may benefit from more memory.
	101	+ ///
	102	+ /// 0 disables it.
	103	+ #[arg(long, short = 'o', default_value_t = 200)]
	104	+ pub object_cache_size_mb: usize,
	105	+ /// Find identical copies in the entire tree, not only in the set of modified files.
	106	+ ///
	107	+ /// This is an expensive option, and typically cuts speed in half.
	108	+ #[arg(long, short = 'C')]
	109	+ pub find_copies_harder: bool,
	110	+ /// path to the git repository to generate the database for
	111	+ #[arg(default_value = ".")]
	112	+ pub repo_dir: std::path::PathBuf,
	113	+ #[clap(subcommand)]
	114	+ pub cmd: Option<query::Command>,
	115	+ }
122	116
123		−mod validator {
124		− use std::{ffi::OsStr, path::PathBuf};
125		−
126		− use anyhow::Context;
127		−
128		− #[derive(Clone)]
129		− pub struct IsRepo;
130		−
131		− impl clap::builder::TypedValueParser for IsRepo {
132		− type Value = PathBuf;
133		−
134		− fn parse_ref(
135		− &self,
136		− cmd: &clap::Command,
137		− _arg: Option<&clap::Arg>,
138		− value: &OsStr,
139		− ) -> Result<Self::Value, clap::Error> {
140		− assure_is_repo(value).map_err(\|e\| {
141		− let mut err = clap::Error::new(clap::error::ErrorKind::InvalidValue).with_cmd(cmd);
142		− err.insert(
143		− clap::error::ContextKind::InvalidValue,
144		− clap::error::ContextValue::String(e.to_string()),
145		− );
146		− err
147		− })?;
148		− Ok(value.into())
	117	+ #[cfg(feature = "gitoxide-core-tools-query")]
	118	+ pub mod query {
	119	+ use crate::shared::AsPathSpec;
	120	+
	121	+ #[derive(Debug, clap::Subcommand)]
	122	+ pub enum Command {
	123	+ /// Follow a file through the entire history reachable from HEAD.
	124	+ TracePath {
	125	+ /// The path to trace through history.
	126	+ #[clap(value_parser = AsPathSpec)]
	127	+ path: gix::path::Spec,
	128	+ },
149	129	}
150	130	}
151	131
152		− fn assure_is_repo(dir: &OsStr) -> anyhow::Result<()> {
153		− let git_dir = PathBuf::from(dir).join(".git");
154		− let p = gix::path::realpath(&git_dir)
155		− .with_context(\|\| format!("Could not canonicalize git repository at '{}'", git_dir.display()))?;
156		− if p.extension().unwrap_or_default() == "git"
157		− \|\| p.file_name().unwrap_or_default() == ".git"
158		− \|\| p.join("HEAD").is_file()
159		− {
160		− Ok(())
161		− } else {
162		− Err(anyhow::anyhow!(
163		− "Path '{}' needs to be a directory containing '.git/'",
164		− p.display()
165		− ))
	132	+ #[derive(Debug, clap::Parser)]
	133	+ #[clap(
	134	+ about = "Estimate hours worked based on a commit history",
	135	+ long_about = "See https://github.com/kimmobrunfeldt/git-hours#how-it-works for details",
	136	+ version = clap::crate_version!(),
	137	+ visible_alias = "h",
	138	+ visible_alias = "hours"
	139	+ )]
	140	+ pub struct EstimateHours {
	141	+ /// The directory containing a '.git/' folder.
	142	+ #[clap(value_parser = validator::IsRepo)]
	143	+ #[clap(default_value = ".")]
	144	+ pub working_dir: PathBuf,
	145	+ /// The name of the revision as spec, like 'HEAD' or 'main' at which to start iterating the commit graph.
	146	+ #[clap(default_value("HEAD"), value_parser = crate::shared::AsBString)]
	147	+ pub rev_spec: BString,
	148	+ /// Ignore github bots which match the `[bot]` search string.
	149	+ #[clap(short = 'b', long)]
	150	+ pub no_bots: bool,
	151	+ /// Collect additional information about file modifications, additions and deletions (without rename tracking).
	152	+ #[clap(short = 'f', long)]
	153	+ pub file_stats: bool,
	154	+ /// Collect additional information about lines added and deleted (without rename tracking).
	155	+ ///
	156	+ /// Note that this implies the work to be done for file-stats, so it should be set as well.
	157	+ #[clap(short = 'l', long)]
	158	+ pub line_stats: bool,
	159	+ /// Show personally identifiable information before the summary. Includes names and email addresses.
	160	+ #[clap(short = 'p', long)]
	161	+ pub show_pii: bool,
	162	+ /// Omit unifying identities by name and email which can lead to the same author appear multiple times
	163	+ /// due to using different names or email addresses.
	164	+ #[clap(short = 'i', long)]
	165	+ pub omit_unify_identities: bool,
	166	+ }
	167	+
	168	+ mod validator {
	169	+ use std::{ffi::OsStr, path::PathBuf};
	170	+
	171	+ use anyhow::Context;
	172	+
	173	+ #[derive(Clone)]
	174	+ pub struct IsRepo;
	175	+
	176	+ impl clap::builder::TypedValueParser for IsRepo {
	177	+ type Value = PathBuf;
	178	+
	179	+ fn parse_ref(
	180	+ &self,
	181	+ cmd: &clap::Command,
	182	+ _arg: Option<&clap::Arg>,
	183	+ value: &OsStr,
	184	+ ) -> Result<Self::Value, clap::Error> {
	185	+ assure_is_repo(value).map_err(\|e\| {
	186	+ let mut err = clap::Error::new(clap::error::ErrorKind::InvalidValue).with_cmd(cmd);
	187	+ err.insert(
	188	+ clap::error::ContextKind::InvalidValue,
	189	+ clap::error::ContextValue::String(e.to_string()),
	190	+ );
	191	+ err
	192	+ })?;
	193	+ Ok(value.into())
	194	+ }
	195	+ }
	196	+
	197	+ fn assure_is_repo(dir: &OsStr) -> anyhow::Result<()> {
	198	+ let git_dir = PathBuf::from(dir).join(".git");
	199	+ let p = gix::path::realpath(&git_dir)
	200	+ .with_context(\|\| format!("Could not canonicalize git repository at '{}'", git_dir.display()))?;
	201	+ if p.extension().unwrap_or_default() == "git"
	202	+ \|\| p.file_name().unwrap_or_default() == ".git"
	203	+ \|\| p.join("HEAD").is_file()
	204	+ {
	205	+ Ok(())
	206	+ } else {
	207	+ Err(anyhow::anyhow!(
	208	+ "Path '{}' needs to be a directory containing '.git/'",
	209	+ p.display()
	210	+ ))
	211	+ }
166	212	}
167	213	}
168	214	}

src/shared.rs+16−1

@@ -291,5 +291,20 @@ mod clap { · 290 unmodified lines
291	291	Some(Box::new([PossibleValue::new("SHA1")].into_iter()))
292	292	}
293	293	}
	294	+
	295	+ use clap::builder::{OsStringValueParser, TypedValueParser};
	296	+
	297	+ #[derive(Clone)]
	298	+ pub struct AsPathSpec;
	299	+
	300	+ impl TypedValueParser for AsPathSpec {
	301	+ type Value = gix::path::Spec;
	302	+
	303	+ fn parse_ref(&self, cmd: &Command, arg: Option<&Arg>, value: &OsStr) -> Result<Self::Value, Error> {
	304	+ OsStringValueParser::new()
	305	+ .try_map(\|arg\| gix::path::Spec::try_from(arg.as_os_str()))
	306	+ .parse_ref(cmd, arg, value)
	307	+ }
	308	+ }
294	309	}
295		−pub use self::clap::{AsBString, AsHashKind, AsOutputFormat};
	310	+pub use self::clap::{AsBString, AsHashKind, AsOutputFormat, AsPathSpec};

tests/journey/ein.sh+2−0

@@ -140,12 +140,14 @@ title "Porcelain ${kind}" · 139 unmodified lines
140	140	}
141	141	)
142	142	)
	143	+ if test "$kind" != "max-pure"; then
143	144	(with "running with no further arguments"
144	145	it "succeeds and informs about possible operations" && {
145	146	WITH_SNAPSHOT="$snapshot/no-args-failure" \
146	147	expect_run_sh $WITH_CLAP_FAILURE "$exe t"
147	148	}
148	149	)
	150	+ fi
149	151	)
150	152	)
151	153	)

tests/snapshots/panic-behaviour/expected-failure+1−1

@@ -1,2 +1,2 @@
1		−thread 'main' panicked at 'something went very wrong', src/porcelain/main.rs:39:42
	1	+thread 'main' panicked at 'something went very wrong', src/porcelain/main.rs:41:42
2	2	note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

tests/snapshots/panic-behaviour/expected-failure-in-thread+1−1

@@ -1,3 +1,3 @@
1		−thread 'main' panicked at 'something went very wrong', src/porcelain/main.rs:39:42
	1	+thread 'main' panicked at 'something went very wrong', src/porcelain/main.rs:41:42
2	2	note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
3	3	[2K

tests/snapshots/panic-behaviour/expected-failure-in-thread-with-progress+1−1

@@ -1,3 +1,3 @@
1		−[?1049h[?25lthread '<unnamed>' panicked at 'something went very wrong', src/porcelain/main.rs:39:42
	1	+[?1049h[?25lthread '<unnamed>' panicked at 'something went very wrong', src/porcelain/main.rs:41:42
2	2	note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
3	3	[?25h[?1049l

tests/snapshots/porcelain/tool/no-args-failure+1−0

@@ -5,6 +5,7 @@ Usage: ein tool <COMMAND> · 4 unmodified lines
5	5	Commands:
6	6	find Find all repositories in a given directory
7	7	organize Move all repositories found in a directory into a structure matching their clone URLs
	8	+ query a database accelerated engine to extract information and query it [aliases: q]
8	9	estimate-hours Estimate hours worked based on a commit history [aliases: h, hours]
9	10	help Print this message or the help of the given subcommand(s)
10	11