History

Merge pull request #2011 from blinxen/main

Add Sink that implements git's diffing improvement heuristics

Sebastian Thiel committed 1 year ago

Commit fd49eee2 parents 7ae3797 55056467 files · +497 −3Browse files

Unified · Split

7 files
  1. gix-diff/Cargo.toml +1 −1
  2. gix-diff/src/blob/git_diff.rs +326 −0
  3. gix-diff/src/blob/mod.rs +3 −0
  4. gix-diff/tests/diff/blob/git_diff.rs +164 −0
  5. gix-diff/tests/diff/blob/mod.rs +1 −0
  6. gix-merge/Cargo.toml +1 −1
  7. gix-merge/fuzz/Cargo.toml +1 −1
gix-diff/Cargo.toml+1−1
@@ -42,7 +42,7 @@ gix-trace = { version = "^0.1.12", path = "../gix-trace", optional = true } · 41 unmodified lines
42gix-traverse = { version = "^0.46.2", path = "../gix-traverse", optional = true }42gix-traverse = { version = "^0.46.2", path = "../gix-traverse", optional = true }
43 43
44thiserror = "2.0.0"44thiserror = "2.0.0"
45imara-diff = { version = "0.1.7", optional = true }45imara-diff = { version = "0.1.8", optional = true }
46serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }46serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
47getrandom = { version = "0.2.8", optional = true, default-features = false, features = ["js"] }47getrandom = { version = "0.2.8", optional = true, default-features = false, features = ["js"] }
48bstr = { version = "1.12.0", default-features = false }48bstr = { version = "1.12.0", default-features = false }
gix-diff/src/blob/git_diff.rs+326−0
@@ -0,0 +1,326 @@
1//! Facilities to produce git-formatted diffs.
2
3use crate::blob::GitDiff;
4use bstr::ByteSlice;
5use imara_diff::intern::{InternedInput, Interner, Token};
6use imara_diff::Sink;
7use std::cmp::Ordering;
8use std::ops::Range;
9
10// Explanation for the following numbers can be found here:
11// https://github.com/git/git/blob/324fbaab88126196bd42e7fa383ee94e165d61b5/xdiff/xdiffi.c#L535
12const MAX_INDENT: u8 = 200;
13const MAX_BLANKS: i16 = 20;
14const INDENT_WEIGHT: i16 = 60;
15const INDENT_HEURISTIC_MAX_SLIDING: usize = 100;
16
17const START_OF_FILE_PENALTY: i16 = 1;
18const END_OF_FILE_PENALTY: i16 = 21;
19const TOTAL_BLANK_WEIGHT: i16 = -30;
20const POST_BLANK_WEIGHT: i16 = 6;
21const RELATIVE_INDENT_PENALTY: i16 = -4;
22const RELATIVE_INDENT_WITH_BLANK_PENALTY: i16 = 10;
23const RELATIVE_OUTDENT_PENALTY: i16 = 24;
24const RELATIVE_OUTDENT_WITH_BLANK_PENALTY: i16 = 17;
25const RELATIVE_DEDENT_PENALTY: i16 = 23;
26const RELATIVE_DEDENT_WITH_BLANK_PENALTY: i16 = 17;
27
28pub(super) mod types {
29 use crate::blob::git_diff::ChangeGroup;
30
31 /// A [`Sink`](imara_diff::Sink) that creates a diff like git would.
32 ///
33 /// See the [diff slider repository](https://github.com/mhagger/diff-slider-tools) for more information.
34 pub struct GitDiff<'a, T>
35 where
36 T: AsRef<[u8]>,
37 {
38 pub(crate) after: &'a [imara_diff::intern::Token],
39 pub(crate) interner: &'a imara_diff::intern::Interner<T>,
40 pub(crate) changes: Vec<ChangeGroup>,
41 }
42}
43
44/// An enum indicating the kind of change that occurred.
45#[derive(PartialEq, Debug)]
46pub enum ChangeKind {
47 /// Indicates that a change introduced new lines.
48 Added,
49 /// Indicates that a change removed lines before the starting line of the change.
50 RemovedAbove,
51 /// Indicates that a change removed lines after the ending line of the change.
52 RemovedBelow,
53 /// Indicates that the change modified lines.
54 Modified,
55}
56
57#[derive(PartialEq)]
58struct Score {
59 effective_indent: i16,
60 penalty: i16,
61}
62
63impl PartialOrd for Score {
64 // A score is considered "Greater" if it is equal or less than 0
65 fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
66 let indent_penalty = match self.effective_indent.cmp(&other.effective_indent) {
67 Ordering::Greater => INDENT_WEIGHT,
68 Ordering::Less => -INDENT_WEIGHT,
69 Ordering::Equal => 0,
70 };
71
72 Some((indent_penalty + (self.penalty - other.penalty)).cmp(&0).reverse())
73 }
74}
75
76/// A [`ChangeGroup`] represents a block of changed lines.
77#[derive(PartialEq, Debug)]
78pub struct ChangeGroup {
79 /// Range indicating the lines of the previous block.
80 /// To actually see what the previous block looked like, you need to combine this range with
81 /// the [`InternedInput`].
82 pub before: Range<usize>,
83 /// Range indicating the lines of the new block
84 /// To actually see how the current block looks like, you need to combine this range with
85 /// the [`InternedInput`].
86 pub after: Range<usize>,
87 /// Further specify what kind of change is denoted by the ranges above.
88 pub change_kind: ChangeKind,
89}
90
91impl ChangeGroup {
92 /// Return [before](Self::before) and [after](Self::after) as `u32` ranges for use in [Sink::process_change()].
93 ///
94 /// This is useful for creating [unified diffs](crate::blob::UnifiedDiff), for example.
95 pub fn as_u32_ranges(&self) -> (Range<u32>, Range<u32>) {
96 (
97 self.before.start as u32..self.before.end as u32,
98 self.after.start as u32..self.after.end as u32,
99 )
100 }
101}
102
103// Calculate the indentation of a single line as number of tabs.
104fn get_indent(s: &[u8]) -> Option<u8> {
105 let mut indent = 0;
106
107 for char in s.bytes() {
108 if !char.is_ascii_whitespace() {
109 return Some(indent);
110 } else if char == b' ' {
111 indent += 1;
112 } else if char == b'\t' {
113 indent += 8 - indent % 8;
114 }
115
116 if indent >= MAX_INDENT {
117 return Some(MAX_INDENT);
118 }
119 }
120
121 None
122}
123
124fn measure_and_score_change<T: AsRef<[u8]>>(lines: &[Token], split: usize, interner: &Interner<T>, score: &mut Score) {
125 // Gather information about the surroundings of the change
126 let end_of_file = split >= lines.len();
127 let mut indent: Option<u8> = if split >= lines.len() {
128 None
129 } else {
130 get_indent(interner[lines[split]].as_ref())
131 };
132 let mut pre_blank = 0;
133 let mut pre_indent: Option<u8> = None;
134 let mut post_blank = 0;
135 let mut post_indent: Option<u8> = None;
136
137 for line in (0..=split.saturating_sub(1)).rev() {
138 pre_indent = get_indent(interner[lines[line]].as_ref());
139 if pre_indent.is_none() {
140 pre_blank += 1;
141 if pre_blank == MAX_BLANKS {
142 pre_indent = Some(0);
143 break;
144 }
145 }
146 }
147 for line in split + 1..lines.len() {
148 post_indent = get_indent(interner[lines[line]].as_ref());
149 if post_indent.is_none() {
150 post_blank += 1;
151 if post_blank == MAX_BLANKS {
152 post_indent = Some(0);
153 break;
154 }
155 }
156 }
157
158 // Calculate score of the currently applied split
159 post_blank = if indent.is_none() { 1 + post_blank } else { 0 };
160 let total_blank = pre_blank + post_blank;
161 if indent.is_none() {
162 indent = post_indent;
163 }
164 let any_blanks = total_blank != 0;
165
166 if pre_indent.is_none() && pre_blank == 0 {
167 score.penalty += START_OF_FILE_PENALTY;
168 }
169
170 if end_of_file {
171 score.penalty += END_OF_FILE_PENALTY;
172 }
173
174 score.penalty += TOTAL_BLANK_WEIGHT * total_blank;
175 score.penalty += POST_BLANK_WEIGHT * post_blank;
176
177 score.effective_indent += if let Some(indent) = indent { indent as i16 } else { -1 };
178
179 if indent.is_none() || pre_indent.is_none() || indent == pre_indent {
180 } else if indent > pre_indent {
181 score.penalty += if any_blanks {
182 RELATIVE_INDENT_WITH_BLANK_PENALTY
183 } else {
184 RELATIVE_INDENT_PENALTY
185 };
186 } else if post_indent.is_some() && post_indent > indent {
187 score.penalty += if any_blanks {
188 RELATIVE_OUTDENT_WITH_BLANK_PENALTY
189 } else {
190 RELATIVE_OUTDENT_PENALTY
191 };
192 } else {
193 score.penalty += if any_blanks {
194 RELATIVE_DEDENT_WITH_BLANK_PENALTY
195 } else {
196 RELATIVE_DEDENT_PENALTY
197 };
198 }
199}
200
201impl<'a, T> GitDiff<'a, T>
202where
203 T: AsRef<[u8]>,
204{
205 /// Create a new instance of [`GitDiff`] that can then be passed to [`imara_diff::diff`]
206 /// and generate a more human-readable diff.
207 pub fn new(input: &'a InternedInput<T>) -> Self {
208 Self {
209 after: &input.after,
210 interner: &input.interner,
211 changes: Vec::new(),
212 }
213 }
214}
215
216impl<T> Sink for GitDiff<'_, T>
217where
218 T: AsRef<[u8]>,
219{
220 type Out = Vec<ChangeGroup>;
221
222 fn process_change(&mut self, before: Range<u32>, after: Range<u32>) {
223 let change_kind = match (before.is_empty(), after.is_empty()) {
224 (true, false) => ChangeKind::Added,
225 (false, true) => {
226 if after.start == 0 {
227 ChangeKind::RemovedAbove
228 } else {
229 ChangeKind::RemovedBelow
230 }
231 }
232 _ => ChangeKind::Modified,
233 };
234 self.changes.push(ChangeGroup {
235 before: before.start as usize..before.end as usize,
236 after: after.start as usize..after.end as usize,
237 change_kind,
238 });
239 }
240
241 fn finish(mut self) -> Self::Out {
242 if self.changes.is_empty() {
243 return self.changes;
244 }
245
246 let mut shift: usize;
247 for change in &mut self.changes {
248 // Skip one-liner changes
249 if change.after.is_empty() {
250 continue;
251 }
252
253 // Move this change up by one line if the line before the change and the last line in
254 // the change are equal
255 loop {
256 if change.after.start > 0 && self.after[change.after.start - 1] == self.after[change.after.end - 1] {
257 change.after.start -= 1;
258 change.after.end -= 1;
259 } else {
260 break;
261 }
262 }
263
264 shift = change.after.end;
265
266 // Move this change down by one line if the first line in the change the line after the
267 // change are equal
268 loop {
269 if change.after.end < self.after.len() && self.after[change.after.start] == self.after[change.after.end]
270 {
271 change.after.start += 1;
272 change.after.end += 1;
273 } else {
274 break;
275 }
276 }
277
278 let mut best_shift: Option<usize> = None;
279 let mut best_score = Score {
280 effective_indent: 0,
281 penalty: 0,
282 };
283
284 if change.after.end.saturating_sub(change.after.len()) > shift {
285 shift = change.after.end - change.after.len();
286 }
287
288 if change.after.end.saturating_sub(INDENT_HEURISTIC_MAX_SLIDING) > shift {
289 shift = change.after.end - INDENT_HEURISTIC_MAX_SLIDING;
290 }
291
292 while shift <= change.after.end {
293 let mut score = Score {
294 effective_indent: 0,
295 penalty: 0,
296 };
297
298 measure_and_score_change(self.after, shift, self.interner, &mut score);
299 measure_and_score_change(self.after, shift - change.after.len(), self.interner, &mut score);
300
301 if best_shift.is_none() || score > best_score {
302 best_score = score;
303 best_shift = Some(shift);
304 }
305 shift += 1;
306 }
307
308 if let Some(best_shift) = best_shift {
309 while change.after.end > best_shift {
310 loop {
311 if change.after.start > 0
312 && self.after[change.after.start - 1] == self.after[change.after.end - 1]
313 {
314 change.after.start -= 1;
315 change.after.end -= 1;
316 } else {
317 break;
318 }
319 }
320 }
321 }
322 }
323
324 self.changes
325 }
326}
gix-diff/src/blob/mod.rs+3−0
@@ -14,6 +14,9 @@ pub mod platform; · 13 unmodified lines
14pub mod unified_diff;14pub mod unified_diff;
15pub use unified_diff::_impl::UnifiedDiff;15pub use unified_diff::_impl::UnifiedDiff;
16 16
17pub mod git_diff;
18pub use git_diff::types::GitDiff;
19
17/// Information about the diff performed to detect similarity.20/// Information about the diff performed to detect similarity.
18#[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)]21#[derive(Debug, Default, Clone, Copy, PartialEq, PartialOrd)]
19pub struct DiffLineStats {22pub struct DiffLineStats {
gix-diff/tests/diff/blob/git_diff.rs+164−0
@@ -0,0 +1,164 @@
1use gix_diff::blob::intern::InternedInput;
2use gix_diff::blob::unified_diff::{ContextSize, NewlineSeparator};
3use gix_diff::blob::{
4 git_diff::{ChangeGroup, ChangeKind},
5 Algorithm, GitDiff, Sink, UnifiedDiff,
6};
7use std::hash::Hash;
8
9#[test]
10fn basic() -> crate::Result {
11 let before = r#"struct SomeStruct {
12 field1: f64,
13 field2: f64,
14}
15
16fn main() {
17 // Some comment
18 let c = SomeStruct { field1: 10.0, field2: 10.0 };
19
20 println!(
21 "Print field1 from SomeStruct {}",
22 get_field1(&c)
23 );
24}
25
26fn get_field1(c: &SomeStruct) -> f64 {
27 c.field1
28}
29"#;
30
31 let after = r#"/// This is a struct
32struct SomeStruct {
33 field1: f64,
34 field2: f64,
35}
36
37fn main() {
38 let c = SomeStruct { field1: 10.0, field2: 10.0 };
39
40 println!(
41 "Print field1 and field2 from SomeStruct {} {}",
42 get_field1(&c), get_field2(&c)
43 );
44 println!("Print another line");
45}
46
47fn get_field1(c: &SomeStruct) -> f64 {
48 c.field1
49}
50
51fn get_field2(c: &SomeStruct) -> f64 {
52 c.field2
53}
54"#;
55 use crate::blob::git_diff::ChangeKind;
56
57 let expected = &[
58 ChangeGroup {
59 before: 0..0,
60 after: 0..1,
61 change_kind: ChangeKind::Added,
62 },
63 ChangeGroup {
64 before: 6..7,
65 after: 7..7,
66 change_kind: ChangeKind::RemovedBelow,
67 },
68 ChangeGroup {
69 before: 10..12,
70 after: 10..12,
71 change_kind: ChangeKind::Modified,
72 },
73 ChangeGroup {
74 before: 13..13,
75 after: 13..14,
76 change_kind: ChangeKind::Added,
77 },
78 ChangeGroup {
79 before: 17..17,
80 after: 19..23,
81 change_kind: ChangeKind::Added,
82 },
83 ];
84
85 let input = InternedInput::new(before, after);
86 let actual = gix_diff::blob::diff(Algorithm::Histogram, &input, GitDiff::new(&input));
87 assert_eq!(actual, expected);
88 insta::assert_snapshot!(uni_diff_string(&input, actual), @r#"
89 @@ -1,1 +1,2 @@
90 +/// This is a struct
91 struct SomeStruct {
92 @@ -6,3 +7,2 @@
93 fn main() {
94 - // Some comment
95 let c = SomeStruct { field1: 10.0, field2: 10.0 };
96 @@ -10,5 +10,6 @@
97 println!(
98 - "Print field1 from SomeStruct {}",
99 - get_field1(&c)
100 + "Print field1 and field2 from SomeStruct {} {}",
101 + get_field1(&c), get_field2(&c)
102 );
103 + println!("Print another line");
104 }
105 @@ -17,2 +19,6 @@
106 c.field1
107 +
108 +fn get_field2(c: &SomeStruct) -> f64 {
109 + c.field2
110 +}
111 }
112 "#);
113
114 let standard_slider = gix_diff::blob::diff(Algorithm::Histogram, &input, uni_diff(&input))?;
115 insta::assert_snapshot!(standard_slider, @r#"
116 @@ -1,1 +1,2 @@
117 +/// This is a struct
118 struct SomeStruct {
119 @@ -6,3 +7,2 @@
120 fn main() {
121 - // Some comment
122 let c = SomeStruct { field1: 10.0, field2: 10.0 };
123 @@ -10,5 +10,6 @@
124 println!(
125 - "Print field1 from SomeStruct {}",
126 - get_field1(&c)
127 + "Print field1 and field2 from SomeStruct {} {}",
128 + get_field1(&c), get_field2(&c)
129 );
130 + println!("Print another line");
131 }
132 @@ -17,2 +18,6 @@
133 c.field1
134 +}
135 +
136 +fn get_field2(c: &SomeStruct) -> f64 {
137 + c.field2
138 }
139 "#);
140
141 let input = InternedInput::new(before.as_bytes(), after.as_bytes());
142 let actual = gix_diff::blob::diff(Algorithm::Histogram, &input, GitDiff::new(&input));
143 assert_eq!(actual, expected);
144
145 Ok(())
146}
147
148fn uni_diff<T: Eq + Hash + AsRef<[u8]>>(input: &InternedInput<T>) -> UnifiedDiff<'_, T, String> {
149 UnifiedDiff::new(
150 input,
151 String::default(),
152 NewlineSeparator::AfterHeaderAndLine("\n"),
153 ContextSize::symmetrical(1),
154 )
155}
156
157fn uni_diff_string<T: Eq + Hash + AsRef<[u8]>>(input: &InternedInput<T>, changes: Vec<ChangeGroup>) -> String {
158 let mut uni = uni_diff(input);
159 for change in changes {
160 let (before, after) = change.as_u32_ranges();
161 uni.process_change(before, after);
162 }
163 uni.finish().expect("in-memory is infallible")
164}
gix-diff/tests/diff/blob/mod.rs+1−0
@@ -1,3 +1,4 @@
1mod git_diff;
1pub(crate) mod pipeline;2pub(crate) mod pipeline;
2mod platform;3mod platform;
3mod unified_diff;4mod unified_diff;
gix-merge/Cargo.toml+1−1
@@ -35,7 +35,7 @@ gix-diff = { version = "^0.52.1", path = "../gix-diff", default-features = false · 34 unmodified lines
35gix-index = { version = "^0.40.1", path = "../gix-index" }35gix-index = { version = "^0.40.1", path = "../gix-index" }
36 36
37thiserror = "2.0.0"37thiserror = "2.0.0"
38imara-diff = { version = "0.1.7" }38imara-diff = { version = "0.1.8" }
39bstr = { version = "1.12.0", default-features = false }39bstr = { version = "1.12.0", default-features = false }
40serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }40serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
41 41
gix-merge/fuzz/Cargo.toml+1−1
@@ -12,7 +12,7 @@ cargo-fuzz = true · 11 unmodified lines
12anyhow = "1.0.76"12anyhow = "1.0.76"
13libfuzzer-sys = "0.4"13libfuzzer-sys = "0.4"
14arbitrary = { version = "1.3.2", features = ["derive"] }14arbitrary = { version = "1.3.2", features = ["derive"] }
15imara-diff = { version = "0.1.7" }15imara-diff = { version = "0.1.8" }
16gix-merge = { path = ".." }16gix-merge = { path = ".." }
17 17
18# Prevent this from interfering with workspaces18# Prevent this from interfering with workspaces