use collections::HashMap; use crate::{    example::ActualCursor,    reorder_patch::{Patch, PatchLine},    word_diff::{DiffOp, diff_tokens, tokenize},}; pub type Counts = HashMap<String, usize>;type CountsDelta = HashMap<String, isize>; /// Context characters needed on each side of a change to capture all affected n-gramsconst CONTEXT_CHARS: usize = CHR_F_CHAR_ORDER - 1; #[derive(Default, Debug, Clone)]pub struct ClassificationMetrics {    pub true_positives: usize,    pub false_positives: usize,    pub false_negatives: usize,} impl ClassificationMetrics {    pub fn from_counts(expected: &Counts, actual: &Counts) -> ClassificationMetrics {        let mut true_positives = 0;        let mut false_positives = 0;        let mut false_negatives = 0;         for (ngram, &expected_count) in expected {            let actual_count = *actual.get(ngram).unwrap_or(&0);            if actual_count > expected_count {                false_positives += actual_count - expected_count;            } else {                false_negatives += expected_count - actual_count;            }            true_positives += expected_count.min(actual_count);        }         for (ngram, &actual_count) in actual {            if !expected.contains_key(ngram) {                false_positives += actual_count;            }        }         ClassificationMetrics {            true_positives,            false_positives,            false_negatives,        }    }     pub fn precision(&self) -> f64 {        if self.true_positives + self.false_positives == 0 {            0.0        } else {            self.true_positives as f64 / (self.true_positives + self.false_positives) as f64        }    }     pub fn recall(&self) -> f64 {        if self.true_positives + self.false_negatives == 0 {            0.0        } else {            self.true_positives as f64 / (self.true_positives + self.false_negatives) as f64        }    }     pub fn f1(&self) -> f64 {        let precision = self.precision();        let recall = self.recall();        if precision + recall == 0.0 {            0.0        } else {            2.0 * precision * recall / (precision + recall)        }    }} enum ChrfWhitespace {    /// Preserve whitespace as-is    #[allow(unused)]    Unchanged,     /// Ignore all whitespace differences    #[allow(unused)]    Ignore,     /// Collapse whitespace into single spaces    Collapse,} const CHR_F_CHAR_ORDER: usize = 6;const CHR_F_BETA: f64 = 2.0;const CHR_F_WHITESPACE: ChrfWhitespace = ChrfWhitespace::Collapse; /// Computes a delta-chrF score that compares two sets of edits.////// This metric works by:/// 1. Computing n-gram count differences (deltas) between original→expected and original→actual/// 2. Comparing these deltas to measure how well actual edits match expected edits////// Returns a score from 0.0 to 100.0, where 100.0 means the actual edits perfectly match/// the expected edits.pub fn delta_chr_f(original: &str, expected: &str, actual: &str) -> f64 {    // Edge case: if all texts are identical, the edits match perfectly    if original == expected && expected == actual {        return 100.0;    }     // Pre-filter whitespace once for all texts    let orig_chars: Vec<char> = filter_whitespace_chars(original);    let exp_chars: Vec<char> = filter_whitespace_chars(expected);    let act_chars: Vec<char> = filter_whitespace_chars(actual);     // Find the changed regions between original→expected and original→actual    // We only need to compute n-grams on these regions (plus context for boundary n-grams)    let (orig_for_exp, exp_region) = extract_changed_regions(&orig_chars, &exp_chars);    let (orig_for_act, act_region) = extract_changed_regions(&orig_chars, &act_chars);     let mut total_precision = 0.0;    let mut total_recall = 0.0;     for order in 1..=CHR_F_CHAR_ORDER {        // Compute n-grams only on the affected regions        let orig_ngrams_for_exp = count_ngrams_from_chars(&orig_for_exp, order);        let exp_ngrams = count_ngrams_from_chars(&exp_region, order);        let expected_delta = compute_ngram_delta(&exp_ngrams, &orig_ngrams_for_exp);         let orig_ngrams_for_act = count_ngrams_from_chars(&orig_for_act, order);        let act_ngrams = count_ngrams_from_chars(&act_region, order);        let actual_delta = compute_ngram_delta(&act_ngrams, &orig_ngrams_for_act);         if expected_delta.is_empty() && actual_delta.is_empty() {            total_precision += 1.0;            total_recall += 1.0;            continue;        }         let expected_counts = ngram_delta_to_counts(&expected_delta);        let actual_counts = ngram_delta_to_counts(&actual_delta);         let score = ClassificationMetrics::from_counts(&expected_counts, &actual_counts);        total_precision += score.precision();        total_recall += score.recall();    }     let prec = total_precision / CHR_F_CHAR_ORDER as f64;    let recall = total_recall / CHR_F_CHAR_ORDER as f64;    let f_score = if prec + recall == 0.0 {        0.0    } else {        (1.0 + CHR_F_BETA * CHR_F_BETA) * prec * recall / (CHR_F_BETA * CHR_F_BETA * prec + recall)    };     f_score * 100.0} /// Reference implementation of delta_chr_f (original, non-optimized version)./// Used for testing that the optimized version produces identical results.#[cfg(test)]fn delta_chr_f_reference(original: &str, expected: &str, actual: &str) -> f64 {    if original == expected && expected == actual {        return 100.0;    }     let original_ngrams = chr_f_ngram_counts(original);    let expected_ngrams = chr_f_ngram_counts(expected);    let actual_ngrams = chr_f_ngram_counts(actual);     let mut total_precision = 0.0;    let mut total_recall = 0.0;     for order in 0..CHR_F_CHAR_ORDER {        let expected_delta = compute_ngram_delta(&expected_ngrams[order], &original_ngrams[order]);        let actual_delta = compute_ngram_delta(&actual_ngrams[order], &original_ngrams[order]);         if expected_delta.is_empty() && actual_delta.is_empty() {            total_precision += 1.0;            total_recall += 1.0;            continue;        }         let expected_counts = ngram_delta_to_counts(&expected_delta);        let actual_counts = ngram_delta_to_counts(&actual_delta);         let score = ClassificationMetrics::from_counts(&expected_counts, &actual_counts);        total_precision += score.precision();        total_recall += score.recall();    }     let prec = total_precision / CHR_F_CHAR_ORDER as f64;    let recall = total_recall / CHR_F_CHAR_ORDER as f64;    let f_score = if prec + recall == 0.0 {        0.0    } else {        (1.0 + CHR_F_BETA * CHR_F_BETA) * prec * recall / (CHR_F_BETA * CHR_F_BETA * prec + recall)    };     f_score * 100.0} /// Filter whitespace from a string and return as Vec<char>fn filter_whitespace_chars(text: &str) -> Vec<char> {    match CHR_F_WHITESPACE {        ChrfWhitespace::Unchanged => text.chars().collect(),        ChrfWhitespace::Ignore => text.chars().filter(|c| !c.is_whitespace()).collect(),        ChrfWhitespace::Collapse => collapse_whitespace(text.chars()),    }} /// Collapse whitespace into single spaces./// Newlines and spaces are collapsed separately.fn collapse_whitespace(chars: impl Iterator<Item = char>) -> Vec<char> {    let mut result = Vec::new();    let mut last_whitespace = None;    for c in chars {        if c.is_whitespace() && c != '\n' {            if last_whitespace != Some(' ') {                result.push(' ');                last_whitespace = Some(' ');            }        } else if c == '\n' {            if last_whitespace != Some('\n') {                result.push(c);                last_whitespace = Some('\n');            }        } else {            result.push(c);            last_whitespace = None;        }    }    result} /// Extract only the changed regions between two texts, with context for n-gram boundaries.////// Returns (original_affected_region, modified_affected_region) as Vec<char>.////// The key insight: when computing n-gram delta between two nearly-identical texts,/// n-grams from unchanged regions cancel out. We only need to process:/// 1. The changed content itself/// 2. CONTEXT_CHARS (n-1) characters before and after, to capture boundary-crossing n-gramsfn extract_changed_regions(original: &[char], modified: &[char]) -> (Vec<char>, Vec<char>) {    // Find longest common prefix    let prefix_len = original        .iter()        .zip(modified.iter())        .take_while(|(a, b)| a == b)        .count();     // Find longest common suffix (that doesn't overlap with prefix)    let orig_remaining = original.len().saturating_sub(prefix_len);    let mod_remaining = modified.len().saturating_sub(prefix_len);    let max_suffix = orig_remaining.min(mod_remaining);     let suffix_len = original        .iter()        .rev()        .zip(modified.iter().rev())        .take(max_suffix)        .take_while(|(a, b)| a == b)        .count();     // Calculate the changed region boundaries    let orig_change_start = prefix_len;    let orig_change_end = original.len().saturating_sub(suffix_len);    let mod_change_start = prefix_len;    let mod_change_end = modified.len().saturating_sub(suffix_len);     // If there's no actual change, return empty regions    if orig_change_start >= orig_change_end && mod_change_start >= mod_change_end {        return (Vec::new(), Vec::new());    }     // Expand to include context for n-gram boundaries    let orig_context_start = orig_change_start.saturating_sub(CONTEXT_CHARS);    let orig_context_end = (orig_change_end + CONTEXT_CHARS).min(original.len());    let mod_context_start = mod_change_start.saturating_sub(CONTEXT_CHARS);    let mod_context_end = (mod_change_end + CONTEXT_CHARS).min(modified.len());     let orig_region: Vec<char> = original[orig_context_start..orig_context_end].to_vec();    let mod_region: Vec<char> = modified[mod_context_start..mod_context_end].to_vec();     (orig_region, mod_region)} /// Count n-grams directly from a char slice (avoids String allocation for the full text)fn count_ngrams_from_chars(chars: &[char], n: usize) -> Counts {    let mut counts = Counts::default();     if chars.len() < n {        return counts;    }     for window in chars.windows(n) {        let ngram: String = window.iter().collect();        *counts.entry(ngram).or_insert(0) += 1;    }     counts} #[allow(dead_code)]fn chr_f_ngram_counts(text: &str) -> Vec<Counts> {    let text = match CHR_F_WHITESPACE {        ChrfWhitespace::Unchanged => text.to_string(),        ChrfWhitespace::Ignore => text            .chars()            .filter(|c| !c.is_whitespace())            .collect::<String>(),        ChrfWhitespace::Collapse => collapse_whitespace(text.chars())            .into_iter()            .collect::<String>(),    };     (1..=CHR_F_CHAR_ORDER)        .map(|order| count_ngrams(&text, order))        .collect()} fn compute_ngram_delta(after: &Counts, before: &Counts) -> CountsDelta {    let mut delta = CountsDelta::default();     for (ngram, &before_count) in before {        let after_count = *after.get(ngram).unwrap_or(&0);        delta.insert(ngram.clone(), after_count as isize - before_count as isize);    }     for (ngram, &after_count) in after {        if !before.contains_key(ngram) {            delta.insert(ngram.clone(), after_count as isize);        }    }     delta} /// Convert negative counts to special deletion tokens./// For example, if expected delta is {"foo": -1} and actual delta is {"bar": -1},/// we convert it to {"¬foo": +1} and {"¬bar": +1}. This way _not_ deleting "foo"/// will result in a false negative, and mistakenly deleting "bar" will result in a false positive.fn ngram_delta_to_counts(delta: &CountsDelta) -> Counts {    let mut counts = Counts::default();     for (ngram, &delta) in delta {        if delta > 0 {            counts.insert(ngram.clone(), delta as usize);        } else if delta < 0 {            counts.insert(format!("¬{ngram}"), delta.unsigned_abs());        }    }     counts} #[allow(dead_code)]fn count_ngrams(text: &str, n: usize) -> Counts {    let chars: Vec<char> = text.chars().collect();    let mut counts = Counts::default();     for window in chars.windows(n) {        let ngram: String = window.iter().collect();        *counts.entry(ngram).or_insert(0) += 1;    }     counts} pub fn braces_disbalance(text: &str) -> usize {    let mut disbalance = 0isize;     let a = text.chars().filter(|&c| c == '{').count() as isize;    let b = text.chars().filter(|&c| c == '}').count() as isize;    disbalance += (a - b).abs();     let a = text.chars().filter(|&c| c == '(').count() as isize;    let b = text.chars().filter(|&c| c == ')').count() as isize;    disbalance += (a - b).abs();     let a = text.chars().filter(|&c| c == '[').count() as isize;    let b = text.chars().filter(|&c| c == ']').count() as isize;    disbalance += (a - b).abs();     disbalance as usize} /// Extracts changed lines from a unified diff string./// Returns a bag (multiset) of lines that were added (+) or removed (-)./// The +/- prefix is included in the line to distinguish additions from deletions.pub fn extract_changed_lines_from_diff(diff: &str) -> Counts {    let mut counts = Counts::default();     for line in diff.lines() {        // Skip file headers (--- and +++)        if line.starts_with("---") || line.starts_with("+++") {            continue;        }        // Skip hunk headers (@@)        if line.starts_with("@@") {            continue;        }        // Skip diff header lines (diff --git, index, etc.)        if line.starts_with("diff ") || line.starts_with("index ") {            continue;        }        // Include added and removed lines (with their prefix)        if line.starts_with('+') || line.starts_with('-') {            *counts.entry(line.to_string()).or_insert(0) += 1;        }    }     counts} /// Computes exact lines match metrics between expected and actual patches./// Treats changed lines as a bag (multiset) - order is discarded but count matters./// Returns ClassificationMetrics with TP/FP/FN counts.pub fn exact_lines_match(expected_patch: &str, actual_patch: &str) -> ClassificationMetrics {    let expected_lines = extract_changed_lines_from_diff(expected_patch);    let actual_lines = extract_changed_lines_from_diff(actual_patch);    ClassificationMetrics::from_counts(&expected_lines, &actual_lines)} /// Returns whether the patch contains any isolated whitespace-only changes.////// A whitespace-only change is an added or deleted line whose content is empty or/// contains only whitespace. It is "isolated" when it is not adjacent to any/// substantive (non-whitespace) change within the same contiguous change group.pub fn has_isolated_whitespace_changes(patch_str: &str, cursor: Option<&ActualCursor>) -> bool {    let patch = Patch::parse_unified_diff(patch_str);     let cursor_new_file_line = cursor.as_ref().map(|c| (c.row + 1) as usize);     for hunk in &patch.hunks {        let lines = &hunk.lines;        let mut new_text_line = hunk.new_start as usize;         for (i, line) in lines.iter().enumerate() {            let content = match line {                PatchLine::Addition(s) => {                    let addition_line = new_text_line;                    new_text_line += 1;                    if s.trim().is_empty() && cursor_new_file_line == Some(addition_line) {                        continue;                    }                    s.as_str()                }                PatchLine::Deletion(s) => s.as_str(),                PatchLine::Context(_) => {                    new_text_line += 1;                    continue;                }                _ => continue,            };             if !content.trim().is_empty() {                continue;            }             if is_whitespace_change_isolated(lines, i) {                return true;            }        }    }     false} fn is_whitespace_change_isolated(lines: &[PatchLine], index: usize) -> bool {    // Look backward for a non-whitespace change before hitting a context line    for line in lines[..index].iter().rev() {        match line {            PatchLine::Addition(s) | PatchLine::Deletion(s) => {                if !s.trim().is_empty() {                    return false;                }            }            _ => break,        }    }     // Look forward for a non-whitespace change before hitting a context line    for line in &lines[index + 1..] {        match line {            PatchLine::Addition(s) | PatchLine::Deletion(s) => {                if !s.trim().is_empty() {                    return false;                }            }            _ => break,        }    }     true} /// A simple proxy for whether the prediction respects editable region.pub fn is_editable_region_correct(actual_patch: &str) -> bool {    // A typical sign of a wrong editable region: a bunch of lines deletion    // at the beginning or end of the patch.    let patch = Patch::parse_unified_diff(actual_patch);

Lines 1–500 of 1220. Load more