use std::{    io,    path::Path,    slice::Chunks,    sync::atomic::{AtomicUsize, Ordering},}; use bstr::BStr;use filetime::FileTime;use gix_features::parallel::{Reduce, in_parallel_if};use gix_filter::pipeline::convert::ToGitOutcome;use gix_object::FindExt; #[cfg(windows)]use crate::fscache::{FsCache, Metadata as FsCacheMetadata};use crate::index_as_worktree::types::ConflictIndexEntry;use crate::{    AtomicU64, SymlinkCheck,    index_as_worktree::{        Change, Conflict, Context, EntryStatus, Outcome, VisitEntry, traits,        traits::{CompareBlobs, SubmoduleStatus, read_data::Stream},        types::{Error, Options},    },    is_dir_to_mode,}; /// Calculates the changes that need to be applied to an `index` to match the state of the `worktree` and makes them/// observable in `collector`, along with information produced by `compare` which gets to see blobs that may have changes, and/// `submodule` which can take a look at submodules in detail to produce status information (BASE version if its conflicting)./// `options` are used to configure the operation.////// Note `worktree` must be the root path of the worktree, not a path inside of the worktree.////// Note that `index` may require changes to be up-to-date with the working tree and avoid expensive computations by updating/// respective entries with stat information from the worktree, and its timestamp is adjusted to the current time for which it/// will be considered fresh. All changes that would be applied to the index are delegated to the caller, which receives these/// as [`EntryStatus`]./// The `pathspec` is used to determine which index entries to check for status in the first place.////// `objects` is used to access the version of an object in the object database for direct comparison.////// **It's important to note that the `index` should have its [timestamp updated](gix_index::State::set_timestamp()) with a timestamp/// from just before making this call *if* [entries were updated](Outcome::entries_to_update)**////// ### Note////// Technically, this function does more as it also provides additional information, like whether a file has conflicts,/// and files that were added with `git add` are shown as a special as well. It also provides updates to entry filesystem/// stats like `git status` would if it had to determine the hash./// If that happened, the index should be written back after updating the entries with these updated stats, see [Outcome::skipped].////// Thus, some care has to be taken to do the right thing when letting the index match the worktree by evaluating the changes observed/// by the `collector`.#[expect(clippy::too_many_arguments)]pub fn index_as_worktree<'index, T, U, Find, E>(    index: &'index gix_index::State,    worktree: &Path,    collector: &mut impl VisitEntry<'index, ContentChange = T, SubmoduleStatus = U>,    compare: impl CompareBlobs<Output = T> + Send + Clone,    submodule: impl SubmoduleStatus<Output = U, Error = E> + Send + Clone,    objects: Find,    progress: &mut dyn gix_features::progress::Progress,    Context {        pathspec,        stack,        filter,        should_interrupt,    }: Context<'_>,    options: Options,) -> Result<Outcome, Error>where    T: Send,    U: Send,    E: std::error::Error + Send + Sync + 'static,    Find: gix_object::Find + Send + Clone,{    // the order is absolutely critical here we use the old timestamp to detect racy index entries    // (modified at or after the last index update) during the index update we then set those    // entries size to 0 (see below) to ensure they keep showing up as racy and reset the timestamp.    let timestamp = index.timestamp();    let (chunk_size, thread_limit, _) = gix_features::parallel::optimize_chunk_size_and_thread_limit(        500, // just like git        index.entries().len().into(),        options.thread_limit,        None,    );     let range = index        .prefixed_entries_range(pathspec.common_prefix())        .unwrap_or(0..index.entries().len());     let (entries, path_backing) = (index.entries(), index.path_backing());    let mut num_entries = entries.len();    let entry_index_offset = range.start;    let entries = &entries[range];     let _span = gix_features::trace::detail!("gix_status::index_as_worktree",                                             num_entries = entries.len(),                                             chunk_size = chunk_size,                                             thread_limit = ?thread_limit);     let entries_skipped_by_common_prefix = num_entries - entries.len();    let (skipped_by_pathspec, skipped_by_entry_flags, symlink_metadata_calls, entries_to_update) = Default::default();    let (worktree_bytes, worktree_reads, odb_bytes, odb_reads, racy_clean) = Default::default();     num_entries = entries.len();    progress.init(entries.len().into(), gix_features::progress::count("files"));    let count = progress.counter();     let new_state = {        let options = &options;        let (skipped_by_pathspec, skipped_by_entry_flags) = (&skipped_by_pathspec, &skipped_by_entry_flags);        let (symlink_metadata_calls, entries_to_update) = (&symlink_metadata_calls, &entries_to_update);        let (racy_clean, worktree_bytes) = (&racy_clean, &worktree_bytes);        let (worktree_reads, odb_bytes, odb_reads) = (&worktree_reads, &odb_bytes, &odb_reads);        move |_| {            (                State {                    buf: Vec::new(),                    buf2: Vec::new(),                    attr_stack: stack,                    path_stack: SymlinkCheck::new(worktree.into()),                    timestamp,                    path_backing,                    filter,                    options,                    #[cfg(windows)]                    fscache: options.fscache.then(|| FsCache::new(worktree)),                     skipped_by_pathspec,                    skipped_by_entry_flags,                    symlink_metadata_calls,                    entries_to_update,                    racy_clean,                    worktree_reads,                    worktree_bytes,                    odb_reads,                    odb_bytes,                },                compare,                submodule,                objects,                pathspec,            )        }    };    in_parallel_if(        || true, // TODO: heuristic: when is parallelization not worth it? Git says 500 items per thread, but to 20 threads, we can be more fine-grained though.        gix_features::interrupt::Iter::new(            OffsetIter {                inner: entries.chunks(chunk_size),                offset: entry_index_offset,            },            should_interrupt,        ),        thread_limit,        new_state,        |(entry_offset, chunk_entries), (state, blobdiff, submdule, objects, pathspec)| {            let all_entries = index.entries();            let mut out = Vec::new();            let mut idx = 0;            while let Some(entry) = chunk_entries.get(idx) {                // Allow a cooperative early-out *within* a chunk: once `should_interrupt` is set                // (e.g. by a caller that only needs to know whether *any* change exists, like                // `Repository::is_dirty`), stop promptly instead of finishing the whole chunk.                // Without this, an in-flight chunk would run to completion (~several hundred entries)                // even after another worker already found the first change.                if should_interrupt.load(Ordering::Relaxed) {                    break;                }                let absolute_entry_index = entry_offset + idx;                if idx == 0 && entry.stage_raw() != 0 {                    let offset = entry_offset.checked_sub(1).and_then(|prev_idx| {                        let prev_entry = &all_entries[prev_idx];                        let entry_path = entry.path_in(state.path_backing);                        if prev_entry.stage_raw() == 0 || prev_entry.path_in(state.path_backing) != entry_path {                            // prev_entry (in previous chunk) does not belong to our conflict                            return None;                        }                        Conflict::try_from_entry(all_entries, state.path_backing, absolute_entry_index, entry_path)                            .map(|(_conflict, offset, _entries)| offset)                    });                    if let Some(entries_to_skip_as_conflict_originates_in_previous_chunk) = offset {                        // skip current entry as it's done, along with following conflict entries                        idx += entries_to_skip_as_conflict_originates_in_previous_chunk + 1;                        continue;                    }                }                let res = state.process(                    all_entries,                    entry,                    absolute_entry_index,                    pathspec,                    blobdiff,                    submdule,                    objects,                    &mut idx,                );                idx += 1;                count.fetch_add(1, Ordering::Relaxed);                if let Some(res) = res {                    out.push(res);                }            }            out        },        ReduceChange {            collector,            entries: index.entries(),        },    )?;     Ok(Outcome {        entries_to_process: num_entries,        entries_processed: count.load(Ordering::Relaxed),        entries_skipped_by_common_prefix,        entries_skipped_by_pathspec: skipped_by_pathspec.load(Ordering::Relaxed),        entries_skipped_by_entry_flags: skipped_by_entry_flags.load(Ordering::Relaxed),        entries_to_update: entries_to_update.load(Ordering::Relaxed),        symlink_metadata_calls: symlink_metadata_calls.load(Ordering::Relaxed),        racy_clean: racy_clean.load(Ordering::Relaxed),        worktree_files_read: worktree_reads.load(Ordering::Relaxed),        worktree_bytes: worktree_bytes.load(Ordering::Relaxed),        odb_objects_read: odb_reads.load(Ordering::Relaxed),        odb_bytes: odb_bytes.load(Ordering::Relaxed),    })} struct State<'a, 'b> {    buf: Vec<u8>,    buf2: Vec<u8>,    timestamp: FileTime,    /// This is the cheap stack that only assure that we don't go through symlinks.    /// It's always used to get the path to perform an lstat on.    path_stack: SymlinkCheck,    /// This is the expensive stack that will need to check for `.gitattributes` files each time    /// it changes directory. It's only used when we know we have to read a worktree file, which in turn    /// requires attributes to drive the filter configuration.    attr_stack: gix_worktree::Stack,    filter: gix_filter::Pipeline,    path_backing: &'b gix_index::PathStorageRef,    options: &'a Options,    /// Optional lazy worktree stats cache for faster status checks on Windows.    /// Lookups happen before falling back to per-file syscalls.    #[cfg(windows)]    fscache: Option<FsCache>,     skipped_by_pathspec: &'a AtomicUsize,    skipped_by_entry_flags: &'a AtomicUsize,    symlink_metadata_calls: &'a AtomicUsize,    entries_to_update: &'a AtomicUsize,    racy_clean: &'a AtomicUsize,    worktree_bytes: &'a AtomicU64,    worktree_reads: &'a AtomicUsize,    odb_bytes: &'a AtomicU64,    odb_reads: &'a AtomicUsize,} type StatusResult<'index, T, U> = Result<(&'index gix_index::Entry, usize, &'index BStr, EntryStatus<T, U>), Error>; impl<'index> State<'_, 'index> {    #[expect(clippy::too_many_arguments)]    fn process<T, U, Find, E>(        &mut self,        entries: &'index [gix_index::Entry],        entry: &'index gix_index::Entry,        entry_index: usize,        pathspec: &mut gix_pathspec::Search,        diff: &mut impl CompareBlobs<Output = T>,        submodule: &mut impl SubmoduleStatus<Output = U, Error = E>,        objects: &Find,        outer_entry_index: &mut usize,    ) -> Option<StatusResult<'index, T, U>>    where        E: std::error::Error + Send + Sync + 'static,        Find: gix_object::Find,    {        if entry.flags.intersects(            gix_index::entry::Flags::UPTODATE                | gix_index::entry::Flags::SKIP_WORKTREE                | gix_index::entry::Flags::ASSUME_VALID                | gix_index::entry::Flags::FSMONITOR_VALID,        ) {            self.skipped_by_entry_flags.fetch_add(1, Ordering::Relaxed);            return None;        }        let path = entry.path_in(self.path_backing);        let is_excluded = pathspec            .pattern_matching_relative_path(                path,                Some(entry.mode.is_submodule()),                &mut |relative_path, case, is_dir, out| {                    self.attr_stack                        .set_case(case)                        .at_entry(relative_path, Some(is_dir_to_mode(is_dir)), objects)                        .is_ok_and(|platform| platform.matching_attributes(out))                },            )            .is_none_or(|m| m.is_excluded());         if is_excluded {            self.skipped_by_pathspec.fetch_add(1, Ordering::Relaxed);            return None;        }        let status = if entry.stage_raw() != 0 {            Ok(                Conflict::try_from_entry(entries, self.path_backing, entry_index, path).map(                    |(conflict, offset, entries)| {                        *outer_entry_index += offset; // let out loop skip over entries related to the conflict                        EntryStatus::Conflict {                            summary: conflict,                            entries: Box::new({                                let mut a: [Option<ConflictIndexEntry>; 3] = Default::default();                                let src = entries.into_iter().map(|e| e.map(ConflictIndexEntry::from));                                for (a, b) in a.iter_mut().zip(src) {                                    *a = b;                                }                                a                            }),                        }                    },                ),            )        } else {            self.compute_status(entry, path, diff, submodule, objects)        };        match status {            Ok(None) => None,            Ok(Some(status)) => Some(Ok((entry, entry_index, path, status))),            Err(err) => Some(Err(err)),        }    }     /// # On how racy-git is handled here    ///    /// Basically the racy detection is a safety mechanism that ensures we can always just compare the stat    /// information between index and worktree and if they match we don't need to look at the content.    /// This usually just works but if a file updates quickly we could run into the following situation:    ///    /// * save file version `A` from disk into worktree (git add)    /// * file is changed so fast that the mtime doesn't change - *we only looks at seconds by default*    /// * file contents change but file-size stays the same, so `"foo" -> "bar"` has the same size but different content    ///    /// Now both `mtime` and `size`, and all other stat information, is the same but the file has actually changed.    /// This case is called *racily clean*. *The file should show up as changed but due to a data race it doesn't.*    /// This is the racy git problem.    ///    /// To solve this we do the following trick: Whenever we modify the index, which includes `git status`, we save the    /// current timestamp before the modification starts. This timestamp fundamentally represents a checkpoint of sorts.    /// We "promise" ourselves that after the modification finishes all entries modified before this timestamp have the    /// racy git problem resolved.    ///    /// So now when we modify the index we must resolve the racy git problem somehow. To do that we only need to look at    /// unchanged entries. Changed entries are not interesting since they are already showing up as changed anyway so there    /// isn't really a race-condition to worry about. This also explains why removing the `return` here doesn't have an apparent effect.    /// This entire branch here is just the optimization of "don't even look at index entries where the stat hasn't changed".    /// If we don't have this optimization the result shouldn't change, our status implementation will just be super slow :D    ///    /// We calculate whether this change is `racy_clean`, so if the last `timestamp` is before or the same as the `mtime` of the entry    /// which is what `new_stat.is_racy(..)` does in the branch, and only if we are sure that there is no race condition    /// do we `return` early. Since we don't `return` early we just do a full content comparison below,    /// which always yields the correct result, there is no race condition there.    ///    /// If a file showed up as racily clean and didn't change then we don't need to do anything. After this status check is    /// complete and the file won't show up as racily clean anymore, since it's mtime is now before the new timestamp.    /// However, if the file did actually change then we really ran into one of those rare race conditions in that case we,    /// and git does the same, set the size of the file in the index to 0. This will always make the file show up as changed.    /// This adds the need to treat all files of size 0 in the index as changed. This is not quite right of course because 0 sized files    /// could be entirely valid and unchanged. Therefore this only applies if the oid doesn't match the oid of an empty file,    /// which is a constant.    ///    /// Adapted from [here](https://github.com/GitoxideLabs/gitoxide/pull/805#discussion_r1164676777).    fn compute_status<T, U, Find, E>(        &mut self,        entry: &gix_index::Entry,        rela_path: &BStr,        diff: &mut impl CompareBlobs<Output = T>,        submodule: &mut impl SubmoduleStatus<Output = U, Error = E>,        objects: &Find,    ) -> Result<Option<EntryStatus<T, U>>, Error>    where        E: std::error::Error + Send + Sync + 'static,        Find: gix_object::Find,    {        let worktree_path = match self.path_stack.verified_path(gix_path::from_bstr(rela_path).as_ref()) {            Ok(path) => path,            Err(err) if crate::stack::is_symlink_step_error(&err) => return Ok(Some(Change::Removed.into())),            Err(err) if gix_fs::io_err::is_not_found(err.kind(), err.raw_os_error()) => {                return Ok(Some(Change::Removed.into()));            }            Err(err) => return Err(Error::Io(err.into())),        };         // Acquire metadata. On Windows we consult the precomputed stats first and        // only fall back to a syscall on miss; on other platforms per-file        // `lstat` is already fast, so we just do the syscall directly.        #[cfg(windows)]        let metadata = if let Some(cached) = self.fscache.as_mut().and_then(|c| c.get(rela_path)) {            FsCacheMetadata::Cached(cached)        } else {            self.symlink_metadata_calls.fetch_add(1, Ordering::Relaxed);            match live_metadata(worktree_path)? {                Some(md) => FsCacheMetadata::Live(md),                None => return Ok(Some(Change::Removed.into())),            }        };        #[cfg(not(windows))]        let metadata = {            self.symlink_metadata_calls.fetch_add(1, Ordering::Relaxed);            match live_metadata(worktree_path)? {                Some(md) => md,                None => return Ok(Some(Change::Removed.into())),            }        };         // Handle directory: index entries are normally only for files/symlinks.        // If a file turned into a directory it was removed.        // The only exception here are submodules which are part of the index despite being directories.        if metadata.is_dir() {            if entry.mode.is_submodule() {                let status = submodule                    .status(entry, rela_path)                    .map_err(|err| Error::SubmoduleStatus {                        rela_path: rela_path.into(),                        source: Box::new(err),                    })?;                return Ok(status.map(|status| Change::SubmoduleModification(status).into()));            } else {                return Ok(Some(Change::Removed.into()));            }        }         if entry.flags.contains(gix_index::entry::Flags::INTENT_TO_ADD) {            return Ok(Some(EntryStatus::IntentToAdd));        }         #[cfg(windows)]        let new_stat = metadata.to_stat()?;        #[cfg(not(windows))]        let new_stat = gix_index::entry::Stat::from_fs(&metadata)?;         #[cfg(windows)]        let mode_change = metadata.mode_change(entry.mode, self.options.fs.symlink, self.options.fs.executable_bit);        #[cfg(not(windows))]        let mode_change =            entry                .mode                .change_to_match_fs(&metadata, self.options.fs.symlink, self.options.fs.executable_bit);        let executable_bit_changed = match mode_change {            Some(gix_index::entry::mode::Change::Type { new_mode }) => {                return Ok(Some(                    Change::Type {                        worktree_mode: new_mode,                    }                    .into(),                ));            }            Some(gix_index::entry::mode::Change::ExecutableBit) => true,            None => false,        };         // We implement racy-git. See racy-git.txt in the git documentation for detailed documentation.        //        // A file is racy if:        // 1. Its `mtime` is at or after the last index timestamp and its entry stat information        //   matches the on-disk file, but the file contents are actually modified        // 2. Its size is 0 (set after detecting a file was racy previously)        //        // The first case is detected below by checking the timestamp if the file is marked unmodified.        // The second case is usually detected either because the on-disk file is not empty, hence        // the basic stat match fails, or by checking whether the size doesn't fit the oid.        let mut racy_clean = false;        if !executable_bit_changed            && new_stat.matches(&entry.stat, self.options.stat)            // TODO: find a test for the following line or remove it. Is this more often hit with smudge/clean filters?            && (!entry.id.is_empty_blob() || entry.stat.size == 0)        {            racy_clean = new_stat.is_racy(self.timestamp, self.options.stat);            if !racy_clean {                return Ok(None);            } else {                self.racy_clean.fetch_add(1, Ordering::Relaxed);            }        }         self.buf.clear();        self.buf2.clear();        let file_size_bytes = if cfg!(windows) && metadata.is_symlink() {            // symlinks on Windows seem to have a length of zero, so just pretend            // they have the correct length to avoid short-cutting, and enforce a full buffer check.            u64::from(entry.stat.size)        } else {            metadata.len()        };        let fetch_data = ReadDataImpl {            buf: &mut self.buf,            path: worktree_path,            rela_path,            entry,            file_len: file_size_bytes,

Lines 1–500 of 729. Load more