118 lines · 4.8 KB · Rust
RawDownload
//! Filesystem utilities//!//! These are will be parallel if the `parallel` feature is enabled, at the expense of compiling additional dependencies//! along with runtime costs for maintaining a global [`rayon`](https://docs.rs/rayon) thread pool.//!//! For information on how to use the [`WalkDir`] type, have a look at//! * [`jwalk::WalkDir`](https://docs.rs/jwalk/0.5.1/jwalk/type.WalkDir.html) if `parallel` feature is enabled//! * [walkdir::WalkDir](https://docs.rs/walkdir/2.3.1/walkdir/struct.WalkDir.html) otherwise #[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]mod shared {    /// The desired level of parallelism.    pub enum Parallelism {        /// Do not parallelize at all by making a serial traversal on the current thread.        Serial,        /// Create a new thread pool for each traversal with up to 16 threads or the amount of logical cores of the machine.        ThreadPoolPerTraversal {            /// The base name of the threads we create as part of the thread-pool.            thread_name: &'static str,        },    }} ///#[cfg(feature = "fs-walkdir-parallel")]pub mod walkdir {    use std::path::Path;     pub use jwalk::{DirEntry as DirEntryGeneric, DirEntryIter as DirEntryIterGeneric, Error, WalkDir};     pub use super::shared::Parallelism;     /// An alias for an uncustomized directory entry to match the one of the non-parallel version offered by `walkdir`.    pub type DirEntry = DirEntryGeneric<((), ())>;     impl From<Parallelism> for jwalk::Parallelism {        fn from(v: Parallelism) -> Self {            match v {                Parallelism::Serial => jwalk::Parallelism::Serial,                Parallelism::ThreadPoolPerTraversal { thread_name } => std::thread::available_parallelism()                    .map_or_else(                        |_| Parallelism::Serial.into(),                        |threads| {                            let pool = jwalk::rayon::ThreadPoolBuilder::new()                                .num_threads(threads.get().min(16))                                .stack_size(128 * 1024)                                .thread_name(move |idx| format!("{thread_name} {idx}"))                                .build()                                .expect("we only set options that can't cause a build failure");                            jwalk::Parallelism::RayonExistingPool {                                pool: pool.into(),                                busy_timeout: None,                            }                        },                    ),            }        }    }     /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.    pub fn walkdir_new(root: impl AsRef<Path>, parallelism: Parallelism) -> WalkDir {        WalkDir::new(root).skip_hidden(false).parallelism(parallelism.into())    }     /// Instantiate a new directory iterator which will not skip hidden files and is sorted    pub fn walkdir_sorted_new(root: impl AsRef<Path>, parallelism: Parallelism) -> WalkDir {        WalkDir::new(root)            .skip_hidden(false)            .sort(true)            .parallelism(parallelism.into())    }     /// The Iterator yielding directory items    pub type DirEntryIter = DirEntryIterGeneric<((), ())>;} #[cfg(all(feature = "walkdir", not(feature = "fs-walkdir-parallel")))]///pub mod walkdir {    use std::path::Path;     pub use walkdir::{DirEntry, Error, WalkDir};     pub use super::shared::Parallelism;     /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.    pub fn walkdir_new(root: impl AsRef<Path>, _: Parallelism) -> WalkDir {        WalkDir::new(root)    }     /// Instantiate a new directory iterator which will not skip hidden files and is sorted, with the given level of `parallelism`.    pub fn walkdir_sorted_new(root: impl AsRef<Path>, _: Parallelism) -> WalkDir {        WalkDir::new(root).sort_by_file_name()    }     /// The Iterator yielding directory items    pub type DirEntryIter = walkdir::IntoIter;} #[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]pub use self::walkdir::{walkdir_new, walkdir_sorted_new, WalkDir}; /// Prepare open options which won't follow symlinks when the file is opened.////// Note: only effective on unix currently.pub fn open_options_no_follow() -> std::fs::OpenOptions {    #[cfg_attr(not(unix), allow(unused_mut))]    let mut options = std::fs::OpenOptions::new();    #[cfg(unix)]    {        /// Make sure that it's impossible to follow through to the target of symlinks.        /// Note that this will still follow symlinks in the path, which is what we assume        /// has been checked separately.        use std::os::unix::fs::OpenOptionsExt;        options.custom_flags(libc::O_NOFOLLOW);    }    options}