use aho_corasick::{AhoCorasick, AhoCorasickBuilder};use anyhow::Result;use client::proto;use fancy_regex::{Captures, Regex, RegexBuilder};use gpui::Entity;use itertools::Itertools as _;use language::{Buffer, BufferSnapshot, CharKind};use smol::future::yield_now;use std::{ borrow::Cow, io::{BufRead, BufReader, Read}, ops::Range, sync::{Arc, LazyLock},};use text::Anchor;use util::{ paths::{PathMatcher, PathStyle}, rel_path::RelPath,}; #[derive(Debug)]pub enum SearchResult { Buffer { buffer: Entity<Buffer>, ranges: Vec<Range<Anchor>>, }, LimitReached,} #[derive(Clone, Copy, PartialEq)]pub enum SearchInputKind { Query, Include, Exclude,} #[derive(Clone, Debug)]pub struct SearchInputs { query: Arc<str>, files_to_include: PathMatcher, files_to_exclude: PathMatcher, match_full_paths: bool, buffers: Option<Vec<Entity<Buffer>>>,} impl SearchInputs { pub fn as_str(&self) -> &str { self.query.as_ref() } pub fn files_to_include(&self) -> &PathMatcher { &self.files_to_include } pub fn files_to_exclude(&self) -> &PathMatcher { &self.files_to_exclude } pub fn buffers(&self) -> &Option<Vec<Entity<Buffer>>> { &self.buffers }}#[derive(Clone, Debug)]pub enum SearchQuery { Text { search: AhoCorasick, replacement: Option<String>, whole_word: bool, case_sensitive: bool, include_ignored: bool, inner: SearchInputs, }, Regex { regex: Regex, replacement: Option<String>, multiline: bool, whole_word: bool, case_sensitive: bool, include_ignored: bool, one_match_per_line: bool, inner: SearchInputs, },} static WORD_MATCH_TEST: LazyLock<Regex> = LazyLock::new(|| { RegexBuilder::new(r"\B") .build() .expect("Failed to create WORD_MATCH_TEST")}); impl SearchQuery { /// Create a text query /// /// If `match_full_paths` is true, include/exclude patterns will always be matched against fully qualified project paths beginning with a project root. /// If `match_full_paths` is false, patterns will be matched against worktree-relative paths. pub fn text( query: impl ToString, whole_word: bool, case_sensitive: bool, include_ignored: bool, files_to_include: PathMatcher, files_to_exclude: PathMatcher, match_full_paths: bool, buffers: Option<Vec<Entity<Buffer>>>, ) -> Result<Self> { let query = query.to_string(); if !case_sensitive && !query.is_ascii() { // AhoCorasickBuilder doesn't support case-insensitive search with unicode characters // Fallback to regex search as recommended by // https://docs.rs/aho-corasick/1.1/aho_corasick/struct.AhoCorasickBuilder.html#method.ascii_case_insensitive return Self::regex( regex::escape(&query), whole_word, case_sensitive, include_ignored, false, files_to_include, files_to_exclude, false, buffers, ); } let search = AhoCorasickBuilder::new() .ascii_case_insensitive(!case_sensitive) .build([&query])?; let inner = SearchInputs { query: query.into(), files_to_exclude, files_to_include, match_full_paths, buffers, }; Ok(Self::Text { search, replacement: None, whole_word, case_sensitive, include_ignored, inner, }) } /// Create a regex query /// /// If `match_full_paths` is true, include/exclude patterns will be matched against fully qualified project paths /// beginning with a project root name. If false, they will be matched against project-relative paths (which don't start /// with their respective project root). pub fn regex( query: impl ToString, whole_word: bool, mut case_sensitive: bool, include_ignored: bool, one_match_per_line: bool, files_to_include: PathMatcher, files_to_exclude: PathMatcher, match_full_paths: bool, buffers: Option<Vec<Entity<Buffer>>>, ) -> Result<Self> { let mut query = query.to_string(); let initial_query = Arc::from(query.as_str()); if let Some((case_sensitive_from_pattern, new_query)) = Self::case_sensitive_from_pattern(&query) { case_sensitive = case_sensitive_from_pattern; query = new_query } if whole_word { let mut word_query = String::new(); if let Some(first) = query.get(0..1) && WORD_MATCH_TEST.is_match(first).is_ok_and(|x| !x) { word_query.push_str("\\b"); } word_query.push_str(&query); if let Some(last) = query.get(query.len() - 1..) && WORD_MATCH_TEST.is_match(last).is_ok_and(|x| !x) { word_query.push_str("\\b"); } query = word_query } let multiline = query.contains('\n') || query.contains("\\n"); if multiline { query.insert_str(0, "(?m)"); } let regex = RegexBuilder::new(&query) .case_insensitive(!case_sensitive) .build()?; let inner = SearchInputs { query: initial_query, files_to_exclude, files_to_include, match_full_paths, buffers, }; Ok(Self::Regex { regex, replacement: None, multiline, whole_word, case_sensitive, include_ignored, inner, one_match_per_line, }) } /// Extracts case sensitivity settings from pattern items in the provided /// query and returns the same query, with the pattern items removed. /// /// The following pattern modifiers are supported: /// /// - `\c` (case_sensitive: false) /// - `\C` (case_sensitive: true) /// /// If no pattern item were found, `None` will be returned. fn case_sensitive_from_pattern(query: &str) -> Option<(bool, String)> { if !(query.contains("\\c") || query.contains("\\C")) { return None; } let mut was_escaped = false; let mut new_query = String::new(); let mut is_case_sensitive = None; for c in query.chars() { if was_escaped { if c == 'c' { is_case_sensitive = Some(false); } else if c == 'C' { is_case_sensitive = Some(true); } else { new_query.push('\\'); new_query.push(c); } was_escaped = false } else if c == '\\' { was_escaped = true } else { new_query.push(c); } } is_case_sensitive.map(|c| (c, new_query)) } pub fn from_proto(message: proto::SearchQuery, path_style: PathStyle) -> Result<Self> { let files_to_include = if message.files_to_include.is_empty() { message .files_to_include_legacy .split(',') .map(str::trim) .filter(|&glob_str| !glob_str.is_empty()) .map(|s| s.to_string()) .collect() } else { message.files_to_include }; let files_to_exclude = if message.files_to_exclude.is_empty() { message .files_to_exclude_legacy .split(',') .map(str::trim) .filter(|&glob_str| !glob_str.is_empty()) .map(|s| s.to_string()) .collect() } else { message.files_to_exclude }; if message.regex { Self::regex( message.query, message.whole_word, message.case_sensitive, message.include_ignored, false, PathMatcher::new(files_to_include, path_style)?, PathMatcher::new(files_to_exclude, path_style)?, message.match_full_paths, None, // search opened only don't need search remote ) } else { Self::text( message.query, message.whole_word, message.case_sensitive, message.include_ignored, PathMatcher::new(files_to_include, path_style)?, PathMatcher::new(files_to_exclude, path_style)?, message.match_full_paths, None, // search opened only don't need search remote ) } } pub fn with_replacement(mut self, new_replacement: String) -> Self { match self { Self::Text { ref mut replacement, .. } | Self::Regex { ref mut replacement, .. } => { *replacement = Some(new_replacement); self } } } pub fn to_proto(&self) -> proto::SearchQuery { let mut files_to_include = self.files_to_include().sources(); let mut files_to_exclude = self.files_to_exclude().sources(); proto::SearchQuery { query: self.as_str().to_string(), regex: self.is_regex(), whole_word: self.whole_word(), case_sensitive: self.case_sensitive(), include_ignored: self.include_ignored(), files_to_include: files_to_include.clone().map(ToOwned::to_owned).collect(), files_to_exclude: files_to_exclude.clone().map(ToOwned::to_owned).collect(), match_full_paths: self.match_full_paths(), // Populate legacy fields for backwards compatibility files_to_include_legacy: files_to_include.join(","), files_to_exclude_legacy: files_to_exclude.join(","), } } pub(crate) async fn detect( &self, mut reader: BufReader<Box<dyn Read + Send + Sync>>, ) -> Result<bool> { let query_str = self.as_str(); if query_str.is_empty() { return Ok(false); } // Yield from this function every 20KB scanned. const YIELD_THRESHOLD: usize = 20 * 1024; match self { Self::Text { search, .. } => { let mut text = String::new(); if query_str.contains('\n') { reader.read_to_string(&mut text)?; Ok(search.is_match(&text)) } else { let mut bytes_read = 0; while reader.read_line(&mut text)? > 0 { if search.is_match(&text) { return Ok(true); } bytes_read += text.len(); if bytes_read >= YIELD_THRESHOLD { bytes_read = 0; smol::future::yield_now().await; } text.clear(); } Ok(false) } } Self::Regex { regex, multiline, .. } => { let mut text = String::new(); if *multiline { reader.read_to_string(&mut text)?; Ok(regex.is_match(&text)?) } else { let mut bytes_read = 0; while reader.read_line(&mut text)? > 0 { if regex.is_match(&text)? { return Ok(true); } bytes_read += text.len(); if bytes_read >= YIELD_THRESHOLD { bytes_read = 0; smol::future::yield_now().await; } text.clear(); } Ok(false) } } } } /// Returns the replacement text for this `SearchQuery`. pub fn replacement(&self) -> Option<&str> { match self { SearchQuery::Text { replacement, .. } | SearchQuery::Regex { replacement, .. } => { replacement.as_deref() } } } /// Replaces search hits if replacement is set. `text` is assumed to be a string that matches this `SearchQuery` exactly, without any leftovers on either side. pub fn replacement_for<'a>(&self, text: &'a str) -> Option<Cow<'a, str>> { match self { SearchQuery::Text { replacement, .. } => replacement.clone().map(Cow::from), SearchQuery::Regex { regex, replacement, .. } => { if let Some(replacement) = replacement { static TEXT_REPLACEMENT_SPECIAL_CHARACTERS_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\\\\|\\n|\\t").unwrap()); let replacement = TEXT_REPLACEMENT_SPECIAL_CHARACTERS_REGEX.replace_all( replacement, |c: &Captures| match c.get(0).unwrap().as_str() { r"\\" => "\\", r"\n" => "\n", r"\t" => "\t", x => unreachable!("Unexpected escape sequence: {}", x), }, ); Some(regex.replace(text, replacement)) } else { None } } } } pub async fn search( &self, buffer: &BufferSnapshot, subrange: Option<Range<usize>>, ) -> Vec<Range<usize>> { const YIELD_INTERVAL: usize = 20000; if self.as_str().is_empty() { return Default::default(); } let range_offset = subrange.as_ref().map(|r| r.start).unwrap_or(0); let rope = if let Some(range) = subrange { buffer.as_rope().slice(range) } else { buffer.as_rope().clone() }; let mut matches = Vec::new(); match self { Self::Text { search, whole_word, .. } => { for (ix, mat) in search .stream_find_iter(rope.bytes_in_range(0..rope.len())) .enumerate() { if (ix + 1) % YIELD_INTERVAL == 0 { yield_now().await; } let mat = mat.unwrap(); if *whole_word { let classifier = buffer.char_classifier_at(range_offset + mat.start()); let prev_kind = rope .reversed_chars_at(mat.start()) .next() .map(|c| classifier.kind(c)); let start_kind = classifier.kind(rope.chars_at(mat.start()).next().unwrap()); let end_kind = classifier.kind(rope.reversed_chars_at(mat.end()).next().unwrap()); let next_kind = rope.chars_at(mat.end()).next().map(|c| classifier.kind(c)); if (Some(start_kind) == prev_kind && start_kind == CharKind::Word) || (Some(end_kind) == next_kind && end_kind == CharKind::Word) { continue; } } matches.push(mat.start()..mat.end()) } } Self::Regex { regex, multiline, .. } => { if *multiline { let text = rope.to_string(); for (ix, mat) in regex.find_iter(&text).enumerate() { if (ix + 1) % YIELD_INTERVAL == 0 { yield_now().await; } if let Ok(mat) = mat { matches.push(mat.start()..mat.end()); } } } else { let mut line = String::new(); let mut line_offset = 0; for (chunk_ix, chunk) in rope.chunks().chain(["\n"]).enumerate() { if (chunk_ix + 1) % YIELD_INTERVAL == 0 { yield_now().await;Lines 1–500 of 623. Load more
use aho_corasick::{AhoCorasick, AhoCorasickBuilder}; use anyhow::Result; use client::proto; use fancy_regex::{Captures, Regex, RegexBuilder}; use gpui::Entity; use itertools::Itertools as _; use language::{Buffer, BufferSnapshot, CharKind}; use smol::future::yield_now; use std::{ borrow::Cow, io::{BufRead, BufReader, Read}, ops::Range, sync::{Arc, LazyLock}, }; use text::Anchor; use util::{ paths::{PathMatcher, PathStyle}, rel_path::RelPath, }; #[derive(Debug)] pub enum SearchResult { Buffer { buffer: Entity<Buffer>, ranges: Vec<Range<Anchor>>, }, LimitReached, } #[derive(Clone, Copy, PartialEq)] pub enum SearchInputKind { Query, Include, Exclude, } #[derive(Clone, Debug)] pub struct SearchInputs { query: Arc<str>, files_to_include: PathMatcher, files_to_exclude: PathMatcher, match_full_paths: bool, buffers: Option<Vec<Entity<Buffer>>>, } impl SearchInputs { pub fn as_str(&self) -> &str { self.query.as_ref() } pub fn files_to_include(&self) -> &PathMatcher { &self.files_to_include } pub fn files_to_exclude(&self) -> &PathMatcher { &self.files_to_exclude } pub fn buffers(&self) -> &Option<Vec<Entity<Buffer>>> { &self.buffers } } #[derive(Clone, Debug)] pub enum SearchQuery { Text { search: AhoCorasick, replacement: Option<String>, whole_word: bool, case_sensitive: bool, include_ignored: bool, inner: SearchInputs, }, Regex { regex: Regex, replacement: Option<String>, multiline: bool, whole_word: bool, case_sensitive: bool, include_ignored: bool, one_match_per_line: bool, inner: SearchInputs, }, } static WORD_MATCH_TEST: LazyLock<Regex> = LazyLock::new(|| { RegexBuilder::new(r"\B") .build() .expect("Failed to create WORD_MATCH_TEST") }); impl SearchQuery { /// Create a text query /// /// If `match_full_paths` is true, include/exclude patterns will always be matched against fully qualified project paths beginning with a project root. /// If `match_full_paths` is false, patterns will be matched against worktree-relative paths. pub fn text( query: impl ToString, whole_word: bool, case_sensitive: bool, include_ignored: bool, files_to_include: PathMatcher, files_to_exclude: PathMatcher, match_full_paths: bool, buffers: Option<Vec<Entity<Buffer>>>, ) -> Result<Self> { let query = query.to_string(); if !case_sensitive && !query.is_ascii() { // AhoCorasickBuilder doesn't support case-insensitive search with unicode characters // Fallback to regex search as recommended by // https://docs.rs/aho-corasick/1.1/aho_corasick/struct.AhoCorasickBuilder.html#method.ascii_case_insensitive return Self::regex( regex::escape(&query), whole_word, case_sensitive, include_ignored, false, files_to_include, files_to_exclude, false, buffers, ); } let search = AhoCorasickBuilder::new() .ascii_case_insensitive(!case_sensitive) .build([&query])?; let inner = SearchInputs { query: query.into(), files_to_exclude, files_to_include, match_full_paths, buffers, }; Ok(Self::Text { search, replacement: None, whole_word, case_sensitive, include_ignored, inner, }) } /// Create a regex query /// /// If `match_full_paths` is true, include/exclude patterns will be matched against fully qualified project paths /// beginning with a project root name. If false, they will be matched against project-relative paths (which don't start /// with their respective project root). pub fn regex( query: impl ToString, whole_word: bool, mut case_sensitive: bool, include_ignored: bool, one_match_per_line: bool, files_to_include: PathMatcher, files_to_exclude: PathMatcher, match_full_paths: bool, buffers: Option<Vec<Entity<Buffer>>>, ) -> Result<Self> { let mut query = query.to_string(); let initial_query = Arc::from(query.as_str()); if let Some((case_sensitive_from_pattern, new_query)) = Self::case_sensitive_from_pattern(&query) { case_sensitive = case_sensitive_from_pattern; query = new_query } if whole_word { let mut word_query = String::new(); if let Some(first) = query.get(0..1) && WORD_MATCH_TEST.is_match(first).is_ok_and(|x| !x) { word_query.push_str("\\b"); } word_query.push_str(&query); if let Some(last) = query.get(query.len() - 1..) && WORD_MATCH_TEST.is_match(last).is_ok_and(|x| !x) { word_query.push_str("\\b"); } query = word_query } let multiline = query.contains('\n') || query.contains("\\n"); if multiline { query.insert_str(0, "(?m)"); } let regex = RegexBuilder::new(&query) .case_insensitive(!case_sensitive) .build()?; let inner = SearchInputs { query: initial_query, files_to_exclude, files_to_include, match_full_paths, buffers, }; Ok(Self::Regex { regex, replacement: None, multiline, whole_word, case_sensitive, include_ignored, inner, one_match_per_line, }) } /// Extracts case sensitivity settings from pattern items in the provided /// query and returns the same query, with the pattern items removed. /// /// The following pattern modifiers are supported: /// /// - `\c` (case_sensitive: false) /// - `\C` (case_sensitive: true) /// /// If no pattern item were found, `None` will be returned. fn case_sensitive_from_pattern(query: &str) -> Option<(bool, String)> { if !(query.contains("\\c") || query.contains("\\C")) { return None; } let mut was_escaped = false; let mut new_query = String::new(); let mut is_case_sensitive = None; for c in query.chars() { if was_escaped { if c == 'c' { is_case_sensitive = Some(false); } else if c == 'C' { is_case_sensitive = Some(true); } else { new_query.push('\\'); new_query.push(c); } was_escaped = false } else if c == '\\' { was_escaped = true } else { new_query.push(c); } } is_case_sensitive.map(|c| (c, new_query)) } pub fn from_proto(message: proto::SearchQuery, path_style: PathStyle) -> Result<Self> { let files_to_include = if message.files_to_include.is_empty() { message .files_to_include_legacy .split(',') .map(str::trim) .filter(|&glob_str| !glob_str.is_empty()) .map(|s| s.to_string()) .collect() } else { message.files_to_include }; let files_to_exclude = if message.files_to_exclude.is_empty() { message .files_to_exclude_legacy .split(',') .map(str::trim) .filter(|&glob_str| !glob_str.is_empty()) .map(|s| s.to_string()) .collect() } else { message.files_to_exclude }; if message.regex { Self::regex( message.query, message.whole_word, message.case_sensitive, message.include_ignored, false, PathMatcher::new(files_to_include, path_style)?, PathMatcher::new(files_to_exclude, path_style)?, message.match_full_paths, None, // search opened only don't need search remote ) } else { Self::text( message.query, message.whole_word, message.case_sensitive, message.include_ignored, PathMatcher::new(files_to_include, path_style)?, PathMatcher::new(files_to_exclude, path_style)?, message.match_full_paths, None, // search opened only don't need search remote ) } } pub fn with_replacement(mut self, new_replacement: String) -> Self { match self { Self::Text { ref mut replacement, .. } | Self::Regex { ref mut replacement, .. } => { *replacement = Some(new_replacement); self } } } pub fn to_proto(&self) -> proto::SearchQuery { let mut files_to_include = self.files_to_include().sources(); let mut files_to_exclude = self.files_to_exclude().sources(); proto::SearchQuery { query: self.as_str().to_string(), regex: self.is_regex(), whole_word: self.whole_word(), case_sensitive: self.case_sensitive(), include_ignored: self.include_ignored(), files_to_include: files_to_include.clone().map(ToOwned::to_owned).collect(), files_to_exclude: files_to_exclude.clone().map(ToOwned::to_owned).collect(), match_full_paths: self.match_full_paths(), // Populate legacy fields for backwards compatibility files_to_include_legacy: files_to_include.join(","), files_to_exclude_legacy: files_to_exclude.join(","), } } pub(crate) async fn detect( &self, mut reader: BufReader<Box<dyn Read + Send + Sync>>, ) -> Result<bool> { let query_str = self.as_str(); if query_str.is_empty() { return Ok(false); } // Yield from this function every 20KB scanned. const YIELD_THRESHOLD: usize = 20 * 1024; match self { Self::Text { search, .. } => { let mut text = String::new(); if query_str.contains('\n') { reader.read_to_string(&mut text)?; Ok(search.is_match(&text)) } else { let mut bytes_read = 0; while reader.read_line(&mut text)? > 0 { if search.is_match(&text) { return Ok(true); } bytes_read += text.len(); if bytes_read >= YIELD_THRESHOLD { bytes_read = 0; smol::future::yield_now().await; } text.clear(); } Ok(false) } } Self::Regex { regex, multiline, .. } => { let mut text = String::new(); if *multiline { reader.read_to_string(&mut text)?; Ok(regex.is_match(&text)?) } else { let mut bytes_read = 0; while reader.read_line(&mut text)? > 0 { if regex.is_match(&text)? { return Ok(true); } bytes_read += text.len(); if bytes_read >= YIELD_THRESHOLD { bytes_read = 0; smol::future::yield_now().await; } text.clear(); } Ok(false) } } } } /// Returns the replacement text for this `SearchQuery`. pub fn replacement(&self) -> Option<&str> { match self { SearchQuery::Text { replacement, .. } | SearchQuery::Regex { replacement, .. } => { replacement.as_deref() } } } /// Replaces search hits if replacement is set. `text` is assumed to be a string that matches this `SearchQuery` exactly, without any leftovers on either side. pub fn replacement_for<'a>(&self, text: &'a str) -> Option<Cow<'a, str>> { match self { SearchQuery::Text { replacement, .. } => replacement.clone().map(Cow::from), SearchQuery::Regex { regex, replacement, .. } => { if let Some(replacement) = replacement { static TEXT_REPLACEMENT_SPECIAL_CHARACTERS_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\\\\|\\n|\\t").unwrap()); let replacement = TEXT_REPLACEMENT_SPECIAL_CHARACTERS_REGEX.replace_all( replacement, |c: &Captures| match c.get(0).unwrap().as_str() { r"\\" => "\\", r"\n" => "\n", r"\t" => "\t", x => unreachable!("Unexpected escape sequence: {}", x), }, ); Some(regex.replace(text, replacement)) } else { None } } } } pub async fn search( &self, buffer: &BufferSnapshot, subrange: Option<Range<usize>>, ) -> Vec<Range<usize>> { const YIELD_INTERVAL: usize = 20000; if self.as_str().is_empty() { return Default::default(); } let range_offset = subrange.as_ref().map(|r| r.start).unwrap_or(0); let rope = if let Some(range) = subrange { buffer.as_rope().slice(range) } else { buffer.as_rope().clone() }; let mut matches = Vec::new(); match self { Self::Text { search, whole_word, .. } => { for (ix, mat) in search .stream_find_iter(rope.bytes_in_range(0..rope.len())) .enumerate() { if (ix + 1) % YIELD_INTERVAL == 0 { yield_now().await; } let mat = mat.unwrap(); if *whole_word { let classifier = buffer.char_classifier_at(range_offset + mat.start()); let prev_kind = rope .reversed_chars_at(mat.start()) .next() .map(|c| classifier.kind(c)); let start_kind = classifier.kind(rope.chars_at(mat.start()).next().unwrap()); let end_kind = classifier.kind(rope.reversed_chars_at(mat.end()).next().unwrap()); let next_kind = rope.chars_at(mat.end()).next().map(|c| classifier.kind(c)); if (Some(start_kind) == prev_kind && start_kind == CharKind::Word) || (Some(end_kind) == next_kind && end_kind == CharKind::Word) { continue; } } matches.push(mat.start()..mat.end()) } } Self::Regex { regex, multiline, .. } => { if *multiline { let text = rope.to_string(); for (ix, mat) in regex.find_iter(&text).enumerate() { if (ix + 1) % YIELD_INTERVAL == 0 { yield_now().await; } if let Ok(mat) = mat { matches.push(mat.start()..mat.end()); } } } else { let mut line = String::new(); let mut line_offset = 0; for (chunk_ix, chunk) in rope.chunks().chain(["\n"]).enumerate() { if (chunk_ix + 1) % YIELD_INTERVAL == 0 { yield_now().await;