History

updated vector store indexing to only use languages with an embedding.scm treesitter query

KCaverly committed 3 years ago

Commit 36907bb1 parent 0db08765 files · +98 −7Browse files

Unified · Split

5 files
  1. crates/language/src/language.rs +44 −0
  2. crates/vector_store/src/vector_store.rs +16 −6
  3. crates/vector_store/src/vector_store_tests.rs +1 −1
  4. crates/zed/src/languages.rs +1 −0
  5. crates/zed/src/languages/rust/embedding.scm +36 −0
crates/language/src/language.rs+44−0
@@ -350,6 +350,7 @@ pub struct LanguageQueries { · 349 unmodified lines
350350 pub brackets: Option<Cow<'static, str>>,
351351 pub indents: Option<Cow<'static, str>>,
352352 pub outline: Option<Cow<'static, str>>,
353+ pub embedding: Option<Cow<'static, str>>,
353354 pub injections: Option<Cow<'static, str>>,
354355 pub overrides: Option<Cow<'static, str>>,
355356 }
@@ -495,6 +496,7 @@ pub struct Grammar { · 139 unmodified lines
495496 pub(crate) brackets_config: Option<BracketConfig>,
496497 pub(crate) indents_config: Option<IndentConfig>,
497498 pub outline_config: Option<OutlineConfig>,
499+ pub embedding_config: Option<EmbeddingConfig>,
498500 pub(crate) injection_config: Option<InjectionConfig>,
499501 pub(crate) override_config: Option<OverrideConfig>,
500502 pub(crate) highlight_map: Mutex<HighlightMap>,
@@ -516,6 +518,15 @@ pub struct OutlineConfig { · 15 unmodified lines
516518 pub extra_context_capture_ix: Option<u32>,
517519 }
518520
521+#[derive(Debug)]
522+pub struct EmbeddingConfig {
523+ pub query: Query,
524+ pub item_capture_ix: u32,
525+ pub name_capture_ix: u32,
526+ pub context_capture_ix: Option<u32>,
527+ pub extra_context_capture_ix: Option<u32>,
528+}
529+
519530 struct InjectionConfig {
520531 query: Query,
521532 content_capture_ix: u32,
@@ -1145,6 +1156,7 @@ impl Language { · 623 unmodified lines
11451156 highlights_query: None,
11461157 brackets_config: None,
11471158 outline_config: None,
1159+ embedding_config: None,
11481160 indents_config: None,
11491161 injection_config: None,
11501162 override_config: None,
@@ -1181,6 +1193,9 @@ impl Language { · 30 unmodified lines
11811193 if let Some(query) = queries.outline {
11821194 self = self.with_outline_query(query.as_ref())?;
11831195 }
1196+ if let Some(query) = queries.embedding {
1197+ self = self.with_embedding_query(query.as_ref())?;
1198+ }
11841199 if let Some(query) = queries.injections {
11851200 self = self.with_injection_query(query.as_ref())?;
11861201 }
@@ -1189,6 +1204,7 @@ impl Language { · 2 unmodified lines
11891204 }
11901205 Ok(self)
11911206 }
1207+
11921208 pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
11931209 let grammar = self.grammar_mut();
11941210 grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?);
@@ -1223,6 +1239,34 @@ impl Language { · 28 unmodified lines
12231239 Ok(self)
12241240 }
12251241
1242+ pub fn with_embedding_query(mut self, source: &str) -> Result<Self> {
1243+ let grammar = self.grammar_mut();
1244+ let query = Query::new(grammar.ts_language, source)?;
1245+ let mut item_capture_ix = None;
1246+ let mut name_capture_ix = None;
1247+ let mut context_capture_ix = None;
1248+ let mut extra_context_capture_ix = None;
1249+ get_capture_indices(
1250+ &query,
1251+ &mut [
1252+ ("item", &mut item_capture_ix),
1253+ ("name", &mut name_capture_ix),
1254+ ("context", &mut context_capture_ix),
1255+ ("context.extra", &mut extra_context_capture_ix),
1256+ ],
1257+ );
1258+ if let Some((item_capture_ix, name_capture_ix)) = item_capture_ix.zip(name_capture_ix) {
1259+ grammar.embedding_config = Some(EmbeddingConfig {
1260+ query,
1261+ item_capture_ix,
1262+ name_capture_ix,
1263+ context_capture_ix,
1264+ extra_context_capture_ix,
1265+ });
1266+ }
1267+ Ok(self)
1268+ }
1269+
12261270 pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
12271271 let grammar = self.grammar_mut();
12281272 let query = Query::new(grammar.ts_language, source)?;
crates/vector_store/src/vector_store.rs+16−6
@@ -136,8 +136,8 @@ impl VectorStore { · 135 unmodified lines
136136 content: String,
137137 ) -> Result<IndexedFile> {
138138 let grammar = language.grammar().ok_or_else(|| anyhow!("no grammar"))?;
139− let outline_config = grammar
140− .outline_config
139+ let embedding_config = grammar
140+ .embedding_config
141141 .as_ref()
142142 .ok_or_else(|| anyhow!("no outline query"))?;
143143
@@ -148,13 +148,17 @@ impl VectorStore { · 4 unmodified lines
148148
149149 let mut documents = Vec::new();
150150 let mut context_spans = Vec::new();
151− for mat in cursor.matches(&outline_config.query, tree.root_node(), content.as_bytes()) {
151+ for mat in cursor.matches(
152+ &embedding_config.query,
153+ tree.root_node(),
154+ content.as_bytes(),
155+ ) {
152156 let mut item_range = None;
153157 let mut name_range = None;
154158 for capture in mat.captures {
155− if capture.index == outline_config.item_capture_ix {
159+ if capture.index == embedding_config.item_capture_ix {
156160 item_range = Some(capture.node.byte_range());
157− } else if capture.index == outline_config.name_capture_ix {
161+ } else if capture.index == embedding_config.name_capture_ix {
158162 name_range = Some(capture.node.byte_range());
159163 }
160164 }
@@ -266,7 +270,11 @@ impl VectorStore { · 105 unmodified lines
266270 .language_for_file(&absolute_path, None)
267271 .await
268272 {
269− if language.name().as_ref() != "Rust" {
273+ if language
274+ .grammar()
275+ .and_then(|grammar| grammar.embedding_config.as_ref())
276+ .is_none()
277+ {
270278 continue;
271279 }
272280
@@ -359,6 +367,8 @@ impl VectorStore { · 86 unmodified lines
359367 this.worktree_db_ids.extend(worktree_db_ids);
360368 });
361369
370+ log::info!("Semantic Indexing Complete!");
371+
362372 anyhow::Ok(())
363373 })
364374 }
crates/vector_store/src/vector_store_tests.rs+1−1
@@ -46,7 +46,7 @@ async fn test_vector_store(cx: &mut TestAppContext) { · 45 unmodified lines
4646 },
4747 Some(tree_sitter_rust::language()),
4848 )
49− .with_outline_query(
49+ .with_embedding_query(
5050 r#"
5151 (function_item
5252 name: (identifier) @name
crates/zed/src/languages.rs+1−0
@@ -170,6 +170,7 @@ fn load_queries(name: &str) -> LanguageQueries { · 169 unmodified lines
170170 brackets: load_query(name, "/brackets"),
171171 indents: load_query(name, "/indents"),
172172 outline: load_query(name, "/outline"),
173+ embedding: load_query(name, "/embedding"),
173174 injections: load_query(name, "/injections"),
174175 overrides: load_query(name, "/overrides"),
175176 }
crates/zed/src/languages/rust/embedding.scm+36−0
@@ -0,0 +1,36 @@
1+(struct_item
2+ (visibility_modifier)? @context
3+ "struct" @context
4+ name: (_) @name) @item
5+
6+(enum_item
7+ (visibility_modifier)? @context
8+ "enum" @context
9+ name: (_) @name) @item
10+
11+(impl_item
12+ "impl" @context
13+ trait: (_)? @name
14+ "for"? @context
15+ type: (_) @name) @item
16+
17+(trait_item
18+ (visibility_modifier)? @context
19+ "trait" @context
20+ name: (_) @name) @item
21+
22+(function_item
23+ (visibility_modifier)? @context
24+ (function_modifiers)? @context
25+ "fn" @context
26+ name: (_) @name) @item
27+
28+(function_signature_item
29+ (visibility_modifier)? @context
30+ (function_modifiers)? @context
31+ "fn" @context
32+ name: (_) @name) @item
33+
34+(macro_definition
35+ . "macro_rules!" @context
36+ name: (_) @name) @item

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t