History

updated vector store indexing to only use languages with an embedding.scm treesitter query

KCaverly committed 3 years ago

Commit 36907bb1 parent 0db08765 files · +98 −7Browse files

Unified · Split

5 files
  1. crates/language/src/language.rs +44 −0
  2. crates/vector_store/src/vector_store.rs +16 −6
  3. crates/vector_store/src/vector_store_tests.rs +1 −1
  4. crates/zed/src/languages.rs +1 −0
  5. crates/zed/src/languages/rust/embedding.scm +36 −0
crates/language/src/language.rs+44−0
@@ -350,6 +350,7 @@ pub struct LanguageQueries { · 349 unmodified lines
350 pub brackets: Option<Cow<'static, str>>,350 pub brackets: Option<Cow<'static, str>>,
351 pub indents: Option<Cow<'static, str>>,351 pub indents: Option<Cow<'static, str>>,
352 pub outline: Option<Cow<'static, str>>,352 pub outline: Option<Cow<'static, str>>,
353 pub embedding: Option<Cow<'static, str>>,
353 pub injections: Option<Cow<'static, str>>,354 pub injections: Option<Cow<'static, str>>,
354 pub overrides: Option<Cow<'static, str>>,355 pub overrides: Option<Cow<'static, str>>,
355}356}
@@ -495,6 +496,7 @@ pub struct Grammar { · 139 unmodified lines
495 pub(crate) brackets_config: Option<BracketConfig>,496 pub(crate) brackets_config: Option<BracketConfig>,
496 pub(crate) indents_config: Option<IndentConfig>,497 pub(crate) indents_config: Option<IndentConfig>,
497 pub outline_config: Option<OutlineConfig>,498 pub outline_config: Option<OutlineConfig>,
499 pub embedding_config: Option<EmbeddingConfig>,
498 pub(crate) injection_config: Option<InjectionConfig>,500 pub(crate) injection_config: Option<InjectionConfig>,
499 pub(crate) override_config: Option<OverrideConfig>,501 pub(crate) override_config: Option<OverrideConfig>,
500 pub(crate) highlight_map: Mutex<HighlightMap>,502 pub(crate) highlight_map: Mutex<HighlightMap>,
@@ -516,6 +518,15 @@ pub struct OutlineConfig { · 15 unmodified lines
516 pub extra_context_capture_ix: Option<u32>,518 pub extra_context_capture_ix: Option<u32>,
517}519}
518 520
521#[derive(Debug)]
522pub struct EmbeddingConfig {
523 pub query: Query,
524 pub item_capture_ix: u32,
525 pub name_capture_ix: u32,
526 pub context_capture_ix: Option<u32>,
527 pub extra_context_capture_ix: Option<u32>,
528}
529
519struct InjectionConfig {530struct InjectionConfig {
520 query: Query,531 query: Query,
521 content_capture_ix: u32,532 content_capture_ix: u32,
@@ -1145,6 +1156,7 @@ impl Language { · 623 unmodified lines
1145 highlights_query: None,1156 highlights_query: None,
1146 brackets_config: None,1157 brackets_config: None,
1147 outline_config: None,1158 outline_config: None,
1159 embedding_config: None,
1148 indents_config: None,1160 indents_config: None,
1149 injection_config: None,1161 injection_config: None,
1150 override_config: None,1162 override_config: None,
@@ -1181,6 +1193,9 @@ impl Language { · 30 unmodified lines
1181 if let Some(query) = queries.outline {1193 if let Some(query) = queries.outline {
1182 self = self.with_outline_query(query.as_ref())?;1194 self = self.with_outline_query(query.as_ref())?;
1183 }1195 }
1196 if let Some(query) = queries.embedding {
1197 self = self.with_embedding_query(query.as_ref())?;
1198 }
1184 if let Some(query) = queries.injections {1199 if let Some(query) = queries.injections {
1185 self = self.with_injection_query(query.as_ref())?;1200 self = self.with_injection_query(query.as_ref())?;
1186 }1201 }
@@ -1189,6 +1204,7 @@ impl Language { · 2 unmodified lines
1189 }1204 }
1190 Ok(self)1205 Ok(self)
1191 }1206 }
1207
1192 pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {1208 pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
1193 let grammar = self.grammar_mut();1209 let grammar = self.grammar_mut();
1194 grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?);1210 grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?);
@@ -1223,6 +1239,34 @@ impl Language { · 28 unmodified lines
1223 Ok(self)1239 Ok(self)
1224 }1240 }
1225 1241
1242 pub fn with_embedding_query(mut self, source: &str) -> Result<Self> {
1243 let grammar = self.grammar_mut();
1244 let query = Query::new(grammar.ts_language, source)?;
1245 let mut item_capture_ix = None;
1246 let mut name_capture_ix = None;
1247 let mut context_capture_ix = None;
1248 let mut extra_context_capture_ix = None;
1249 get_capture_indices(
1250 &query,
1251 &mut [
1252 ("item", &mut item_capture_ix),
1253 ("name", &mut name_capture_ix),
1254 ("context", &mut context_capture_ix),
1255 ("context.extra", &mut extra_context_capture_ix),
1256 ],
1257 );
1258 if let Some((item_capture_ix, name_capture_ix)) = item_capture_ix.zip(name_capture_ix) {
1259 grammar.embedding_config = Some(EmbeddingConfig {
1260 query,
1261 item_capture_ix,
1262 name_capture_ix,
1263 context_capture_ix,
1264 extra_context_capture_ix,
1265 });
1266 }
1267 Ok(self)
1268 }
1269
1226 pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {1270 pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
1227 let grammar = self.grammar_mut();1271 let grammar = self.grammar_mut();
1228 let query = Query::new(grammar.ts_language, source)?;1272 let query = Query::new(grammar.ts_language, source)?;
crates/vector_store/src/vector_store.rs+16−6
@@ -136,8 +136,8 @@ impl VectorStore { · 135 unmodified lines
136 content: String,136 content: String,
137 ) -> Result<IndexedFile> {137 ) -> Result<IndexedFile> {
138 let grammar = language.grammar().ok_or_else(|| anyhow!("no grammar"))?;138 let grammar = language.grammar().ok_or_else(|| anyhow!("no grammar"))?;
139 let outline_config = grammar139 let embedding_config = grammar
140 .outline_config140 .embedding_config
141 .as_ref()141 .as_ref()
142 .ok_or_else(|| anyhow!("no outline query"))?;142 .ok_or_else(|| anyhow!("no outline query"))?;
143 143
@@ -148,13 +148,17 @@ impl VectorStore { · 4 unmodified lines
148 148
149 let mut documents = Vec::new();149 let mut documents = Vec::new();
150 let mut context_spans = Vec::new();150 let mut context_spans = Vec::new();
151 for mat in cursor.matches(&outline_config.query, tree.root_node(), content.as_bytes()) {151 for mat in cursor.matches(
152 &embedding_config.query,
153 tree.root_node(),
154 content.as_bytes(),
155 ) {
152 let mut item_range = None;156 let mut item_range = None;
153 let mut name_range = None;157 let mut name_range = None;
154 for capture in mat.captures {158 for capture in mat.captures {
155 if capture.index == outline_config.item_capture_ix {159 if capture.index == embedding_config.item_capture_ix {
156 item_range = Some(capture.node.byte_range());160 item_range = Some(capture.node.byte_range());
157 } else if capture.index == outline_config.name_capture_ix {161 } else if capture.index == embedding_config.name_capture_ix {
158 name_range = Some(capture.node.byte_range());162 name_range = Some(capture.node.byte_range());
159 }163 }
160 }164 }
@@ -266,7 +270,11 @@ impl VectorStore { · 105 unmodified lines
266 .language_for_file(&absolute_path, None)270 .language_for_file(&absolute_path, None)
267 .await271 .await
268 {272 {
269 if language.name().as_ref() != "Rust" {273 if language
274 .grammar()
275 .and_then(|grammar| grammar.embedding_config.as_ref())
276 .is_none()
277 {
270 continue;278 continue;
271 }279 }
272 280
@@ -359,6 +367,8 @@ impl VectorStore { · 86 unmodified lines
359 this.worktree_db_ids.extend(worktree_db_ids);367 this.worktree_db_ids.extend(worktree_db_ids);
360 });368 });
361 369
370 log::info!("Semantic Indexing Complete!");
371
362 anyhow::Ok(())372 anyhow::Ok(())
363 })373 })
364 }374 }
crates/vector_store/src/vector_store_tests.rs+1−1
@@ -46,7 +46,7 @@ async fn test_vector_store(cx: &mut TestAppContext) { · 45 unmodified lines
46 },46 },
47 Some(tree_sitter_rust::language()),47 Some(tree_sitter_rust::language()),
48 )48 )
49 .with_outline_query(49 .with_embedding_query(
50 r#"50 r#"
51 (function_item51 (function_item
52 name: (identifier) @name52 name: (identifier) @name
crates/zed/src/languages.rs+1−0
@@ -170,6 +170,7 @@ fn load_queries(name: &str) -> LanguageQueries { · 169 unmodified lines
170 brackets: load_query(name, "/brackets"),170 brackets: load_query(name, "/brackets"),
171 indents: load_query(name, "/indents"),171 indents: load_query(name, "/indents"),
172 outline: load_query(name, "/outline"),172 outline: load_query(name, "/outline"),
173 embedding: load_query(name, "/embedding"),
173 injections: load_query(name, "/injections"),174 injections: load_query(name, "/injections"),
174 overrides: load_query(name, "/overrides"),175 overrides: load_query(name, "/overrides"),
175 }176 }
crates/zed/src/languages/rust/embedding.scm+36−0
@@ -0,0 +1,36 @@
1(struct_item
2 (visibility_modifier)? @context
3 "struct" @context
4 name: (_) @name) @item
5
6(enum_item
7 (visibility_modifier)? @context
8 "enum" @context
9 name: (_) @name) @item
10
11(impl_item
12 "impl" @context
13 trait: (_)? @name
14 "for"? @context
15 type: (_) @name) @item
16
17(trait_item
18 (visibility_modifier)? @context
19 "trait" @context
20 name: (_) @name) @item
21
22(function_item
23 (visibility_modifier)? @context
24 (function_modifiers)? @context
25 "fn" @context
26 name: (_) @name) @item
27
28(function_signature_item
29 (visibility_modifier)? @context
30 (function_modifiers)? @context
31 "fn" @context
32 name: (_) @name) @item
33
34(macro_definition
35 . "macro_rules!" @context
36 name: (_) @name) @item

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t