More fixes to the semantic index's chunking (#11376)

This fixes a tricky intermittent issue I was seeing, where failed to
chunk certain files correctly because of the way we reuse Tree-sitter
`Parser` instances across parses.

I've also accounted for leading comments in chunk boundaries, so that
items are grouped with their leading comments whenever possible when
chunking.

Finally, we've changed the `debug project index` action so that it opens
a simple debug view in a pane, instead of printing paths to the console.
This lets you click into a path and see how it was chunked.

Release Notes:

- N/A

---------

Co-authored-by: Marshall <marshall@zed.dev>

Cargo.lock

@@ -8713,9 +8713,12 @@ dependencies = [
  "sha2 0.10.7",
  "smol",
  "tempfile",
+ "theme",
  "tree-sitter",
+ "ui",
  "unindent",
  "util",
+ "workspace",
  "worktree",
 ]
 

crates/assistant2/src/assistant2.rs

@@ -21,7 +21,7 @@ use gpui::{
 use language::{language_settings::SoftWrap, LanguageRegistry};
 use open_ai::{FunctionContent, ToolCall, ToolCallContent};
 use rich_text::RichText;
-use semantic_index::{CloudEmbeddingProvider, ProjectIndex, SemanticIndex};
+use semantic_index::{CloudEmbeddingProvider, ProjectIndex, ProjectIndexDebugView, SemanticIndex};
 use serde::Deserialize;
 use settings::Settings;
 use std::sync::Arc;
@@ -83,6 +83,14 @@ pub fn init(client: Arc<Client>, cx: &mut AppContext) {
             workspace.register_action(|workspace, _: &ToggleFocus, cx| {
                 workspace.toggle_panel_focus::<AssistantPanel>(cx);
             });
+            workspace.register_action(|workspace, _: &DebugProjectIndex, cx| {
+                if let Some(panel) = workspace.panel::<AssistantPanel>(cx) {
+                    if let Some(index) = panel.read(cx).chat.read(cx).project_index.clone() {
+                        let view = cx.new_view(|cx| ProjectIndexDebugView::new(index, cx));
+                        workspace.add_item_to_center(Box::new(view), cx);
+                    }
+                }
+            });
         },
     )
     .detach();
@@ -107,8 +115,6 @@ impl AssistantPanel {
                 (workspace.app_state().clone(), workspace.project().clone())
             })?;
 
-            let user_store = app_state.user_store.clone();
-
             cx.new_view(|cx| {
                 let project_index = cx.update_global(|semantic_index: &mut SemanticIndex, cx| {
                     semantic_index.project_index(project.clone(), cx)
@@ -117,7 +123,7 @@ impl AssistantPanel {
                 let mut tool_registry = ToolRegistry::new();
                 tool_registry
                     .register(
-                        ProjectIndexTool::new(project_index.clone(), app_state.fs.clone()),
+                        ProjectIndexTool::new(project_index.clone(), project.read(cx).fs().clone()),
                         cx,
                     )
                     .context("failed to register ProjectIndexTool")
@@ -135,9 +141,9 @@ impl AssistantPanel {
 
                 Self::new(
                     app_state.languages.clone(),
-                    Arc::new(attachment_store),
                     Arc::new(tool_registry),
-                    user_store,
+                    Arc::new(attachment_store),
+                    app_state.user_store.clone(),
                     Some(project_index),
                     workspace,
                     cx,
@@ -148,8 +154,8 @@ impl AssistantPanel {
 
     pub fn new(
         language_registry: Arc<LanguageRegistry>,
-        attachment_store: Arc<UserAttachmentStore>,
         tool_registry: Arc<ToolRegistry>,
+        attachment_store: Arc<UserAttachmentStore>,
         user_store: Model<UserStore>,
         project_index: Option<Model<ProjectIndex>>,
         workspace: WeakView<Workspace>,
@@ -157,9 +163,9 @@ impl AssistantPanel {
     ) -> Self {
         let chat = cx.new_view(|cx| {
             AssistantChat::new(
-                language_registry.clone(),
+                language_registry,
-                attachment_store.clone(),
                 tool_registry.clone(),
+                attachment_store,
                 user_store,
                 project_index,
                 workspace,
@@ -257,8 +263,8 @@ struct EditingMessage {
 impl AssistantChat {
     fn new(
         language_registry: Arc<LanguageRegistry>,
-        attachment_store: Arc<UserAttachmentStore>,
         tool_registry: Arc<ToolRegistry>,
+        attachment_store: Arc<UserAttachmentStore>,
         user_store: Model<UserStore>,
         project_index: Option<Model<ProjectIndex>>,
         workspace: WeakView<Workspace>,
@@ -429,14 +435,6 @@ impl AssistantChat {
         }));
     }
 
-    fn debug_project_index(&mut self, _: &DebugProjectIndex, cx: &mut ViewContext<Self>) {
-        if let Some(index) = &self.project_index {
-            index.update(cx, |project_index, cx| {
-                project_index.debug(cx).detach_and_log_err(cx)
-            });
-        }
-    }
-
     async fn request_completion(
         this: WeakView<Self>,
         mode: SubmitMode,
@@ -846,7 +844,6 @@ impl Render for AssistantChat {
             .key_context("AssistantChat")
             .on_action(cx.listener(Self::submit))
             .on_action(cx.listener(Self::cancel))
-            .on_action(cx.listener(Self::debug_project_index))
             .text_color(Color::Default.color(cx))
             .child(list(self.list_state.clone()).flex_1())
             .child(Composer::new(

crates/editor/src/editor.rs

@@ -2768,7 +2768,7 @@ impl Editor {
                         indent.len = cmp::min(indent.len, start_point.column);
                         let start = selection.start;
                         let end = selection.end;
-                        let is_cursor = start == end;
+                        let selection_is_empty = start == end;
                         let language_scope = buffer.language_scope_at(start);
                         let (comment_delimiter, insert_extra_newline) = if let Some(language) =
                             &language_scope
@@ -2802,13 +2802,18 @@ impl Editor {
                                             pair_start,
                                         )
                                 });
+
                             // Comment extension on newline is allowed only for cursor selections
-                            let comment_delimiter = language.line_comment_prefixes().filter(|_| {
+                            let comment_delimiter = maybe!({
-                                let is_comment_extension_enabled =
+                                if !selection_is_empty {
-                                    multi_buffer.settings_at(0, cx).extend_comment_on_newline;
+                                    return None;
-                                is_cursor && is_comment_extension_enabled
+                                }
-                            });
+
-                            let get_comment_delimiter = |delimiters: &[Arc<str>]| {
+                                if !multi_buffer.settings_at(0, cx).extend_comment_on_newline {
+                                    return None;
+                                }
+
+                                let delimiters = language.line_comment_prefixes();
                                 let max_len_of_delimiter =
                                     delimiters.iter().map(|delimiter| delimiter.len()).max()?;
                                 let (snapshot, range) =
@@ -2837,12 +2842,7 @@ impl Editor {
                                 } else {
                                     None
                                 }
-                            };
+                            });
-                            let comment_delimiter = if let Some(delimiters) = comment_delimiter {
-                                get_comment_delimiter(delimiters)
-                            } else {
-                                None
-                            };
                             (comment_delimiter, insert_extra_newline)
                         } else {
                             (None, false)
@@ -7181,10 +7181,8 @@ impl Editor {
                 }
 
                 // If the language has line comments, toggle those.
-                if let Some(full_comment_prefixes) = language
+                let full_comment_prefixes = language.line_comment_prefixes();
-                    .line_comment_prefixes()
+                if !full_comment_prefixes.is_empty() {
-                    .filter(|prefixes| !prefixes.is_empty())
-                {
                     let first_prefix = full_comment_prefixes
                         .first()
                         .expect("prefixes is non-empty");

crates/language/src/buffer_tests.rs

@@ -1818,7 +1818,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
         let snapshot = buffer.snapshot();
 
         let config = snapshot.language_scope_at(0).unwrap();
-        assert_eq!(config.line_comment_prefixes().unwrap(), &[Arc::from("// ")]);
+        assert_eq!(config.line_comment_prefixes(), &[Arc::from("// ")]);
         // Both bracket pairs are enabled
         assert_eq!(
             config.brackets().map(|e| e.1).collect::<Vec<_>>(),
@@ -1828,10 +1828,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
         let string_config = snapshot
             .language_scope_at(text.find("b\"").unwrap())
             .unwrap();
-        assert_eq!(
+        assert_eq!(string_config.line_comment_prefixes(), &[Arc::from("// ")]);
-            string_config.line_comment_prefixes().unwrap(),
-            &[Arc::from("// ")]
-        );
         // Second bracket pair is disabled
         assert_eq!(
             string_config.brackets().map(|e| e.1).collect::<Vec<_>>(),
@@ -1842,7 +1839,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
         let element_config = snapshot
             .language_scope_at(text.find("<F>").unwrap())
             .unwrap();
-        assert_eq!(element_config.line_comment_prefixes(), None);
+        assert_eq!(element_config.line_comment_prefixes(), &[]);
         assert_eq!(
             element_config.block_comment_delimiters(),
             Some((&"{/*".into(), &"*/}".into()))
@@ -1856,10 +1853,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
         let tag_config = snapshot
             .language_scope_at(text.find(" d=").unwrap() + 1)
             .unwrap();
-        assert_eq!(
+        assert_eq!(tag_config.line_comment_prefixes(), &[Arc::from("// ")]);
-            tag_config.line_comment_prefixes().unwrap(),
-            &[Arc::from("// ")]
-        );
         assert_eq!(
             tag_config.brackets().map(|e| e.1).collect::<Vec<_>>(),
             &[true, true]
@@ -1870,9 +1864,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
             .language_scope_at(text.find('{').unwrap() + 1)
             .unwrap();
         assert_eq!(
-            expression_in_element_config
+            expression_in_element_config.line_comment_prefixes(),
-                .line_comment_prefixes()
-                .unwrap(),
             &[Arc::from("// ")]
         );
         assert_eq!(
@@ -1988,17 +1980,14 @@ fn test_language_scope_at_with_combined_injections(cx: &mut AppContext) {
 
         let snapshot = buffer.snapshot();
         let html_config = snapshot.language_scope_at(Point::new(2, 4)).unwrap();
-        assert_eq!(html_config.line_comment_prefixes(), Some(&vec![]));
+        assert_eq!(html_config.line_comment_prefixes(), &[]);
         assert_eq!(
             html_config.block_comment_delimiters(),
             Some((&"<!--".into(), &"-->".into()))
         );
 
         let ruby_config = snapshot.language_scope_at(Point::new(3, 12)).unwrap();
-        assert_eq!(
+        assert_eq!(ruby_config.line_comment_prefixes(), &[Arc::from("# ")]);
-            ruby_config.line_comment_prefixes().unwrap(),
-            &[Arc::from("# ")]
-        );
         assert_eq!(ruby_config.block_comment_delimiters(), None);
 
         buffer

crates/language/src/language.rs

@@ -20,6 +20,7 @@ mod task_context;
 mod buffer_tests;
 pub mod markdown;
 
+use crate::language_settings::SoftWrap;
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use collections::{HashMap, HashSet};
@@ -41,12 +42,11 @@ use smol::future::FutureExt as _;
 use std::num::NonZeroU32;
 use std::{
     any::Any,
-    cell::RefCell,
     ffi::OsStr,
     fmt::Debug,
     hash::Hash,
     mem,
-    ops::Range,
+    ops::{DerefMut, Range},
     path::{Path, PathBuf},
     pin::Pin,
     str,
@@ -74,8 +74,6 @@ pub use syntax_map::{OwnedSyntaxLayer, SyntaxLayer};
 pub use text::LineEnding;
 pub use tree_sitter::{Node, Parser, Tree, TreeCursor};
 
-use crate::language_settings::SoftWrap;
-
 /// Initializes the `language` crate.
 ///
 /// This should be called before making use of items from the create.
@@ -83,29 +81,30 @@ pub fn init(cx: &mut AppContext) {
     language_settings::init(cx);
 }
 
-thread_local! {
+static QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Mutex::new(vec![]);
-    static PARSER: RefCell<Parser> = {
+static PARSERS: Mutex<Vec<Parser>> = Mutex::new(vec![]);
-        let mut parser = Parser::new();
-        parser.set_wasm_store(WasmStore::new(WASM_ENGINE.clone()).unwrap()).unwrap();
-        RefCell::new(parser)
-    };
-}
 
 pub fn with_parser<F, R>(func: F) -> R
 where
     F: FnOnce(&mut Parser) -> R,
 {
-    PARSER.with(|parser| {
+    let mut parser = PARSERS.lock().pop().unwrap_or_else(|| {
-        let mut parser = parser.borrow_mut();
+        let mut parser = Parser::new();
-        func(&mut parser)
+        parser
-    })
+            .set_wasm_store(WasmStore::new(WASM_ENGINE.clone()).unwrap())
+            .unwrap();
+        parser
+    });
+    parser.set_included_ranges(&[]).unwrap();
+    let result = func(&mut parser);
+    PARSERS.lock().push(parser);
+    result
 }
 
 pub fn with_query_cursor<F, R>(func: F) -> R
 where
     F: FnOnce(&mut QueryCursor) -> R,
 {
-    use std::ops::DerefMut;
     let mut cursor = QueryCursorHandle::new();
     func(cursor.deref_mut())
 }
@@ -1340,11 +1339,12 @@ impl LanguageScope {
 
     /// Returns line prefix that is inserted in e.g. line continuations or
     /// in `toggle comments` action.
-    pub fn line_comment_prefixes(&self) -> Option<&Vec<Arc<str>>> {
+    pub fn line_comment_prefixes(&self) -> &[Arc<str>] {
         Override::as_option(
             self.config_override().map(|o| &o.line_comments),
             Some(&self.language.config.line_comments),
         )
+        .map_or(&[] as &[_], |e| e.as_slice())
     }
 
     pub fn block_comment_delimiters(&self) -> Option<(&Arc<str>, &Arc<str>)> {
@@ -1445,8 +1445,7 @@ impl Grammar {
     }
 
     fn parse_text(&self, text: &Rope, old_tree: Option<Tree>) -> Tree {
-        PARSER.with(|parser| {
+        with_parser(|parser| {
-            let mut parser = parser.borrow_mut();
             parser
                 .set_language(&self.ts_language)
                 .expect("incompatible grammar");

crates/language/src/language_registry.rs

@@ -3,8 +3,8 @@ use crate::{
         all_language_settings, AllLanguageSettingsContent, LanguageSettingsContent,
     },
     task_context::ContextProvider,
-    CachedLspAdapter, File, Language, LanguageConfig, LanguageId, LanguageMatcher,
+    with_parser, CachedLspAdapter, File, Language, LanguageConfig, LanguageId, LanguageMatcher,
-    LanguageServerName, LspAdapter, LspAdapterDelegate, PARSER, PLAIN_TEXT,
+    LanguageServerName, LspAdapter, LspAdapterDelegate, PLAIN_TEXT,
 };
 use anyhow::{anyhow, Context as _, Result};
 use collections::{hash_map, HashMap};
@@ -668,8 +668,7 @@ impl LanguageRegistry {
                                     .file_stem()
                                     .and_then(OsStr::to_str)
                                     .ok_or_else(|| anyhow!("invalid grammar filename"))?;
-                                anyhow::Ok(PARSER.with(|parser| {
+                                anyhow::Ok(with_parser(|parser| {
-                                    let mut parser = parser.borrow_mut();
                                     let mut store = parser.take_wasm_store().unwrap();
                                     let grammar = store.load_language(&grammar_name, &wasm_bytes);
                                     parser.set_wasm_store(store).unwrap();

crates/language/src/syntax_map.rs

@@ -1,10 +1,11 @@
 #[cfg(test)]
 mod syntax_map_tests;
 
-use crate::{Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry};
+use crate::{
+    with_parser, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS,
+};
 use collections::HashMap;
 use futures::FutureExt;
-use parking_lot::Mutex;
 use std::{
     borrow::Cow,
     cmp::{self, Ordering, Reverse},
@@ -17,10 +18,6 @@ use sum_tree::{Bias, SeekTarget, SumTree};
 use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
 use tree_sitter::{Node, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree};
 
-use super::PARSER;
-
-static QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Mutex::new(vec![]);
-
 #[derive(Default)]
 pub struct SyntaxMap {
     snapshot: SyntaxSnapshot,
@@ -1177,8 +1174,7 @@ fn parse_text(
     ranges: Vec<tree_sitter::Range>,
     old_tree: Option<Tree>,
 ) -> anyhow::Result<Tree> {
-    PARSER.with(|parser| {
+    with_parser(|parser| {
-        let mut parser = parser.borrow_mut();
         let mut chunks = text.chunks_in_range(start_byte..text.len());
         parser.set_included_ranges(&ranges)?;
         parser.set_language(&grammar.ts_language)?;

crates/semantic_index/Cargo.toml

@@ -37,9 +37,12 @@ serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 smol.workspace = true
+theme.workspace = true
 tree-sitter.workspace = true
+ui. workspace = true
 util. workspace = true
 unindent.workspace = true
+workspace.workspace = true
 worktree.workspace = true
 
 [dev-dependencies]
@@ -54,3 +57,4 @@ project = { workspace = true, features = ["test-support"] }
 tempfile.workspace = true
 util = { workspace = true, features = ["test-support"] }
 worktree = { workspace = true, features = ["test-support"] }
+workspace = { workspace = true, features = ["test-support"] }

crates/semantic_index/src/chunking.rs

@@ -1,9 +1,10 @@
-use language::{with_parser, with_query_cursor, Grammar};
+use language::{with_parser, with_query_cursor, Language};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::{
     cmp::{self, Reverse},
     ops::Range,
+    path::Path,
     sync::Arc,
 };
 use tree_sitter::QueryCapture;
@@ -26,52 +27,95 @@ pub struct Chunk {
     pub digest: [u8; 32],
 }
 
-pub fn chunk_text(text: &str, grammar: Option<&Arc<Grammar>>) -> Vec<Chunk> {
+pub fn chunk_text(text: &str, language: Option<&Arc<Language>>, path: &Path) -> Vec<Chunk> {
-    chunk_text_with_size_range(text, grammar, CHUNK_SIZE_RANGE)
+    chunk_text_with_size_range(text, language, path, CHUNK_SIZE_RANGE)
 }
 
 fn chunk_text_with_size_range(
     text: &str,
-    grammar: Option<&Arc<Grammar>>,
+    language: Option<&Arc<Language>>,
+    path: &Path,
     size_config: ChunkSizeRange,
 ) -> Vec<Chunk> {
-    let mut syntactic_ranges = Vec::new();
+    let ranges = syntactic_ranges(text, language, path).unwrap_or_default();
+    chunk_text_with_syntactic_ranges(text, &ranges, size_config)
+}
 
-    if let Some(grammar) = grammar {
+fn syntactic_ranges(
-        if let Some(outline) = grammar.outline_config.as_ref() {
+    text: &str,
-            let tree = with_parser(|parser| {
+    language: Option<&Arc<Language>>,
-                parser.set_language(&grammar.ts_language).log_err()?;
+    path: &Path,
-                parser.parse(&text, None)
+) -> Option<Vec<Range<usize>>> {
-            });
+    let language = language?;
+    let grammar = language.grammar()?;
+    let outline = grammar.outline_config.as_ref()?;
+    let tree = with_parser(|parser| {
+        parser.set_language(&grammar.ts_language).log_err()?;
+        parser.parse(&text, None)
+    });
 
-            if let Some(tree) = tree {
+    let Some(tree) = tree else {
-                with_query_cursor(|cursor| {
+        log::error!("failed to parse file {path:?} for chunking");
-                    // Retrieve a list of ranges of outline items (types, functions, etc) in the document.
+        return None;
-                    // Omit single-line outline items (e.g. struct fields, constant declarations), because
+    };
-                    // we'll already be attempting to split on lines.
+
-                    syntactic_ranges = cursor
+    struct RowInfo {
-                        .matches(&outline.query, tree.root_node(), text.as_bytes())
+        offset: usize,
-                        .filter_map(|mat| {
+        is_comment: bool,
-                            mat.captures
-                                .iter()
-                                .find_map(|QueryCapture { node, index }| {
-                                    if *index == outline.item_capture_ix {
-                                        if node.end_position().row > node.start_position().row {
-                                            return Some(node.byte_range());
-                                        }
-                                    }
-                                    None
-                                })
-                        })
-                        .collect::<Vec<_>>();
-                    syntactic_ranges
-                        .sort_unstable_by_key(|range| (range.start, Reverse(range.end)));
-                });
-            }
-        }
     }
 
-    chunk_text_with_syntactic_ranges(text, &syntactic_ranges, size_config)
+    let scope = language.default_scope();
+    let line_comment_prefixes = scope.line_comment_prefixes();
+    let row_infos = text
+        .split('\n')
+        .map({
+            let mut offset = 0;
+            move |line| {
+                let line = line.trim_start();
+                let is_comment = line_comment_prefixes
+                    .iter()
+                    .any(|prefix| line.starts_with(prefix.as_ref()));
+                let result = RowInfo { offset, is_comment };
+                offset += line.len() + 1;
+                result
+            }
+        })
+        .collect::<Vec<_>>();
+
+    // Retrieve a list of ranges of outline items (types, functions, etc) in the document.
+    // Omit single-line outline items (e.g. struct fields, constant declarations), because
+    // we'll already be attempting to split on lines.
+    let mut ranges = with_query_cursor(|cursor| {
+        cursor
+            .matches(&outline.query, tree.root_node(), text.as_bytes())
+            .filter_map(|mat| {
+                mat.captures
+                    .iter()
+                    .find_map(|QueryCapture { node, index }| {
+                        if *index == outline.item_capture_ix {
+                            let mut start_offset = node.start_byte();
+                            let mut start_row = node.start_position().row;
+                            let end_offset = node.end_byte();
+                            let end_row = node.end_position().row;
+
+                            // Expand the range to include any preceding comments.
+                            while start_row > 0 && row_infos[start_row - 1].is_comment {
+                                start_offset = row_infos[start_row - 1].offset;
+                                start_row -= 1;
+                            }
+
+                            if end_row > start_row {
+                                return Some(start_offset..end_offset);
+                            }
+                        }
+                        None
+                    })
+            })
+            .collect::<Vec<_>>()
+    });
+
+    ranges.sort_unstable_by_key(|range| (range.start, Reverse(range.end)));
+    Some(ranges)
 }
 
 fn chunk_text_with_syntactic_ranges(
@@ -148,7 +192,7 @@ fn chunk_text_with_syntactic_ranges(
     if !range.is_empty() {
         chunks.push(Chunk {
             range: range.clone(),
-            digest: Sha256::digest(&text[range.clone()]).into(),
+            digest: Sha256::digest(&text[range]).into(),
         });
     }
 
@@ -177,6 +221,8 @@ mod tests {
                     Self { first_name, last_name, age }
                 }
 
+                /// Returns the first name
+                /// something something something
                 fn first_name(&self) -> &str {
                     &self.first_name
                 }
@@ -185,8 +231,8 @@ mod tests {
                     &self.last_name
                 }
 
-                fn age(&self) -> usize {
+                fn age(&self) -> u32 {
-                    self.ages
+                    self.age
                 }
             }
         "
@@ -194,7 +240,8 @@ mod tests {
 
         let chunks = chunk_text_with_size_range(
             &text,
-            language.grammar(),
+            Some(&language),
+            Path::new("lib.rs"),
             ChunkSizeRange {
                 min: text.find('}').unwrap(),
                 max: text.find("Self {").unwrap(),
@@ -209,8 +256,8 @@ mod tests {
             &[
                 "struct Person {", // ...
                 "impl Person {",
-                "    fn first_name",
+                "    /// Returns the first name",
-                "    fn age",
+                "    fn last_name",
             ],
         );
 
@@ -227,7 +274,8 @@ mod tests {
 
         let chunks = chunk_text_with_size_range(
             &text,
-            language.grammar(),
+            Some(&language),
+            Path::new("lib.rs"),
             ChunkSizeRange {
                 min: text.find('{').unwrap(),
                 max: text.find('V').unwrap(),
@@ -263,7 +311,8 @@ mod tests {
 
         let chunks = chunk_text_with_size_range(
             &text,
-            language.grammar(),
+            Some(&language),
+            Path::new("lib.rs"),
             ChunkSizeRange { min: 32, max: 64 },
         );
 
@@ -331,33 +380,35 @@ mod tests {
     #[test]
     fn test_chunk_text() {
         let text = "a\n".repeat(1000);
-        let chunks = chunk_text(&text, None);
+        let chunks = chunk_text(&text, None, Path::new("lib.rs"));
         assert_eq!(
             chunks.len(),
             ((2000_f64) / (CHUNK_SIZE_RANGE.max as f64)).ceil() as usize
         );
     }
 
-    fn rust_language() -> Language {
+    fn rust_language() -> Arc<Language> {
-        Language::new(
+        Arc::new(
-            LanguageConfig {
+            Language::new(
-                name: "Rust".into(),
+                LanguageConfig {
-                matcher: LanguageMatcher {
+                    name: "Rust".into(),
-                    path_suffixes: vec!["rs".to_string()],
+                    matcher: LanguageMatcher {
+                        path_suffixes: vec!["rs".to_string()],
+                        ..Default::default()
+                    },
                     ..Default::default()
                 },
-                ..Default::default()
+                Some(tree_sitter_rust::language()),
-            },
+            )
-            Some(tree_sitter_rust::language()),
+            .with_outline_query(
-        )
+                "
-        .with_outline_query(
-            "
             (function_item name: (_) @name) @item
             (impl_item type: (_) @name) @item
             (struct_item name: (_) @name) @item
             (field_declaration name: (_) @name) @item
         ",
+            )
+            .unwrap(),
         )
-        .unwrap()
     }
 }

crates/semantic_index/src/project_index_debug_view.rs

@@ -0,0 +1,300 @@
+use crate::ProjectIndex;
+use gpui::{
+    canvas, div, list, uniform_list, AnyElement, AppContext, CursorStyle, EventEmitter,
+    FocusHandle, FocusableView, IntoElement, ListOffset, ListState, Model, MouseMoveEvent, Render,
+    UniformListScrollHandle, View,
+};
+use project::WorktreeId;
+use settings::Settings;
+use std::{path::Path, sync::Arc};
+use theme::ThemeSettings;
+use ui::prelude::*;
+use workspace::item::{Item, TabContentParams};
+
+pub struct ProjectIndexDebugView {
+    index: Model<ProjectIndex>,
+    rows: Vec<Row>,
+    selected_path: Option<PathState>,
+    hovered_row_ix: Option<usize>,
+    focus_handle: FocusHandle,
+    list_scroll_handle: UniformListScrollHandle,
+    _subscription: gpui::Subscription,
+}
+
+struct PathState {
+    path: Arc<Path>,
+    chunks: Vec<SharedString>,
+    list_state: ListState,
+}
+
+enum Row {
+    Worktree(Arc<Path>),
+    Entry(WorktreeId, Arc<Path>),
+}
+
+impl ProjectIndexDebugView {
+    pub fn new(index: Model<ProjectIndex>, cx: &mut ViewContext<Self>) -> Self {
+        let mut this = Self {
+            rows: Vec::new(),
+            list_scroll_handle: UniformListScrollHandle::new(),
+            selected_path: None,
+            hovered_row_ix: None,
+            focus_handle: cx.focus_handle(),
+            _subscription: cx.subscribe(&index, |this, _, _, cx| this.update_rows(cx)),
+            index,
+        };
+        this.update_rows(cx);
+        this
+    }
+
+    fn update_rows(&mut self, cx: &mut ViewContext<Self>) {
+        let worktree_indices = self.index.read(cx).worktree_indices(cx);
+        cx.spawn(|this, mut cx| async move {
+            let mut rows = Vec::new();
+
+            for index in worktree_indices {
+                let (root_path, worktree_id, worktree_paths) =
+                    index.read_with(&cx, |index, cx| {
+                        let worktree = index.worktree.read(cx);
+                        (worktree.abs_path(), worktree.id(), index.paths(cx))
+                    })?;
+                rows.push(Row::Worktree(root_path));
+                rows.extend(
+                    worktree_paths
+                        .await?
+                        .into_iter()
+                        .map(|path| Row::Entry(worktree_id, path)),
+                );
+            }
+
+            this.update(&mut cx, |this, cx| {
+                this.rows = rows;
+                cx.notify();
+            })
+        })
+        .detach();
+    }
+
+    fn handle_path_click(
+        &mut self,
+        worktree_id: WorktreeId,
+        file_path: Arc<Path>,
+        cx: &mut ViewContext<Self>,
+    ) -> Option<()> {
+        let project_index = self.index.read(cx);
+        let fs = project_index.fs.clone();
+        let worktree_index = project_index.worktree_index(worktree_id, cx)?.read(cx);
+        let root_path = worktree_index.worktree.read(cx).abs_path();
+        let chunks = worktree_index.chunks_for_path(file_path.clone(), cx);
+
+        cx.spawn(|this, mut cx| async move {
+            let chunks = chunks.await?;
+            let content = fs.load(&root_path.join(&file_path)).await?;
+            let chunks = chunks
+                .into_iter()
+                .map(|chunk| {
+                    let mut start = chunk.chunk.range.start.min(content.len());
+                    let mut end = chunk.chunk.range.end.min(content.len());
+                    while !content.is_char_boundary(start) {
+                        start += 1;
+                    }
+                    while !content.is_char_boundary(end) {
+                        end -= 1;
+                    }
+                    content[start..end].to_string().into()
+                })
+                .collect::<Vec<_>>();
+
+            this.update(&mut cx, |this, cx| {
+                let view = cx.view().downgrade();
+                this.selected_path = Some(PathState {
+                    path: file_path,
+                    list_state: ListState::new(
+                        chunks.len(),
+                        gpui::ListAlignment::Top,
+                        px(100.),
+                        move |ix, cx| {
+                            if let Some(view) = view.upgrade() {
+                                view.update(cx, |view, cx| view.render_chunk(ix, cx))
+                            } else {
+                                div().into_any()
+                            }
+                        },
+                    ),
+                    chunks,
+                });
+                cx.notify();
+            })
+        })
+        .detach();
+        None
+    }
+
+    fn render_chunk(&mut self, ix: usize, cx: &mut ViewContext<Self>) -> AnyElement {
+        let buffer_font = ThemeSettings::get_global(cx).buffer_font.family.clone();
+        let Some(state) = &self.selected_path else {
+            return div().into_any();
+        };
+
+        let colors = cx.theme().colors();
+        let chunk = &state.chunks[ix];
+
+        div()
+            .text_ui(cx)
+            .w_full()
+            .font_family(buffer_font)
+            .child(
+                h_flex()
+                    .justify_between()
+                    .child(format!(
+                        "chunk {} of {}. length: {}",
+                        ix + 1,
+                        state.chunks.len(),
+                        chunk.len(),
+                    ))
+                    .child(
+                        h_flex()
+                            .child(
+                                Button::new(("prev", ix), "prev")
+                                    .disabled(ix == 0)
+                                    .on_click(cx.listener(move |this, _, _| {
+                                        this.scroll_to_chunk(ix.saturating_sub(1))
+                                    })),
+                            )
+                            .child(
+                                Button::new(("next", ix), "next")
+                                    .disabled(ix + 1 == state.chunks.len())
+                                    .on_click(
+                                        cx.listener(move |this, _, _| this.scroll_to_chunk(ix + 1)),
+                                    ),
+                            ),
+                    ),
+            )
+            .child(
+                div()
+                    .bg(colors.editor_background)
+                    .text_xs()
+                    .child(chunk.clone()),
+            )
+            .into_any_element()
+    }
+
+    fn scroll_to_chunk(&mut self, ix: usize) {
+        if let Some(state) = self.selected_path.as_mut() {
+            state.list_state.scroll_to(ListOffset {
+                item_ix: ix,
+                offset_in_item: px(0.),
+            })
+        }
+    }
+}
+
+impl Render for ProjectIndexDebugView {
+    fn render(&mut self, cx: &mut gpui::ViewContext<'_, Self>) -> impl IntoElement {
+        if let Some(selected_path) = self.selected_path.as_ref() {
+            v_flex()
+                .child(
+                    div()
+                        .id("selected-path-name")
+                        .child(
+                            h_flex()
+                                .justify_between()
+                                .child(selected_path.path.to_string_lossy().to_string())
+                                .child("x"),
+                        )
+                        .border_b_1()
+                        .border_color(cx.theme().colors().border)
+                        .cursor(CursorStyle::PointingHand)
+                        .on_click(cx.listener(|this, _, cx| {
+                            this.selected_path.take();
+                            cx.notify();
+                        })),
+                )
+                .child(list(selected_path.list_state.clone()).size_full())
+                .size_full()
+                .into_any_element()
+        } else {
+            let mut list = uniform_list(
+                cx.view().clone(),
+                "ProjectIndexDebugView",
+                self.rows.len(),
+                move |this, range, cx| {
+                    this.rows[range]
+                        .iter()
+                        .enumerate()
+                        .map(|(ix, row)| match row {
+                            Row::Worktree(root_path) => div()
+                                .id(ix)
+                                .child(Label::new(root_path.to_string_lossy().to_string())),
+                            Row::Entry(worktree_id, file_path) => div()
+                                .id(ix)
+                                .pl_8()
+                                .child(Label::new(file_path.to_string_lossy().to_string()))
+                                .on_mouse_move(cx.listener(move |this, _: &MouseMoveEvent, cx| {
+                                    if this.hovered_row_ix != Some(ix) {
+                                        this.hovered_row_ix = Some(ix);
+                                        cx.notify();
+                                    }
+                                }))
+                                .cursor(CursorStyle::PointingHand)
+                                .on_click(cx.listener({
+                                    let worktree_id = *worktree_id;
+                                    let file_path = file_path.clone();
+                                    move |this, _, cx| {
+                                        this.handle_path_click(worktree_id, file_path.clone(), cx);
+                                    }
+                                })),
+                        })
+                        .collect()
+                },
+            )
+            .track_scroll(self.list_scroll_handle.clone())
+            .size_full()
+            .text_bg(cx.theme().colors().background)
+            .into_any_element();
+
+            canvas(
+                move |bounds, cx| {
+                    list.prepaint_as_root(bounds.origin, bounds.size.into(), cx);
+                    list
+                },
+                |_, mut list, cx| list.paint(cx),
+            )
+            .size_full()
+            .into_any_element()
+        }
+    }
+}
+
+impl EventEmitter<()> for ProjectIndexDebugView {}
+
+impl Item for ProjectIndexDebugView {
+    type Event = ();
+
+    fn tab_content(&self, params: TabContentParams, _: &WindowContext<'_>) -> AnyElement {
+        Label::new("Project Index (Debug)")
+            .color(if params.selected {
+                Color::Default
+            } else {
+                Color::Muted
+            })
+            .into_any_element()
+    }
+
+    fn clone_on_split(
+        &self,
+        _: workspace::WorkspaceId,
+        cx: &mut ViewContext<Self>,
+    ) -> Option<View<Self>>
+    where
+        Self: Sized,
+    {
+        Some(cx.new_view(|cx| Self::new(self.index.clone(), cx)))
+    }
+}
+
+impl FocusableView for ProjectIndexDebugView {
+    fn focus_handle(&self, _: &AppContext) -> gpui::FocusHandle {
+        self.focus_handle.clone()
+    }
+}

crates/semantic_index/src/semantic_index.rs

@@ -1,5 +1,6 @@
 mod chunking;
 mod embedding;
+mod project_index_debug_view;
 
 use anyhow::{anyhow, Context as _, Result};
 use chunking::{chunk_text, Chunk};
@@ -31,6 +32,8 @@ use std::{
 use util::ResultExt;
 use worktree::LocalSnapshot;
 
+pub use project_index_debug_view::ProjectIndexDebugView;
+
 pub struct SemanticIndex {
     embedding_provider: Arc<dyn EmbeddingProvider>,
     db_connection: heed::Env,
@@ -397,26 +400,35 @@ impl ProjectIndex {
         Ok(result)
     }
 
-    pub fn debug(&self, cx: &mut ModelContext<Self>) -> Task<Result<()>> {
+    pub(crate) fn worktree_index(
-        let indices = self
+        &self,
+        worktree_id: WorktreeId,
+        cx: &AppContext,
+    ) -> Option<Model<WorktreeIndex>> {
+        for index in self.worktree_indices.values() {
+            if let WorktreeIndexHandle::Loaded { index, .. } = index {
+                if index.read(cx).worktree.read(cx).id() == worktree_id {
+                    return Some(index.clone());
+                }
+            }
+        }
+        None
+    }
+
+    pub(crate) fn worktree_indices(&self, cx: &AppContext) -> Vec<Model<WorktreeIndex>> {
+        let mut result = self
             .worktree_indices
             .values()
-            .filter_map(|worktree_index| {
+            .filter_map(|index| {
-                if let WorktreeIndexHandle::Loaded { index, .. } = worktree_index {
+                if let WorktreeIndexHandle::Loaded { index, .. } = index {
                     Some(index.clone())
                 } else {
                     None
                 }
             })
             .collect::<Vec<_>>();
-
+        result.sort_by_key(|index| index.read(cx).worktree.read(cx).id());
-        cx.spawn(|_, mut cx| async move {
+        result
-            eprintln!("semantic index contents:");
-            for index in indices {
-                index.update(&mut cx, |index, cx| index.debug(cx))?.await?
-            }
-            Ok(())
-        })
     }
 }
 
@@ -726,10 +738,8 @@ impl WorktreeIndex {
                                     .language_for_file_path(&entry.path)
                                     .await
                                     .ok();
-                                let grammar =
-                                    language.as_ref().and_then(|language| language.grammar());
                                 let chunked_file = ChunkedFile {
-                                    chunks: chunk_text(&text, grammar),
+                                    chunks: chunk_text(&text, language.as_ref(), &entry.path),
                                     handle,
                                     path: entry.path,
                                     mtime: entry.mtime,
@@ -861,7 +871,6 @@ impl WorktreeIndex {
                     db.put(&mut txn, &key, file)?;
                 }
                 txn.commit()?;
-                eprintln!("committed {:?}", embedded_files.len());
 
                 drop(embedded_files);
                 log::debug!("committed");
@@ -871,18 +880,38 @@ impl WorktreeIndex {
         })
     }
 
-    fn debug(&mut self, cx: &mut ModelContext<Self>) -> Task<Result<()>> {
+    fn paths(&self, cx: &AppContext) -> Task<Result<Vec<Arc<Path>>>> {
         let connection = self.db_connection.clone();
         let db = self.db;
         cx.background_executor().spawn(async move {
             let tx = connection
                 .read_txn()
                 .context("failed to create read transaction")?;
-            for record in db.iter(&tx)? {
+            let result = db
-                let (key, _) = record?;
+                .iter(&tx)?
-                eprintln!("{}", path_for_db_key(key));
+                .map(|entry| Ok(entry?.1.path.clone()))
-            }
+                .collect::<Result<Vec<Arc<Path>>>>();
-            Ok(())
+            drop(tx);
+            result
+        })
+    }
+
+    fn chunks_for_path(
+        &self,
+        path: Arc<Path>,
+        cx: &AppContext,
+    ) -> Task<Result<Vec<EmbeddedChunk>>> {
+        let connection = self.db_connection.clone();
+        let db = self.db;
+        cx.background_executor().spawn(async move {
+            let tx = connection
+                .read_txn()
+                .context("failed to create read transaction")?;
+            Ok(db
+                .get(&tx, &db_key_for_path(&path))?
+                .ok_or_else(|| anyhow!("no such path"))?
+                .chunks
+                .clone())
         })
     }
 
@@ -927,7 +956,7 @@ struct EmbeddedFile {
     chunks: Vec<EmbeddedChunk>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 struct EmbeddedChunk {
     chunk: Chunk,
     embedding: Embedding,
@@ -981,10 +1010,6 @@ fn db_key_for_path(path: &Arc<Path>) -> String {
     path.to_string_lossy().replace('/', "\0")
 }
 
-fn path_for_db_key(key: &str) -> String {
-    key.replace('\0', "/")
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;