Introduce a new Grammar struct and allow it to be optional

Co-Authored-By: Nathan Sobo <nathan@zed.dev>

Cargo.lock

@@ -5132,16 +5132,6 @@ dependencies = [
  "regex",
 ]
 
-[[package]]
-name = "tree-sitter-markdown"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7c1cfe0396b0e500cc99067bcd2d48720eb08a077e2690194f0c3a3d105f9a0"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tree-sitter-rust"
 version = "0.19.0"
@@ -5731,7 +5721,6 @@ dependencies = [
  "tiny_http",
  "toml",
  "tree-sitter",
- "tree-sitter-markdown",
  "tree-sitter-rust",
  "unindent",
  "url",

crates/editor/src/display_map.rs

@@ -778,7 +778,7 @@ mod tests {
                     path_suffixes: vec![".test".to_string()],
                     ..Default::default()
                 },
-                tree_sitter_rust::language(),
+                Some(tree_sitter_rust::language()),
             )
             .with_highlights_query(
                 r#"
@@ -865,7 +865,7 @@ mod tests {
                     path_suffixes: vec![".test".to_string()],
                     ..Default::default()
                 },
-                tree_sitter_rust::language(),
+                Some(tree_sitter_rust::language()),
             )
             .with_highlights_query(
                 r#"

crates/editor/src/lib.rs

@@ -5295,7 +5295,7 @@ mod tests {
         let settings = cx.read(EditorSettings::test);
         let language = Some(Arc::new(Language::new(
             LanguageConfig::default(),
-            tree_sitter_rust::language(),
+            Some(tree_sitter_rust::language()),
         )));
 
         let text = r#"
@@ -5452,7 +5452,7 @@ mod tests {
                 ],
                 ..Default::default()
             },
-            tree_sitter_rust::language(),
+            Some(tree_sitter_rust::language()),
         )));
 
         let text = r#"
@@ -5551,7 +5551,7 @@ mod tests {
                 line_comment: Some("// ".to_string()),
                 ..Default::default()
             },
-            tree_sitter_rust::language(),
+            Some(tree_sitter_rust::language()),
         )));
 
         let text = "
@@ -5649,7 +5649,7 @@ mod tests {
                 ],
                 ..Default::default()
             },
-            tree_sitter_rust::language(),
+            Some(tree_sitter_rust::language()),
         )));
 
         let text = concat!(

crates/language/src/language.rs

@@ -1,12 +1,12 @@
 use crate::HighlightMap;
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use gpui::{executor::Background, AppContext};
 use lsp::LanguageServer;
 use parking_lot::Mutex;
 use serde::Deserialize;
 use std::{collections::HashSet, path::Path, str, sync::Arc};
 use theme::SyntaxTheme;
-use tree_sitter::{Language as Grammar, Query};
+use tree_sitter::{self, Query};
 pub use tree_sitter::{Parser, Tree};
 
 #[derive(Default, Deserialize)]
@@ -37,7 +37,11 @@ pub struct BracketPair {
 
 pub struct Language {
     pub(crate) config: LanguageConfig,
-    pub(crate) grammar: Grammar,
+    pub(crate) grammar: Option<Arc<Grammar>>,
+}
+
+pub struct Grammar {
+    pub(crate) ts_language: tree_sitter::Language,
     pub(crate) highlights_query: Query,
     pub(crate) brackets_query: Query,
     pub(crate) indents_query: Query,
@@ -86,29 +90,48 @@ impl LanguageRegistry {
 }
 
 impl Language {
-    pub fn new(config: LanguageConfig, grammar: Grammar) -> Self {
+    pub fn new(config: LanguageConfig, ts_language: Option<tree_sitter::Language>) -> Self {
         Self {
             config,
-            brackets_query: Query::new(grammar, "").unwrap(),
-            highlights_query: Query::new(grammar, "").unwrap(),
-            indents_query: Query::new(grammar, "").unwrap(),
-            grammar,
-            highlight_map: Default::default(),
+            grammar: ts_language.map(|ts_language| {
+                Arc::new(Grammar {
+                    brackets_query: Query::new(ts_language, "").unwrap(),
+                    highlights_query: Query::new(ts_language, "").unwrap(),
+                    indents_query: Query::new(ts_language, "").unwrap(),
+                    ts_language,
+                    highlight_map: Default::default(),
+                })
+            }),
         }
     }
 
     pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
-        self.highlights_query = Query::new(self.grammar, source)?;
+        let grammar = self
+            .grammar
+            .as_mut()
+            .and_then(Arc::get_mut)
+            .ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?;
+        grammar.highlights_query = Query::new(grammar.ts_language, source)?;
         Ok(self)
     }
 
     pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
-        self.brackets_query = Query::new(self.grammar, source)?;
+        let grammar = self
+            .grammar
+            .as_mut()
+            .and_then(Arc::get_mut)
+            .ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?;
+        grammar.brackets_query = Query::new(grammar.ts_language, source)?;
         Ok(self)
     }
 
     pub fn with_indents_query(mut self, source: &str) -> Result<Self> {
-        self.indents_query = Query::new(self.grammar, source)?;
+        let grammar = self
+            .grammar
+            .as_mut()
+            .and_then(Arc::get_mut)
+            .ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?;
+        grammar.indents_query = Query::new(grammar.ts_language, source)?;
         Ok(self)
     }
 
@@ -156,14 +179,18 @@ impl Language {
         &self.config.brackets
     }
 
+    pub fn set_theme(&self, theme: &SyntaxTheme) {
+        if let Some(grammar) = self.grammar.as_ref() {
+            *grammar.highlight_map.lock() =
+                HighlightMap::new(grammar.highlights_query.capture_names(), theme);
+        }
+    }
+}
+
+impl Grammar {
     pub fn highlight_map(&self) -> HighlightMap {
         self.highlight_map.lock().clone()
     }
-
-    pub fn set_theme(&self, theme: &SyntaxTheme) {
-        *self.highlight_map.lock() =
-            HighlightMap::new(self.highlights_query.capture_names(), theme);
-    }
 }
 
 #[cfg(any(test, feature = "test-support"))]
@@ -189,7 +216,6 @@ mod tests {
 
     #[test]
     fn test_select_language() {
-        let grammar = tree_sitter_rust::language();
         let registry = LanguageRegistry {
             languages: vec![
                 Arc::new(Language::new(
@@ -198,7 +224,7 @@ mod tests {
                         path_suffixes: vec!["rs".to_string()],
                         ..Default::default()
                     },
-                    grammar,
+                    Some(tree_sitter_rust::language()),
                 )),
                 Arc::new(Language::new(
                     LanguageConfig {
@@ -206,7 +232,7 @@ mod tests {
                         path_suffixes: vec!["Makefile".to_string(), "mk".to_string()],
                         ..Default::default()
                     },
-                    grammar,
+                    Some(tree_sitter_rust::language()),
                 )),
             ],
         };

crates/language/src/lib.rs

@@ -6,7 +6,9 @@ mod tests;
 
 pub use self::{
     highlight_map::{HighlightId, HighlightMap},
-    language::{BracketPair, Language, LanguageConfig, LanguageRegistry, LanguageServerConfig},
+    language::{
+        BracketPair, Grammar, Language, LanguageConfig, LanguageRegistry, LanguageServerConfig,
+    },
 };
 use anyhow::{anyhow, Result};
 pub use buffer::{Buffer as TextBuffer, Operation as _, *};
@@ -594,13 +596,13 @@ impl Buffer {
             return false;
         }
 
-        if let Some(language) = self.language.clone() {
+        if let Some(grammar) = self.grammar().cloned() {
             let old_tree = self.syntax_tree();
             let text = self.as_rope().clone();
             let parsed_version = self.version();
             let parse_task = cx.background().spawn({
-                let language = language.clone();
-                async move { Self::parse_text(&text, old_tree, &language) }
+                let grammar = grammar.clone();
+                async move { Self::parse_text(&text, old_tree, &grammar) }
             });
 
             match cx
@@ -616,11 +618,10 @@ impl Buffer {
                     cx.spawn(move |this, mut cx| async move {
                         let new_tree = parse_task.await;
                         this.update(&mut cx, move |this, cx| {
-                            let language_changed =
-                                this.language.as_ref().map_or(true, |curr_language| {
-                                    !Arc::ptr_eq(curr_language, &language)
-                                });
-                            let parse_again = this.version.gt(&parsed_version) || language_changed;
+                            let grammar_changed = this
+                                .grammar()
+                                .map_or(true, |curr_grammar| !Arc::ptr_eq(&grammar, curr_grammar));
+                            let parse_again = this.version.gt(&parsed_version) || grammar_changed;
                             this.parsing_in_background = false;
                             this.did_finish_parsing(new_tree, parsed_version, cx);
 
@@ -636,11 +637,11 @@ impl Buffer {
         false
     }
 
-    fn parse_text(text: &Rope, old_tree: Option<Tree>, language: &Language) -> Tree {
+    fn parse_text(text: &Rope, old_tree: Option<Tree>, grammar: &Grammar) -> Tree {
         PARSER.with(|parser| {
             let mut parser = parser.borrow_mut();
             parser
-                .set_language(language.grammar)
+                .set_language(grammar.ts_language)
                 .expect("incompatible grammar");
             let mut chunks = text.chunks_in_range(0..text.len());
             let tree = parser
@@ -1069,15 +1070,15 @@ impl Buffer {
         &self,
         range: Range<T>,
     ) -> Option<(Range<usize>, Range<usize>)> {
-        let (lang, tree) = self.language.as_ref().zip(self.syntax_tree())?;
-        let open_capture_ix = lang.brackets_query.capture_index_for_name("open")?;
-        let close_capture_ix = lang.brackets_query.capture_index_for_name("close")?;
+        let (grammar, tree) = self.grammar().zip(self.syntax_tree())?;
+        let open_capture_ix = grammar.brackets_query.capture_index_for_name("open")?;
+        let close_capture_ix = grammar.brackets_query.capture_index_for_name("close")?;
 
         // Find bracket pairs that *inclusively* contain the given range.
         let range = range.start.to_offset(self).saturating_sub(1)..range.end.to_offset(self) + 1;
         let mut cursor = QueryCursorHandle::new();
         let matches = cursor.set_byte_range(range).matches(
-            &lang.brackets_query,
+            &grammar.brackets_query,
             tree.root_node(),
             TextProvider(self.as_rope()),
         );
@@ -1342,6 +1343,10 @@ impl Buffer {
         cx.notify();
     }
 
+    fn grammar(&self) -> Option<&Arc<Grammar>> {
+        self.language.as_ref().and_then(|l| l.grammar.as_ref())
+    }
+
     pub fn add_selection_set<T: ToOffset>(
         &mut self,
         selections: &[Selection<T>],
@@ -1550,19 +1555,19 @@ impl Snapshot {
         row_range: Range<u32>,
     ) -> Option<impl Iterator<Item = IndentSuggestion> + 'a> {
         let mut query_cursor = QueryCursorHandle::new();
-        if let Some((language, tree)) = self.language.as_ref().zip(self.tree.as_ref()) {
+        if let Some((grammar, tree)) = self.grammar().zip(self.tree.as_ref()) {
             let prev_non_blank_row = self.prev_non_blank_row(row_range.start);
 
             // Get the "indentation ranges" that intersect this row range.
-            let indent_capture_ix = language.indents_query.capture_index_for_name("indent");
-            let end_capture_ix = language.indents_query.capture_index_for_name("end");
+            let indent_capture_ix = grammar.indents_query.capture_index_for_name("indent");
+            let end_capture_ix = grammar.indents_query.capture_index_for_name("end");
             query_cursor.set_point_range(
                 Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0).to_ts_point()
                     ..Point::new(row_range.end, 0).to_ts_point(),
             );
             let mut indentation_ranges = Vec::<(Range<Point>, &'static str)>::new();
             for mat in query_cursor.matches(
-                &language.indents_query,
+                &grammar.indents_query,
                 tree.root_node(),
                 TextProvider(self.as_rope()),
             ) {
@@ -1682,7 +1687,7 @@ impl Snapshot {
             diagnostic_endpoints
                 .sort_unstable_by_key(|endpoint| (endpoint.offset, !endpoint.is_start));
 
-            if let Some((language, tree)) = self.language.as_ref().zip(self.tree.as_ref()) {
+            if let Some((grammar, tree)) = self.grammar().zip(self.tree.as_ref()) {
                 let mut query_cursor = QueryCursorHandle::new();
 
                 // TODO - add a Tree-sitter API to remove the need for this.
@@ -1690,7 +1695,7 @@ impl Snapshot {
                     std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
                 };
                 let captures = cursor.set_byte_range(range.clone()).captures(
-                    &language.highlights_query,
+                    &grammar.highlights_query,
                     tree.root_node(),
                     TextProvider(self.text.as_rope()),
                 );
@@ -1698,7 +1703,7 @@ impl Snapshot {
                     captures,
                     next_capture: None,
                     stack: Default::default(),
-                    highlight_map: language.highlight_map(),
+                    highlight_map: grammar.highlight_map(),
                     _query_cursor: query_cursor,
                     theme,
                 })
@@ -1719,6 +1724,12 @@ impl Snapshot {
             highlights,
         }
     }
+
+    fn grammar(&self) -> Option<&Arc<Grammar>> {
+        self.language
+            .as_ref()
+            .and_then(|language| language.grammar.as_ref())
+    }
 }
 
 impl Clone for Snapshot {

crates/language/src/tests.rs

@@ -970,7 +970,7 @@ fn rust_lang() -> Language {
             language_server: None,
             ..Default::default()
         },
-        tree_sitter_rust::language(),
+        Some(tree_sitter_rust::language()),
     )
     .with_indents_query(
         r#"

crates/project/src/worktree.rs

@@ -3671,7 +3671,7 @@ mod tests {
                 language_server: Some(language_server_config),
                 ..Default::default()
             },
-            tree_sitter_rust::language(),
+            Some(tree_sitter_rust::language()),
         )));
 
         let dir = temp_tree(json!({

crates/server/src/rpc.rs

@@ -1616,7 +1616,7 @@ mod tests {
                 language_server: Some(language_server_config),
                 ..Default::default()
             },
-            tree_sitter_rust::language(),
+            Some(tree_sitter_rust::language()),
         )));
 
         let lang_registry = Arc::new(lang_registry);

crates/zed/Cargo.toml

@@ -85,7 +85,6 @@ time = "0.3"
 tiny_http = "0.8"
 toml = "0.5"
 tree-sitter = "0.19.5"
-tree-sitter-markdown = "0.7"
 tree-sitter-rust = "0.19.0"
 url = "2.2"
 

crates/zed/src/language.rs

@@ -17,7 +17,7 @@ pub fn build_language_registry() -> LanguageRegistry {
 fn rust() -> Language {
     let grammar = tree_sitter_rust::language();
     let config = toml::from_slice(&LanguageDir::get("rust/config.toml").unwrap().data).unwrap();
-    Language::new(config, grammar)
+    Language::new(config, Some(grammar))
         .with_highlights_query(load_query("rust/highlights.scm").as_ref())
         .unwrap()
         .with_brackets_query(load_query("rust/brackets.scm").as_ref())
@@ -27,9 +27,8 @@ fn rust() -> Language {
 }
 
 fn markdown() -> Language {
-    let grammar = tree_sitter_markdown::language();
     let config = toml::from_slice(&LanguageDir::get("markdown/config.toml").unwrap().data).unwrap();
-    Language::new(config, grammar)
+    Language::new(config, None)
 }
 
 fn load_query(path: &str) -> Cow<'static, str> {

crates/zed/src/test.rs

@@ -30,7 +30,7 @@ pub fn test_app_state(cx: &mut MutableAppContext) -> Arc<AppState> {
             path_suffixes: vec!["rs".to_string()],
             ..Default::default()
         },
-        tree_sitter_rust::language(),
+        Some(tree_sitter_rust::language()),
     )));
     Arc::new(AppState {
         settings_tx: Arc::new(Mutex::new(settings_tx)),