Reload grammars in extensions when they are updated on disk (#7531)

Release Notes:

- N/A

---------

Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>
Co-authored-by: Marshall <marshall@zed.dev>

crates/editor/src/editor.rs

@@ -8772,6 +8772,10 @@ impl Editor {
                 cx.emit(EditorEvent::ExcerptsRemoved { ids: ids.clone() })
             }
             multi_buffer::Event::Reparsed => cx.emit(EditorEvent::Reparsed),
+            multi_buffer::Event::LanguageChanged => {
+                cx.emit(EditorEvent::Reparsed);
+                cx.notify();
+            }
             multi_buffer::Event::DirtyChanged => cx.emit(EditorEvent::DirtyChanged),
             multi_buffer::Event::Saved => cx.emit(EditorEvent::Saved),
             multi_buffer::Event::FileHandleChanged | multi_buffer::Event::Reloaded => {

crates/extension/src/extension_store.rs

@@ -1,5 +1,5 @@
-use anyhow::Result;
-use collections::{HashMap, HashSet};
+use anyhow::{Context as _, Result};
+use collections::HashMap;
 use fs::Fs;
 use futures::StreamExt as _;
 use gpui::{actions, AppContext, Context, Global, Model, ModelContext, Task};
@@ -36,7 +36,7 @@ impl Global for GlobalExtensionStore {}
 
 #[derive(Deserialize, Serialize, Default)]
 pub struct Manifest {
-    pub grammars: HashMap<String, GrammarManifestEntry>,
+    pub grammars: HashMap<Arc<str>, GrammarManifestEntry>,
     pub languages: HashMap<Arc<str>, LanguageManifestEntry>,
     pub themes: HashMap<String, ThemeManifestEntry>,
 }
@@ -52,6 +52,7 @@ pub struct LanguageManifestEntry {
     extension: String,
     path: PathBuf,
     matcher: LanguageMatcher,
+    grammar: Option<Arc<str>>,
 }
 
 #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)]
@@ -152,6 +153,7 @@ impl ExtensionStore {
             self.language_registry.register_extension(
                 language_path.into(),
                 language_name.clone(),
+                language.grammar.clone(),
                 language.matcher.clone(),
                 load_plugin_queries,
             );
@@ -188,19 +190,29 @@ impl ExtensionStore {
         let events_task = cx.background_executor().spawn(async move {
             let mut events = fs.watch(&extensions_dir, Duration::from_millis(250)).await;
             while let Some(events) = events.next().await {
-                let mut changed_languages = HashSet::default();
-                let mut changed_themes = HashSet::default();
+                let mut changed_grammars = Vec::default();
+                let mut changed_languages = Vec::default();
+                let mut changed_themes = Vec::default();
 
                 {
                     let manifest = manifest.read();
                     for event in events {
+                        for (grammar_name, grammar) in &manifest.grammars {
+                            let mut grammar_path = extensions_dir.clone();
+                            grammar_path
+                                .extend([grammar.extension.as_ref(), grammar.path.as_path()]);
+                            if event.path.starts_with(&grammar_path) || event.path == grammar_path {
+                                changed_grammars.push(grammar_name.clone());
+                            }
+                        }
+
                         for (language_name, language) in &manifest.languages {
                             let mut language_path = extensions_dir.clone();
                             language_path
                                 .extend([language.extension.as_ref(), language.path.as_path()]);
                             if event.path.starts_with(&language_path) || event.path == language_path
                             {
-                                changed_languages.insert(language_name.clone());
+                                changed_languages.push(language_name.clone());
                             }
                         }
 
@@ -208,18 +220,19 @@ impl ExtensionStore {
                             let mut theme_path = extensions_dir.clone();
                             theme_path.extend([theme.extension.as_ref(), theme.path.as_path()]);
                             if event.path.starts_with(&theme_path) || event.path == theme_path {
-                                changed_themes.insert(theme_path.clone());
+                                changed_themes.push(theme_path.clone());
                             }
                         }
                     }
                 }
 
-                language_registry.reload_languages(&changed_languages);
+                language_registry.reload_languages(&changed_languages, &changed_grammars);
 
                 for theme_path in &changed_themes {
                     theme_registry
                         .load_user_theme(&theme_path, fs.clone())
                         .await
+                        .context("failed to load user theme")
                         .log_err();
                 }
 
@@ -253,7 +266,10 @@ impl ExtensionStore {
                 .spawn(async move {
                     let mut manifest = Manifest::default();
 
-                    let mut extension_paths = fs.read_dir(&extensions_dir).await?;
+                    let mut extension_paths = fs
+                        .read_dir(&extensions_dir)
+                        .await
+                        .context("failed to read extensions directory")?;
                     while let Some(extension_dir) = extension_paths.next().await {
                         let extension_dir = extension_dir?;
                         let Some(extension_name) =
@@ -305,6 +321,7 @@ impl ExtensionStore {
                                         extension: extension_name.into(),
                                         path: relative_path.into(),
                                         matcher: config.matcher,
+                                        grammar: config.grammar,
                                     },
                                 );
                             }
@@ -345,7 +362,8 @@ impl ExtensionStore {
                         &serde_json::to_string_pretty(&manifest)?.as_str().into(),
                         Default::default(),
                     )
-                    .await?;
+                    .await
+                    .context("failed to save extension manifest")?;
 
                     anyhow::Ok(manifest)
                 })

crates/extension/src/extension_store_test.rs

@@ -106,6 +106,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
                 LanguageManifestEntry {
                     extension: "zed-ruby".into(),
                     path: "languages/erb".into(),
+                    grammar: Some("embedded_template".into()),
                     matcher: LanguageMatcher {
                         path_suffixes: vec!["erb".into()],
                         first_line_pattern: None,
@@ -117,6 +118,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
                 LanguageManifestEntry {
                     extension: "zed-ruby".into(),
                     path: "languages/ruby".into(),
+                    grammar: Some("ruby".into()),
                     matcher: LanguageMatcher {
                         path_suffixes: vec!["rb".into()],
                         first_line_pattern: None,

crates/language/src/buffer.rs

@@ -758,6 +758,7 @@ impl Buffer {
 
     /// Assign a language to the buffer.
     pub fn set_language(&mut self, language: Option<Arc<Language>>, cx: &mut ModelContext<Self>) {
+        self.parse_count += 1;
         self.syntax_map.lock().clear();
         self.language = language;
         self.reparse(cx);

crates/language/src/language.rs

@@ -740,14 +740,16 @@ type AvailableLanguageId = usize;
 struct AvailableLanguage {
     id: AvailableLanguageId,
     name: Arc<str>,
+    grammar: Option<Arc<str>>,
     source: AvailableLanguageSource,
     lsp_adapters: Vec<Arc<dyn LspAdapter>>,
     loaded: bool,
 }
 
 enum AvailableGrammar {
-    Loaded(tree_sitter::Language),
-    Loading(Vec<oneshot::Sender<Result<tree_sitter::Language>>>),
+    Native(tree_sitter::Language),
+    Loaded(PathBuf, tree_sitter::Language),
+    Loading(PathBuf, Vec<oneshot::Sender<Result<tree_sitter::Language>>>),
     Unloaded(PathBuf),
 }
 
@@ -781,7 +783,7 @@ struct LanguageRegistryState {
     next_language_server_id: usize,
     languages: Vec<Arc<Language>>,
     available_languages: Vec<AvailableLanguage>,
-    grammars: HashMap<String, AvailableGrammar>,
+    grammars: HashMap<Arc<str>, AvailableGrammar>,
     next_available_language_id: AvailableLanguageId,
     loading_languages: HashMap<AvailableLanguageId, Vec<oneshot::Sender<Result<Arc<Language>>>>>,
     subscription: (watch::Sender<()>, watch::Receiver<()>),
@@ -834,8 +836,8 @@ impl LanguageRegistry {
     }
 
     /// Clear out the given languages and reload them from scratch.
-    pub fn reload_languages(&self, languages: &HashSet<Arc<str>>) {
-        self.state.write().reload_languages(languages);
+    pub fn reload_languages(&self, languages: &[Arc<str>], grammars: &[Arc<str>]) {
+        self.state.write().reload_languages(languages, grammars);
     }
 
     pub fn register(
@@ -849,6 +851,7 @@ impl LanguageRegistry {
         state.available_languages.push(AvailableLanguage {
             id: post_inc(&mut state.next_available_language_id),
             name: config.name.clone(),
+            grammar: config.grammar.clone(),
             source: AvailableLanguageSource::BuiltIn {
                 config,
                 get_queries,
@@ -863,6 +866,7 @@ impl LanguageRegistry {
         &self,
         path: Arc<Path>,
         name: Arc<str>,
+        grammar_name: Option<Arc<str>>,
         matcher: LanguageMatcher,
         get_queries: fn(&Path) -> LanguageQueries,
     ) {
@@ -885,6 +889,7 @@ impl LanguageRegistry {
         }
         state.available_languages.push(AvailableLanguage {
             id: post_inc(&mut state.next_available_language_id),
+            grammar: grammar_name,
             name,
             source,
             lsp_adapters: Vec::new(),
@@ -894,16 +899,16 @@ impl LanguageRegistry {
 
     pub fn add_grammars(
         &self,
-        grammars: impl IntoIterator<Item = (impl Into<String>, tree_sitter::Language)>,
+        grammars: impl IntoIterator<Item = (impl Into<Arc<str>>, tree_sitter::Language)>,
     ) {
         self.state.write().grammars.extend(
             grammars
                 .into_iter()
-                .map(|(name, grammar)| (name.into(), AvailableGrammar::Loaded(grammar))),
+                .map(|(name, grammar)| (name.into(), AvailableGrammar::Native(grammar))),
         );
     }
 
-    pub fn register_grammar(&self, name: String, path: PathBuf) {
+    pub fn register_grammar(&self, name: Arc<str>, path: PathBuf) {
         self.state
             .write()
             .grammars
@@ -1124,46 +1129,49 @@ impl LanguageRegistry {
 
         if let Some(grammar) = state.grammars.get_mut(name.as_ref()) {
             match grammar {
-                AvailableGrammar::Loaded(grammar) => {
+                AvailableGrammar::Native(grammar) | AvailableGrammar::Loaded(_, grammar) => {
                     tx.send(Ok(grammar.clone())).ok();
                 }
-                AvailableGrammar::Loading(txs) => {
+                AvailableGrammar::Loading(_, txs) => {
                     txs.push(tx);
                 }
                 AvailableGrammar::Unloaded(wasm_path) => {
                     if let Some(executor) = &self.executor {
                         let this = self.clone();
-                        let wasm_path = wasm_path.clone();
                         executor
-                            .spawn(async move {
-                                let wasm_bytes = std::fs::read(&wasm_path)?;
-                                let grammar_name = wasm_path
-                                    .file_stem()
-                                    .and_then(OsStr::to_str)
-                                    .ok_or_else(|| anyhow!("invalid grammar filename"))?;
-                                let grammar = PARSER.with(|parser| {
-                                    let mut parser = parser.borrow_mut();
-                                    let mut store = parser.take_wasm_store().unwrap();
-                                    let grammar = store.load_language(&grammar_name, &wasm_bytes);
-                                    parser.set_wasm_store(store).unwrap();
-                                    grammar
-                                })?;
+                            .spawn({
+                                let wasm_path = wasm_path.clone();
+                                async move {
+                                    let wasm_bytes = std::fs::read(&wasm_path)?;
+                                    let grammar_name = wasm_path
+                                        .file_stem()
+                                        .and_then(OsStr::to_str)
+                                        .ok_or_else(|| anyhow!("invalid grammar filename"))?;
+                                    let grammar = PARSER.with(|parser| {
+                                        let mut parser = parser.borrow_mut();
+                                        let mut store = parser.take_wasm_store().unwrap();
+                                        let grammar =
+                                            store.load_language(&grammar_name, &wasm_bytes);
+                                        parser.set_wasm_store(store).unwrap();
+                                        grammar
+                                    })?;
 
-                                if let Some(AvailableGrammar::Loading(txs)) =
-                                    this.state.write().grammars.insert(
-                                        name.to_string(),
-                                        AvailableGrammar::Loaded(grammar.clone()),
-                                    )
-                                {
-                                    for tx in txs {
-                                        tx.send(Ok(grammar.clone())).ok();
+                                    if let Some(AvailableGrammar::Loading(_, txs)) =
+                                        this.state.write().grammars.insert(
+                                            name,
+                                            AvailableGrammar::Loaded(wasm_path, grammar.clone()),
+                                        )
+                                    {
+                                        for tx in txs {
+                                            tx.send(Ok(grammar.clone())).ok();
+                                        }
                                     }
-                                }
 
-                                anyhow::Ok(())
+                                    anyhow::Ok(())
+                                }
                             })
                             .detach();
-                        *grammar = AvailableGrammar::Loading(vec![tx]);
+                        *grammar = AvailableGrammar::Loading(wasm_path.clone(), vec![tx]);
                     }
                 }
             }
@@ -1357,16 +1365,42 @@ impl LanguageRegistryState {
         *self.subscription.0.borrow_mut() = ();
     }
 
-    fn reload_languages(&mut self, languages: &HashSet<Arc<str>>) {
-        self.languages
-            .retain(|language| !languages.contains(&language.config.name));
-        self.version += 1;
-        self.reload_count += 1;
+    fn reload_languages(
+        &mut self,
+        languages_to_reload: &[Arc<str>],
+        grammars_to_reload: &[Arc<str>],
+    ) {
+        for (name, grammar) in self.grammars.iter_mut() {
+            if grammars_to_reload.contains(name) {
+                if let AvailableGrammar::Loaded(path, _) = grammar {
+                    *grammar = AvailableGrammar::Unloaded(path.clone());
+                }
+            }
+        }
+
+        self.languages.retain(|language| {
+            let should_reload = languages_to_reload.contains(&language.config.name)
+                || language
+                    .config
+                    .grammar
+                    .as_ref()
+                    .map_or(false, |grammar| grammars_to_reload.contains(&grammar));
+            !should_reload
+        });
+
         for language in &mut self.available_languages {
-            if languages.contains(&language.name) {
+            if languages_to_reload.contains(&language.name)
+                || language
+                    .grammar
+                    .as_ref()
+                    .map_or(false, |grammar| grammars_to_reload.contains(grammar))
+            {
                 language.loaded = false;
             }
         }
+
+        self.version += 1;
+        self.reload_count += 1;
         *self.subscription.0.borrow_mut() = ();
     }
 

crates/util/src/paths.rs

@@ -23,15 +23,10 @@ lazy_static::lazy_static! {
         CONFIG_DIR.join("support")
     };
     pub static ref EXTENSIONS_DIR: PathBuf = if cfg!(target_os="macos") {
-        HOME.join("Library/Application Support/Zed")
+        HOME.join("Library/Application Support/Zed/extensions")
     } else {
         CONFIG_DIR.join("extensions")
     };
-    pub static ref PLUGINS_DIR: PathBuf = if cfg!(target_os="macos") {
-        HOME.join("Library/Application Support/Zed/plugins")
-    } else {
-        CONFIG_DIR.join("plugins")
-    };
     pub static ref LANGUAGES_DIR: PathBuf = if cfg!(target_os="macos") {
         HOME.join("Library/Application Support/Zed/languages")
     } else {