Add support for projects managed with Yarn (#13644)

TODO:
- [ ] File a PR with Yarn to add Zed to the list of supported IDEs.

Fixes: https://github.com/zed-industries/zed/issues/10107
Fixes: https://github.com/zed-industries/zed/issues/13706
Release Notes:

- Improved experience in projects using Yarn. Run `yarn dlx
@yarnpkg/sdks base` in the root of your project in order to elevate your
experience.

---------

Co-authored-by: Saurabh <79586784+m4saurabh@users.noreply.github.com>

crates/fs/src/fs.rs

@@ -67,7 +67,10 @@ pub trait Fs: Send + Sync {
         self.remove_file(path, options).await
     }
     async fn open_sync(&self, path: &Path) -> Result<Box<dyn io::Read>>;
-    async fn load(&self, path: &Path) -> Result<String>;
+    async fn load(&self, path: &Path) -> Result<String> {
+        Ok(String::from_utf8(self.load_bytes(path).await?)?)
+    }
+    async fn load_bytes(&self, path: &Path) -> Result<Vec<u8>>;
     async fn atomic_write(&self, path: PathBuf, text: String) -> Result<()>;
     async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()>;
     async fn canonicalize(&self, path: &Path) -> Result<PathBuf>;
@@ -318,6 +321,11 @@ impl Fs for RealFs {
         let text = smol::unblock(|| std::fs::read_to_string(path)).await?;
         Ok(text)
     }
+    async fn load_bytes(&self, path: &Path) -> Result<Vec<u8>> {
+        let path = path.to_path_buf();
+        let bytes = smol::unblock(|| std::fs::read(path)).await?;
+        Ok(bytes)
+    }
 
     async fn atomic_write(&self, path: PathBuf, data: String) -> Result<()> {
         smol::unblock(move || {
@@ -1433,6 +1441,10 @@ impl Fs for FakeFs {
         Ok(String::from_utf8(content.clone())?)
     }
 
+    async fn load_bytes(&self, path: &Path) -> Result<Vec<u8>> {
+        self.load_internal(path).await
+    }
+
     async fn atomic_write(&self, path: PathBuf, data: String) -> Result<()> {
         self.simulate_random_delay().await;
         let path = normalize_path(path.as_path());

crates/languages/src/typescript.rs

@@ -68,10 +68,22 @@ pub struct TypeScriptLspAdapter {
 impl TypeScriptLspAdapter {
     const OLD_SERVER_PATH: &'static str = "node_modules/typescript-language-server/lib/cli.js";
     const NEW_SERVER_PATH: &'static str = "node_modules/typescript-language-server/lib/cli.mjs";
-
+    const SERVER_NAME: &'static str = "typescript-language-server";
     pub fn new(node: Arc<dyn NodeRuntime>) -> Self {
         TypeScriptLspAdapter { node }
     }
+    async fn tsdk_path(adapter: &Arc<dyn LspAdapterDelegate>) -> &'static str {
+        let is_yarn = adapter
+            .read_text_file(PathBuf::from(".yarn/sdks/typescript/lib/typescript.js"))
+            .await
+            .is_ok();
+
+        if is_yarn {
+            ".yarn/sdks/typescript/lib"
+        } else {
+            "node_modules/typescript/lib"
+        }
+    }
 }
 
 struct TypeScriptVersions {
@@ -82,7 +94,7 @@ struct TypeScriptVersions {
 #[async_trait(?Send)]
 impl LspAdapter for TypeScriptLspAdapter {
     fn name(&self) -> LanguageServerName {
-        LanguageServerName("typescript-language-server".into())
+        LanguageServerName(Self::SERVER_NAME.into())
     }
 
     async fn fetch_latest_server_version(
@@ -196,13 +208,14 @@ impl LspAdapter for TypeScriptLspAdapter {
 
     async fn initialization_options(
         self: Arc<Self>,
-        _: &Arc<dyn LspAdapterDelegate>,
+        adapter: &Arc<dyn LspAdapterDelegate>,
     ) -> Result<Option<serde_json::Value>> {
+        let tsdk_path = Self::tsdk_path(adapter).await;
         Ok(Some(json!({
             "provideFormatter": true,
             "hostInfo": "zed",
             "tsserver": {
-                "path": "node_modules/typescript/lib",
+                "path": tsdk_path,
             },
             "preferences": {
                 "includeInlayParameterNameHints": "all",
@@ -220,8 +233,17 @@ impl LspAdapter for TypeScriptLspAdapter {
     async fn workspace_configuration(
         self: Arc<Self>,
         _: &Arc<dyn LspAdapterDelegate>,
-        _cx: &mut AsyncAppContext,
+        cx: &mut AsyncAppContext,
     ) -> Result<Value> {
+        let override_options = cx.update(|cx| {
+            ProjectSettings::get_global(cx)
+                .lsp
+                .get(Self::SERVER_NAME)
+                .and_then(|s| s.initialization_options.clone())
+        })?;
+        if let Some(options) = override_options {
+            return Ok(options);
+        }
         Ok(json!({
             "completions": {
               "completeFunctionCalls": true

crates/languages/src/vtsls.rs

@@ -5,7 +5,9 @@ use gpui::AsyncAppContext;
 use language::{LanguageServerName, LspAdapter, LspAdapterDelegate};
 use lsp::{CodeActionKind, LanguageServerBinary};
 use node_runtime::NodeRuntime;
+use project::project_settings::ProjectSettings;
 use serde_json::{json, Value};
+use settings::Settings;
 use std::{
     any::Any,
     ffi::OsString,
@@ -28,6 +30,18 @@ impl VtslsLspAdapter {
     pub fn new(node: Arc<dyn NodeRuntime>) -> Self {
         VtslsLspAdapter { node }
     }
+    async fn tsdk_path(adapter: &Arc<dyn LspAdapterDelegate>) -> &'static str {
+        let is_yarn = adapter
+            .read_text_file(PathBuf::from(".yarn/sdks/typescript/lib/typescript.js"))
+            .await
+            .is_ok();
+
+        if is_yarn {
+            ".yarn/sdks/typescript/lib"
+        } else {
+            "node_modules/typescript/lib"
+        }
+    }
 }
 
 struct TypeScriptVersions {
@@ -35,10 +49,11 @@ struct TypeScriptVersions {
     server_version: String,
 }
 
+const SERVER_NAME: &'static str = "vtsls";
 #[async_trait(?Send)]
 impl LspAdapter for VtslsLspAdapter {
     fn name(&self) -> LanguageServerName {
-        LanguageServerName("vtsls".into())
+        LanguageServerName(SERVER_NAME.into())
     }
 
     async fn fetch_latest_server_version(
@@ -159,11 +174,12 @@ impl LspAdapter for VtslsLspAdapter {
 
     async fn initialization_options(
         self: Arc<Self>,
-        _: &Arc<dyn LspAdapterDelegate>,
+        adapter: &Arc<dyn LspAdapterDelegate>,
     ) -> Result<Option<serde_json::Value>> {
+        let tsdk_path = Self::tsdk_path(&adapter).await;
         Ok(Some(json!({
             "typescript": {
-                "tsdk": "node_modules/typescript/lib",
+                "tsdk": tsdk_path,
                 "format": {
                     "enable": true
                 },
@@ -196,22 +212,33 @@ impl LspAdapter for VtslsLspAdapter {
                         "enableServerSideFuzzyMatch": true,
                         "entriesLimit": 5000,
                     }
-                }
+                },
+               "autoUseWorkspaceTsdk": true
             }
         })))
     }
 
     async fn workspace_configuration(
         self: Arc<Self>,
-        _: &Arc<dyn LspAdapterDelegate>,
+        adapter: &Arc<dyn LspAdapterDelegate>,
-        _cx: &mut AsyncAppContext,
+        cx: &mut AsyncAppContext,
     ) -> Result<Value> {
+        let override_options = cx.update(|cx| {
+            ProjectSettings::get_global(cx)
+                .lsp
+                .get(SERVER_NAME)
+                .and_then(|s| s.initialization_options.clone())
+        })?;
+        if let Some(options) = override_options {
+            return Ok(options);
+        }
+        let tsdk_path = Self::tsdk_path(&adapter).await;
         Ok(json!({
             "typescript": {
                 "suggest": {
                     "completeFunctionCalls": true
                 },
-                "tsdk": "node_modules/typescript/lib",
+                "tsdk": tsdk_path,
                 "format": {
                     "enable": true
                 },
@@ -244,7 +271,8 @@ impl LspAdapter for VtslsLspAdapter {
                         "enableServerSideFuzzyMatch": true,
                         "entriesLimit": 5000,
                     }
-                }
+                },
+                "autoUseWorkspaceTsdk": true
             }
         }))
     }

crates/project/src/project.rs

@@ -11,6 +11,7 @@ pub mod terminals;
 #[cfg(test)]
 mod project_tests;
 pub mod search_history;
+mod yarn;
 
 use anyhow::{anyhow, bail, Context as _, Result};
 use async_trait::async_trait;
@@ -116,6 +117,7 @@ use util::{
     NumericPrefixWithSuffix, ResultExt, TryFutureExt as _,
 };
 use worktree::{CreatedEntry, RemoteWorktreeClient, Snapshot, Traversal};
+use yarn::YarnPathStore;
 
 pub use fs::*;
 pub use language::Location;
@@ -231,6 +233,7 @@ pub struct Project {
     dev_server_project_id: Option<client::DevServerProjectId>,
     search_history: SearchHistory,
     snippets: Model<SnippetProvider>,
+    yarn: Model<YarnPathStore>,
 }
 
 pub enum LanguageServerToQuery {
@@ -728,6 +731,7 @@ impl Project {
             let global_snippets_dir = paths::config_dir().join("snippets");
             let snippets =
                 SnippetProvider::new(fs.clone(), BTreeSet::from_iter([global_snippets_dir]), cx);
+            let yarn = YarnPathStore::new(fs.clone(), cx);
             Self {
                 worktrees: Vec::new(),
                 worktrees_reordered: false,
@@ -753,6 +757,7 @@ impl Project {
                 _maintain_buffer_languages: Self::maintain_buffer_languages(languages.clone(), cx),
                 _maintain_workspace_config: Self::maintain_workspace_config(cx),
                 active_entry: None,
+                yarn,
                 snippets,
                 languages,
                 client,
@@ -853,6 +858,7 @@ impl Project {
             let global_snippets_dir = paths::config_dir().join("snippets");
             let snippets =
                 SnippetProvider::new(fs.clone(), BTreeSet::from_iter([global_snippets_dir]), cx);
+            let yarn = YarnPathStore::new(fs.clone(), cx);
             // BIG CAUTION NOTE: The order in which we initialize fields here matters and it should match what's done in Self::local.
             // Otherwise, you might run into issues where worktree id on remote is different than what's on local host.
             // That's because Worktree's identifier is entity id, which should probably be changed.
@@ -891,6 +897,7 @@ impl Project {
                 languages,
                 user_store: user_store.clone(),
                 snippets,
+                yarn,
                 fs,
                 next_entry_id: Default::default(),
                 next_diagnostic_group_id: Default::default(),
@@ -2163,23 +2170,51 @@ impl Project {
     /// LanguageServerName is owned, because it is inserted into a map
     pub fn open_local_buffer_via_lsp(
         &mut self,
-        abs_path: lsp::Url,
+        mut abs_path: lsp::Url,
         language_server_id: LanguageServerId,
         language_server_name: LanguageServerName,
         cx: &mut ModelContext<Self>,
     ) -> Task<Result<Model<Buffer>>> {
         cx.spawn(move |this, mut cx| async move {
+            // Escape percent-encoded string.
+            let current_scheme = abs_path.scheme().to_owned();
+            let _ = abs_path.set_scheme("file");
+
             let abs_path = abs_path
                 .to_file_path()
                 .map_err(|_| anyhow!("can't convert URI to path"))?;
-            let (worktree, relative_path) = if let Some(result) =
+            let p = abs_path.clone();
-                this.update(&mut cx, |this, cx| this.find_local_worktree(&abs_path, cx))?
+            let yarn_worktree = this
-            {
+                .update(&mut cx, move |this, cx| {
-                result
+                    this.yarn.update(cx, |_, cx| {
+                        cx.spawn(|this, mut cx| async move {
+                            let t = this
+                                .update(&mut cx, |this, cx| {
+                                    this.process_path(&p, &current_scheme, cx)
+                                })
+                                .ok()?;
+                            t.await
+                        })
+                    })
+                })?
+                .await;
+            let (worktree_root_target, known_relative_path) =
+                if let Some((zip_root, relative_path)) = yarn_worktree {
+                    (zip_root, Some(relative_path))
+                } else {
+                    (Arc::<Path>::from(abs_path.as_path()), None)
+                };
+            let (worktree, relative_path) = if let Some(result) = this
+                .update(&mut cx, |this, cx| {
+                    this.find_local_worktree(&worktree_root_target, cx)
+                })? {
+                let relative_path =
+                    known_relative_path.unwrap_or_else(|| Arc::<Path>::from(result.1));
+                (result.0, relative_path)
             } else {
                 let worktree = this
                     .update(&mut cx, |this, cx| {
-                        this.create_local_worktree(&abs_path, false, cx)
+                        this.create_local_worktree(&worktree_root_target, false, cx)
                     })?
                     .await?;
                 this.update(&mut cx, |this, cx| {
@@ -2189,12 +2224,17 @@ impl Project {
                     );
                 })
                 .ok();
-                (worktree, PathBuf::new())
+                let worktree_root = worktree.update(&mut cx, |this, _| this.abs_path())?;
+                let relative_path = if let Some(known_path) = known_relative_path {
+                    known_path
+                } else {
+                    abs_path.strip_prefix(worktree_root)?.into()
+                };
+                (worktree, relative_path)
             };
-
             let project_path = ProjectPath {
                 worktree_id: worktree.update(&mut cx, |worktree, _| worktree.id())?,
-                path: relative_path.into(),
+                path: relative_path,
             };
             this.update(&mut cx, |this, cx| this.open_buffer(project_path, cx))?
                 .await

crates/project/src/yarn.rs

@@ -0,0 +1,177 @@
+//! This module deals with everything related to path handling for Yarn, the package manager for Web ecosystem.
+//! Yarn is a bit peculiar, because it references paths within .zip files, which we obviously can't handle.
+//! It also uses virtual paths for peer dependencies.
+//!
+//! Long story short, before we attempt to resolve a path as a "real" path, we try to treat is as a yarn path;
+//! for .zip handling, we unpack the contents into the temp directory (yes, this is bad, against the spirit of Yarn and what-not)
+
+use std::{
+    ffi::OsStr,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use anyhow::Result;
+use collections::HashMap;
+use fs::Fs;
+use gpui::{AppContext, Context, Model, ModelContext, Task};
+use util::ResultExt;
+
+pub(crate) struct YarnPathStore {
+    temp_dirs: HashMap<Arc<Path>, tempfile::TempDir>,
+    fs: Arc<dyn Fs>,
+}
+
+/// Returns `None` when passed path is a malformed virtual path or it's not a virtual path at all.
+fn resolve_virtual(path: &Path) -> Option<Arc<Path>> {
+    let components: Vec<_> = path.components().collect();
+    let mut non_virtual_path = PathBuf::new();
+
+    let mut i = 0;
+    let mut is_virtual = false;
+    while i < components.len() {
+        if let Some(os_str) = components[i].as_os_str().to_str() {
+            // Detect the __virtual__ segment
+            if os_str == "__virtual__" {
+                let pop_count = components
+                    .get(i + 2)?
+                    .as_os_str()
+                    .to_str()?
+                    .parse::<usize>()
+                    .ok()?;
+
+                // Apply dirname operation pop_count times
+                for _ in 0..pop_count {
+                    non_virtual_path.pop();
+                }
+                i += 3; // Skip hash and pop_count components
+                is_virtual = true;
+                continue;
+            }
+        }
+        non_virtual_path.push(&components[i]);
+        i += 1;
+    }
+
+    is_virtual.then(|| Arc::from(non_virtual_path))
+}
+
+impl YarnPathStore {
+    pub(crate) fn new(fs: Arc<dyn Fs>, cx: &mut AppContext) -> Model<Self> {
+        cx.new_model(|_| Self {
+            temp_dirs: Default::default(),
+            fs,
+        })
+    }
+    pub(crate) fn process_path(
+        &mut self,
+        path: &Path,
+        protocol: &str,
+        cx: &ModelContext<Self>,
+    ) -> Task<Option<(Arc<Path>, Arc<Path>)>> {
+        let mut is_zip = protocol.eq("zip");
+
+        let path: &Path = if let Some(non_zip_part) = path
+            .as_os_str()
+            .as_encoded_bytes()
+            .strip_prefix("/zip:".as_bytes())
+        {
+            // typescript-language-server prepends the paths with zip:, which is messy.
+            is_zip = true;
+            Path::new(OsStr::new(
+                std::str::from_utf8(non_zip_part).expect("Invalid UTF-8"),
+            ))
+        } else {
+            path
+        };
+
+        let as_virtual = resolve_virtual(&path);
+        let Some(path) = as_virtual.or_else(|| is_zip.then(|| Arc::from(path))) else {
+            return Task::ready(None);
+        };
+        if let Some(zip_file) = zip_path(&path) {
+            let zip_file: Arc<Path> = Arc::from(zip_file);
+            cx.spawn(|this, mut cx| async move {
+                let dir = this
+                    .update(&mut cx, |this, _| {
+                        this.temp_dirs
+                            .get(&zip_file)
+                            .map(|temp| temp.path().to_owned())
+                    })
+                    .ok()?;
+                let zip_root = if let Some(dir) = dir {
+                    dir
+                } else {
+                    let fs = this.update(&mut cx, |this, _| this.fs.clone()).ok()?;
+                    let tempdir = dump_zip(zip_file.clone(), fs).await.log_err()?;
+                    let new_path = tempdir.path().to_owned();
+                    this.update(&mut cx, |this, _| {
+                        this.temp_dirs.insert(zip_file.clone(), tempdir);
+                    })
+                    .ok()?;
+                    new_path
+                };
+                // Rebase zip-path onto new temp path.
+                let as_relative = path.strip_prefix(zip_file).ok()?.into();
+                Some((zip_root.into(), as_relative))
+            })
+        } else {
+            Task::ready(None)
+        }
+    }
+}
+
+fn zip_path(path: &Path) -> Option<&Path> {
+    let path_str = path.to_str()?;
+    let zip_end = path_str.find(".zip/")?;
+    let zip_path = &path_str[..zip_end + 4]; // ".zip" is 4 characters long
+    Some(Path::new(zip_path))
+}
+
+async fn dump_zip(path: Arc<Path>, fs: Arc<dyn Fs>) -> Result<tempfile::TempDir> {
+    let dir = tempfile::tempdir()?;
+    let contents = fs.load_bytes(&path).await?;
+    node_runtime::extract_zip(dir.path(), futures::io::Cursor::new(contents)).await?;
+    Ok(dir)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::Path;
+
+    #[test]
+    fn test_resolve_virtual() {
+        let test_cases = vec![
+            (
+                "/path/to/some/folder/__virtual__/a0b1c2d3/0/subpath/to/file.dat",
+                Some(Path::new("/path/to/some/folder/subpath/to/file.dat")),
+            ),
+            (
+                "/path/to/some/folder/__virtual__/e4f5a0b1/0/subpath/to/file.dat",
+                Some(Path::new("/path/to/some/folder/subpath/to/file.dat")),
+            ),
+            (
+                "/path/to/some/folder/__virtual__/a0b1c2d3/1/subpath/to/file.dat",
+                Some(Path::new("/path/to/some/subpath/to/file.dat")),
+            ),
+            (
+                "/path/to/some/folder/__virtual__/a0b1c2d3/3/subpath/to/file.dat",
+                Some(Path::new("/path/subpath/to/file.dat")),
+            ),
+            ("/path/to/nonvirtual/", None),
+            ("/path/to/malformed/__virtual__", None),
+            ("/path/to/malformed/__virtual__/a0b1c2d3", None),
+            (
+                "/path/to/malformed/__virtual__/a0b1c2d3/this-should-be-a-number",
+                None,
+            ),
+        ];
+
+        for (input, expected) in test_cases {
+            let input_path = Path::new(input);
+            let resolved_path = resolve_virtual(input_path);
+            assert_eq!(resolved_path.as_deref(), expected);
+        }
+    }
+}

docs/src/SUMMARY.md

@@ -62,6 +62,7 @@
 - [Uiua](./languages/uiua.md)
 - [Vue](./languages/vue.md)
 - [YAML](./languages/yaml.md)
+- [Yarn](./languages/yarn.md)
 - [Zig](./languages/zig.md)
 
 # Developing Zed

docs/src/languages/javascript.md

@@ -142,3 +142,7 @@ You can configure ESLint's `rulesCustomizations` setting:
   }
 }
 ```
+
+
+## Yarn integration
+See [Yarn documentation](./yarn.md) for a walkthrough of configuring your project to use Yarn.

docs/src/languages/typescript.md

@@ -3,7 +3,8 @@
 TypeScript and TSX support are available natively in Zed.
 
 - Tree Sitter: [tree-sitter-typescript](https://github.com/tree-sitter/tree-sitter-typescript)
-- Language Server: [typescript-language-server](https://github.com/typescript-language-server/typescript-language-server)
+- Language Server: [vtsls](https://github.com/yioneko/vtsls)
+- Alternate Language Server: [typescript-language-server](https://github.com/typescript-language-server/typescript-language-server)
 
 ## Inlay Hints
 
@@ -41,3 +42,6 @@ Use
 to override these settings.
 
 See https://github.com/typescript-language-server/typescript-language-server?tab=readme-ov-file#inlay-hints-textdocumentinlayhint for more information.
+
+## Yarn integration
+See [Yarn documentation](./yarn.md) for a walkthrough of configuring your project to use Yarn.

docs/src/languages/yarn.md

@@ -0,0 +1,8 @@
+# Yarn
+[Yarn](https://yarnpkg.com/) is a versatile package manager that improves dependency management and workflow efficiency for JavaScript and other languages. It ensures a deterministic dependency tree, offers offline support, and enhances security for reliable builds.
+
+## Setup
+
+1. Run `yarn dlx @yarnpkg/sdks base` to generate a `.yarn/sdks` directory.
+2. Set your language server (e.g. VTSLS) to use Typescript SDK from `.yarn/sdks/typescript/lib` directory in [LSP initialization options](../configuring-zed.md#lsp). The actual setting for that depends on language server; for example, for VTSLS you should set [`typescript.tsdk`](https://github.com/yioneko/vtsls/blob/6adfb5d3889ad4b82c5e238446b27ae3ee1e3767/packages/service/configuration.schema.json#L5).
+3. Voilla! Language server functionalities such as Go to Definition, Code Completions and On Hover documentation should work.