From 64e512232dd6dbe6c885c656441a7cf1ec57449b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 31 Dec 2023 16:19:48 -0800 Subject: [PATCH] Start work on supporting custom languages --- Cargo.lock | 436 ++++++++++++++++++++++--- Cargo.toml | 7 +- crates/language/src/language.rs | 18 +- crates/language/src/syntax_map.rs | 2 +- crates/language2/src/language2.rs | 109 +++++-- crates/language2/src/syntax_map.rs | 11 +- crates/semantic_index/src/parsing.rs | 2 +- crates/semantic_index2/src/parsing.rs | 2 +- crates/settings/src/settings_store.rs | 4 +- crates/settings2/src/settings_store.rs | 4 +- crates/util/src/paths.rs | 1 + crates/zed2/src/languages.rs | 17 +- 12 files changed, 518 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7256782e31..eeabd6aad9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,6 +327,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + [[package]] name = "arrayref" version = "0.3.7" @@ -2370,7 +2376,15 @@ version = "0.89.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "593b398dd0c5b1e2e3a9c3dae8584e287894ea84e361949ad506376e99196265" dependencies = [ - "cranelift-entity", + "cranelift-entity 0.89.2", +] + +[[package]] +name = "cranelift-bforest" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cranelift-entity 0.103.0", ] [[package]] @@ -2381,14 +2395,34 @@ checksum = "afc0d8faabd099ea15ab33d49d150e5572c04cfeb95d675fd41286739b754629" dependencies = [ "arrayvec 0.7.4", "bumpalo", - "cranelift-bforest", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-entity", - "cranelift-isle", + "cranelift-bforest 0.89.2", + "cranelift-codegen-meta 0.89.2", + "cranelift-codegen-shared 0.89.2", + "cranelift-entity 0.89.2", + "cranelift-isle 0.89.2", "gimli 0.26.2", "log", - "regalloc2", + "regalloc2 0.4.2", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "bumpalo", + "cranelift-bforest 0.103.0", + "cranelift-codegen-meta 0.103.0", + "cranelift-codegen-shared 0.103.0", + "cranelift-control", + "cranelift-entity 0.103.0", + "cranelift-isle 0.103.0", + "gimli 0.28.0", + "hashbrown 0.14.0", + "log", + "regalloc2 0.9.3", "smallvec", "target-lexicon", ] @@ -2399,7 +2433,15 @@ version = "0.89.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ac1669e42579476f001571d6ba4b825fac686282c97b88b18f8e34242066a81" dependencies = [ - "cranelift-codegen-shared", + "cranelift-codegen-shared 0.89.2", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cranelift-codegen-shared 0.103.0", ] [[package]] @@ -2408,6 +2450,19 @@ version = "0.89.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2a1b1eef9640ab72c1e7b583ac678083855a509da34b4b4378bd99954127c20" +[[package]] +name = "cranelift-codegen-shared" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" + +[[package]] +name = "cranelift-control" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "arbitrary", +] + [[package]] name = "cranelift-entity" version = "0.89.2" @@ -2417,13 +2472,33 @@ dependencies = [ "serde", ] +[[package]] +name = "cranelift-entity" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "cranelift-frontend" version = "0.89.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fca1474b5302348799656d43a40eacd716a3b46169405a3af812832c9edf77b4" dependencies = [ - "cranelift-codegen", + "cranelift-codegen 0.89.2", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-frontend" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cranelift-codegen 0.103.0", "log", "smallvec", "target-lexicon", @@ -2435,13 +2510,28 @@ version = "0.89.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77aa537f020ea43483100153278e7215d41695bdcef9eea6642d122675f64249" +[[package]] +name = "cranelift-isle" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" + [[package]] name = "cranelift-native" version = "0.89.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bdc6b65241a95b7d8eafbf4e114c082e49b80162a2dcd9c6bcc5989c3310c9e" dependencies = [ - "cranelift-codegen", + "cranelift-codegen 0.89.2", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-native" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cranelift-codegen 0.103.0", "libc", "target-lexicon", ] @@ -2452,14 +2542,29 @@ version = "0.89.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb6359f606a1c80ccaa04fae9dbbb504615ec7a49b6c212b341080fff7a65dd" dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", + "cranelift-codegen 0.89.2", + "cranelift-entity 0.89.2", + "cranelift-frontend 0.89.2", "itertools 0.10.5", "log", "smallvec", - "wasmparser", - "wasmtime-types", + "wasmparser 0.92.0", + "wasmtime-types 2.0.2", +] + +[[package]] +name = "cranelift-wasm" +version = "0.103.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cranelift-codegen 0.103.0", + "cranelift-entity 0.103.0", + "cranelift-frontend 0.103.0", + "itertools 0.10.5", + "log", + "smallvec", + "wasmparser 0.118.1", + "wasmtime-types 16.0.0", ] [[package]] @@ -3202,6 +3307,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fallible-streaming-iterator" version = "0.1.9" @@ -3840,7 +3951,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" dependencies = [ - "fallible-iterator", + "fallible-iterator 0.2.0", "indexmap 1.9.3", "stable_deref_trait", ] @@ -3850,6 +3961,11 @@ name = "gimli" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +dependencies = [ + "fallible-iterator 0.3.0", + "indexmap 2.0.0", + "stable_deref_trait", +] [[package]] name = "git" @@ -4484,6 +4600,7 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", "hashbrown 0.14.0", + "serde", ] [[package]] @@ -6181,6 +6298,9 @@ version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ + "crc32fast", + "hashbrown 0.14.0", + "indexmap 2.0.0", "memchr", ] @@ -6706,7 +6826,7 @@ dependencies = [ "serde_json", "smol", "wasi-common", - "wasmtime", + "wasmtime 2.0.2", "wasmtime-wasi", ] @@ -7541,6 +7661,19 @@ dependencies = [ "smallvec", ] +[[package]] +name = "regalloc2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6" +dependencies = [ + "hashbrown 0.13.2", + "log", + "rustc-hash", + "slice-group-by", + "smallvec", +] + [[package]] name = "regex" version = "1.9.5" @@ -7923,7 +8056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" dependencies = [ "bitflags 2.4.1", - "fallible-iterator", + "fallible-iterator 0.2.0", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", @@ -8994,6 +9127,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be6c3f39c37a4283ee4b43d1311c828f2e1fb0541e76ea0cb1a2abd9ef2f5b3b" +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + [[package]] name = "sqlez" version = "0.1.0" @@ -10334,10 +10473,12 @@ dependencies = [ [[package]] name = "tree-sitter" version = "0.20.10" -source = "git+https://github.com/tree-sitter/tree-sitter?rev=b5f461a69bf3df7298b1903574d506179e6390b0#b5f461a69bf3df7298b1903574d506179e6390b0" +source = "git+https://github.com/tree-sitter/tree-sitter?rev=31c40449749c4263a91a43593831b82229049a4c#31c40449749c4263a91a43593831b82229049a4c" dependencies = [ "cc", "regex", + "wasmtime 16.0.0", + "wasmtime-c-api-impl", ] [[package]] @@ -11210,6 +11351,16 @@ dependencies = [ "indexmap 1.9.3", ] +[[package]] +name = "wasmparser" +version = "0.118.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ee9723b928e735d53000dec9eae7b07a60e490c85ab54abb66659fc61bfcd9" +dependencies = [ + "indexmap 2.0.0", + "semver", +] + [[package]] name = "wasmtime" version = "2.0.2" @@ -11230,17 +11381,44 @@ dependencies = [ "rayon", "serde", "target-lexicon", - "wasmparser", + "wasmparser 0.92.0", "wasmtime-cache", - "wasmtime-cranelift", - "wasmtime-environ", + "wasmtime-cranelift 2.0.2", + "wasmtime-environ 2.0.2", "wasmtime-fiber", - "wasmtime-jit", - "wasmtime-runtime", + "wasmtime-jit 2.0.2", + "wasmtime-runtime 2.0.2", "wat", "windows-sys 0.36.1", ] +[[package]] +name = "wasmtime" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "anyhow", + "bincode", + "bumpalo", + "cfg-if 1.0.0", + "indexmap 2.0.0", + "libc", + "log", + "object 0.32.1", + "once_cell", + "paste", + "serde", + "serde_derive", + "serde_json", + "target-lexicon", + "wasmparser 0.118.1", + "wasmtime-cranelift 16.0.0", + "wasmtime-environ 16.0.0", + "wasmtime-jit 16.0.0", + "wasmtime-runtime 16.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "wasmtime-asm-macros" version = "2.0.2" @@ -11250,6 +11428,36 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "wasmtime-asm-macros" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "wasmtime-c-api-impl" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "anyhow", + "log", + "once_cell", + "tracing", + "wasmtime 16.0.0", + "wasmtime-c-api-macros", +] + +[[package]] +name = "wasmtime-c-api-macros" +version = "0.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "wasmtime-cache" version = "2.0.2" @@ -11277,18 +11485,57 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017c3605ccce867b3ba7f71d95e5652acc22b9dc2971ad6a6f9df4a8d7af2648" dependencies = [ "anyhow", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", + "cranelift-codegen 0.89.2", + "cranelift-entity 0.89.2", + "cranelift-frontend 0.89.2", + "cranelift-native 0.89.2", + "cranelift-wasm 0.89.2", "gimli 0.26.2", "log", "object 0.29.0", "target-lexicon", "thiserror", - "wasmparser", - "wasmtime-environ", + "wasmparser 0.92.0", + "wasmtime-environ 2.0.2", +] + +[[package]] +name = "wasmtime-cranelift" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "anyhow", + "cfg-if 1.0.0", + "cranelift-codegen 0.103.0", + "cranelift-control", + "cranelift-entity 0.103.0", + "cranelift-frontend 0.103.0", + "cranelift-native 0.103.0", + "cranelift-wasm 0.103.0", + "gimli 0.28.0", + "log", + "object 0.32.1", + "target-lexicon", + "thiserror", + "wasmparser 0.118.1", + "wasmtime-cranelift-shared", + "wasmtime-environ 16.0.0", + "wasmtime-versioned-export-macros", +] + +[[package]] +name = "wasmtime-cranelift-shared" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "anyhow", + "cranelift-codegen 0.103.0", + "cranelift-control", + "cranelift-native 0.103.0", + "gimli 0.28.0", + "object 0.32.1", + "target-lexicon", + "wasmtime-environ 16.0.0", ] [[package]] @@ -11298,7 +11545,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aec5c1f81aab9bb35997113c171b6bb9093afc90e3757c55e0c08dc9ac612e4" dependencies = [ "anyhow", - "cranelift-entity", + "cranelift-entity 0.89.2", "gimli 0.26.2", "indexmap 1.9.3", "log", @@ -11306,8 +11553,27 @@ dependencies = [ "serde", "target-lexicon", "thiserror", - "wasmparser", - "wasmtime-types", + "wasmparser 0.92.0", + "wasmtime-types 2.0.2", +] + +[[package]] +name = "wasmtime-environ" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "anyhow", + "cranelift-entity 0.103.0", + "gimli 0.28.0", + "indexmap 2.0.0", + "log", + "object 0.32.1", + "serde", + "serde_derive", + "target-lexicon", + "thiserror", + "wasmparser 0.118.1", + "wasmtime-types 16.0.0", ] [[package]] @@ -11319,7 +11585,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "rustix 0.35.16", - "wasmtime-asm-macros", + "wasmtime-asm-macros 2.0.2", "windows-sys 0.36.1", ] @@ -11343,12 +11609,33 @@ dependencies = [ "serde", "target-lexicon", "thiserror", - "wasmtime-environ", + "wasmtime-environ 2.0.2", "wasmtime-jit-debug", - "wasmtime-runtime", + "wasmtime-runtime 2.0.2", "windows-sys 0.36.1", ] +[[package]] +name = "wasmtime-jit" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "anyhow", + "bincode", + "cfg-if 1.0.0", + "gimli 0.28.0", + "log", + "object 0.32.1", + "rustix 0.38.21", + "serde", + "serde_derive", + "target-lexicon", + "wasmtime-environ 16.0.0", + "wasmtime-jit-icache-coherence", + "wasmtime-runtime 16.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "wasmtime-jit-debug" version = "2.0.2" @@ -11360,6 +11647,16 @@ dependencies = [ "rustix 0.35.16", ] +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "wasmtime-runtime" version = "2.0.2" @@ -11379,23 +11676,71 @@ dependencies = [ "rand 0.8.5", "rustix 0.35.16", "thiserror", - "wasmtime-asm-macros", - "wasmtime-environ", + "wasmtime-asm-macros 2.0.2", + "wasmtime-environ 2.0.2", "wasmtime-fiber", "wasmtime-jit-debug", "windows-sys 0.36.1", ] +[[package]] +name = "wasmtime-runtime" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "anyhow", + "cc", + "cfg-if 1.0.0", + "indexmap 2.0.0", + "libc", + "log", + "mach", + "memfd", + "memoffset 0.9.0", + "paste", + "psm", + "rustix 0.38.21", + "sptr", + "wasm-encoder", + "wasmtime-asm-macros 16.0.0", + "wasmtime-environ 16.0.0", + "wasmtime-versioned-export-macros", + "wasmtime-wmemcheck", + "windows-sys 0.48.0", +] + [[package]] name = "wasmtime-types" version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dc3dd9521815984b35d6362f79e6b9c72475027cd1c71c44eb8df8fbf33a9fb" dependencies = [ - "cranelift-entity", + "cranelift-entity 0.89.2", "serde", "thiserror", - "wasmparser", + "wasmparser 0.92.0", +] + +[[package]] +name = "wasmtime-types" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "cranelift-entity 0.103.0", + "serde", + "serde_derive", + "thiserror", + "wasmparser 0.118.1", +] + +[[package]] +name = "wasmtime-versioned-export-macros" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", ] [[package]] @@ -11407,10 +11752,15 @@ dependencies = [ "anyhow", "wasi-cap-std-sync", "wasi-common", - "wasmtime", + "wasmtime 2.0.2", "wiggle", ] +[[package]] +name = "wasmtime-wmemcheck" +version = "16.0.0" +source = "git+https://github.com/bytecodealliance/wasmtime?rev=v16.0.0#6613acd1e4817957a4a7745125ef063b43c273a7" + [[package]] name = "wast" version = "35.0.2" @@ -11565,7 +11915,7 @@ dependencies = [ "bitflags 1.3.2", "thiserror", "tracing", - "wasmtime", + "wasmtime 2.0.2", "wiggle-macro", ] diff --git a/Cargo.toml b/Cargo.toml index e8299d6ce3..3ba64e4530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,6 @@ members = [ "crates/picker2", "crates/plugin", "crates/plugin_macros", - "crates/plugin_runtime", "crates/prettier", "crates/prettier2", "crates/project", @@ -176,11 +175,12 @@ thiserror = { version = "1.0.29" } time = { version = "0.3", features = ["serde", "serde-well-known"] } toml = { version = "0.5" } tiktoken-rs = "0.5.7" -tree-sitter = "0.20" +tree-sitter = { version = "0.20", features = ["wasm"] } unindent = { version = "0.1.7" } pretty_assertions = "1.3.0" git2 = { version = "0.15", default-features = false} uuid = { version = "1.1.2", features = ["v4"] } +wasmtime = "16" tree-sitter-bash = { git = "https://github.com/tree-sitter/tree-sitter-bash", rev = "7331995b19b8f8aba2d5e26deb51d2195c18bc94" } tree-sitter-c = "0.20.1" @@ -212,8 +212,9 @@ tree-sitter-vue = {git = "https://github.com/zed-industries/tree-sitter-vue", re tree-sitter-uiua = {git = "https://github.com/shnarazk/tree-sitter-uiua", rev = "9260f11be5900beda4ee6d1a24ab8ddfaf5a19b2"} [patch.crates-io] -tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "b5f461a69bf3df7298b1903574d506179e6390b0" } +tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "31c40449749c4263a91a43593831b82229049a4c" } async-task = { git = "https://github.com/zed-industries/async-task", rev = "341b57d6de98cdfd7b418567b8de2022ca993a6e" } +wasmtime = { git = "https://github.com/bytecodealliance/wasmtime", rev = "v16.0.0" } # TODO - Remove when a version is released with this PR: https://github.com/servo/core-foundation-rs/pull/457 cocoa = { git = "https://github.com/servo/core-foundation-rs", rev = "079665882507dd5e2ff77db3de5070c1f6c0fb85" } diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs index 811e549406..2c286249f2 100644 --- a/crates/language/src/language.rs +++ b/crates/language/src/language.rs @@ -1169,7 +1169,7 @@ impl Language { indents_config: None, injection_config: None, override_config: None, - error_query: Query::new(ts_language, "(ERROR) @error").unwrap(), + error_query: Query::new(&ts_language, "(ERROR) @error").unwrap(), ts_language, highlight_map: Default::default(), }) @@ -1230,13 +1230,13 @@ impl Language { pub fn with_highlights_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?); + grammar.highlights_query = Some(Query::new(&grammar.ts_language, source)?); Ok(self) } pub fn with_outline_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut item_capture_ix = None; let mut name_capture_ix = None; let mut context_capture_ix = None; @@ -1264,7 +1264,7 @@ impl Language { pub fn with_embedding_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut item_capture_ix = None; let mut name_capture_ix = None; let mut context_capture_ix = None; @@ -1295,7 +1295,7 @@ impl Language { pub fn with_brackets_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut open_capture_ix = None; let mut close_capture_ix = None; get_capture_indices( @@ -1317,7 +1317,7 @@ impl Language { pub fn with_indents_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut indent_capture_ix = None; let mut start_capture_ix = None; let mut end_capture_ix = None; @@ -1345,7 +1345,7 @@ impl Language { pub fn with_injection_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut language_capture_ix = None; let mut content_capture_ix = None; get_capture_indices( @@ -1384,7 +1384,7 @@ impl Language { } pub fn with_override_query(mut self, source: &str) -> anyhow::Result { - let query = Query::new(self.grammar_mut().ts_language, source)?; + let query = Query::new(&self.grammar_mut().ts_language, source)?; let mut override_configs_by_id = HashMap::default(); for (ix, name) in query.capture_names().iter().copied().enumerate() { @@ -1695,7 +1695,7 @@ impl Grammar { PARSER.with(|parser| { let mut parser = parser.borrow_mut(); parser - .set_language(self.ts_language) + .set_language(&self.ts_language) .expect("incompatible grammar"); let mut chunks = text.chunks_in_range(0..text.len()); parser diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs index 18f2e9b264..43d30fca60 100644 --- a/crates/language/src/syntax_map.rs +++ b/crates/language/src/syntax_map.rs @@ -1166,7 +1166,7 @@ fn parse_text( .set_included_ranges(&ranges) .expect("overlapping ranges"); parser - .set_language(grammar.ts_language) + .set_language(&grammar.ts_language) .expect("incompatible grammar"); parser .parse_with( diff --git a/crates/language2/src/language2.rs b/crates/language2/src/language2.rs index 8fdf524f69..2a75050bb0 100644 --- a/crates/language2/src/language2.rs +++ b/crates/language2/src/language2.rs @@ -44,7 +44,7 @@ use std::{ }; use syntax_map::SyntaxSnapshot; use theme::{SyntaxTheme, Theme}; -use tree_sitter::{self, Query}; +use tree_sitter::{self, wasmtime, Query, WasmStore}; use unicase::UniCase; use util::{http::HttpClient, paths::PathExt}; use util::{post_inc, ResultExt, TryFutureExt as _, UnwrapFuture}; @@ -84,10 +84,15 @@ impl LspBinaryStatusSender { } thread_local! { - static PARSER: RefCell = RefCell::new(Parser::new()); + static PARSER: RefCell = { + let mut parser = Parser::new(); + parser.set_wasm_store(WasmStore::new(WASM_ENGINE.clone()).unwrap()).unwrap(); + RefCell::new(parser) + }; } lazy_static! { + pub static ref WASM_ENGINE: wasmtime::Engine = wasmtime::Engine::default(); pub static ref NEXT_GRAMMAR_ID: AtomicUsize = Default::default(); pub static ref PLAIN_TEXT: Arc = Arc::new(Language::new( LanguageConfig { @@ -360,6 +365,7 @@ pub struct CodeLabel { #[derive(Clone, Deserialize)] pub struct LanguageConfig { pub name: Arc, + pub grammar_name: Option>, pub path_suffixes: Vec, pub brackets: BracketPairConfig, #[serde(default, deserialize_with = "deserialize_regex")] @@ -446,6 +452,7 @@ impl Default for LanguageConfig { fn default() -> Self { Self { name: "".into(), + grammar_name: None, path_suffixes: Default::default(), brackets: Default::default(), auto_indent_using_last_non_empty_line: auto_indent_using_last_non_empty_line_default(), @@ -620,14 +627,25 @@ type AvailableLanguageId = usize; #[derive(Clone)] struct AvailableLanguage { id: AvailableLanguageId, - path: &'static str, config: LanguageConfig, - grammar: tree_sitter::Language, + grammar: AvailableGrammar, lsp_adapters: Vec>, - get_queries: fn(&str) -> LanguageQueries, loaded: bool, } +#[derive(Clone)] +enum AvailableGrammar { + Native { + grammar: tree_sitter::Language, + asset_dir: &'static str, + get_queries: fn(&str) -> LanguageQueries, + }, + Wasm { + grammar_name: Arc, + path: Arc, + }, +} + pub struct LanguageRegistry { state: RwLock, language_server_download_dir: Option>, @@ -699,7 +717,7 @@ impl LanguageRegistry { pub fn register( &self, - path: &'static str, + asset_dir: &'static str, config: LanguageConfig, grammar: tree_sitter::Language, lsp_adapters: Vec>, @@ -708,11 +726,24 @@ impl LanguageRegistry { let state = &mut *self.state.write(); state.available_languages.push(AvailableLanguage { id: post_inc(&mut state.next_available_language_id), - path, config, - grammar, + grammar: AvailableGrammar::Native { + grammar, + get_queries, + asset_dir, + }, lsp_adapters, - get_queries, + loaded: false, + }); + } + + pub fn register_wasm(&self, path: Arc, grammar_name: Arc, config: LanguageConfig) { + let state = &mut *self.state.write(); + state.available_languages.push(AvailableLanguage { + id: post_inc(&mut state.next_available_language_id), + config, + grammar: AvailableGrammar::Wasm { grammar_name, path }, + lsp_adapters: Vec::new(), loaded: false, }); } @@ -837,13 +868,43 @@ impl LanguageRegistry { executor .spawn(async move { let id = language.id; - let queries = (language.get_queries)(&language.path); - let language = - Language::new(language.config, Some(language.grammar)) + let name = language.config.name.clone(); + let language = async { + let (grammar, queries) = match language.grammar { + AvailableGrammar::Native { + grammar, + asset_dir, + get_queries, + } => (grammar, (get_queries)(asset_dir)), + AvailableGrammar::Wasm { grammar_name, path } => { + let mut wasm_path = path.join(grammar_name.as_ref()); + wasm_path.set_extension("wasm"); + let wasm_bytes = std::fs::read(&wasm_path)?; + let grammar = PARSER.with(|parser| { + let mut parser = parser.borrow_mut(); + let mut store = parser.take_wasm_store().unwrap(); + let grammar = + store.load_language(&grammar_name, &wasm_bytes); + parser.set_wasm_store(store).unwrap(); + grammar + })?; + let mut queries = LanguageQueries::default(); + if let Ok(contents) = std::fs::read_to_string( + &path.join("highlights.scm"), + ) { + queries.highlights = Some(contents.into()); + } + (grammar, queries) + } + }; + Language::new(language.config, Some(grammar)) .with_lsp_adapters(language.lsp_adapters) - .await; - let name = language.name(); - match language.with_queries(queries) { + .await + .with_queries(queries) + } + .await; + + match language { Ok(language) => { let language = Arc::new(language); let mut state = this.state.write(); @@ -1175,7 +1236,7 @@ impl Language { indents_config: None, injection_config: None, override_config: None, - error_query: Query::new(ts_language, "(ERROR) @error").unwrap(), + error_query: Query::new(&ts_language, "(ERROR) @error").unwrap(), ts_language, highlight_map: Default::default(), }) @@ -1236,13 +1297,13 @@ impl Language { pub fn with_highlights_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?); + grammar.highlights_query = Some(Query::new(&grammar.ts_language, source)?); Ok(self) } pub fn with_outline_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut item_capture_ix = None; let mut name_capture_ix = None; let mut context_capture_ix = None; @@ -1270,7 +1331,7 @@ impl Language { pub fn with_embedding_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut item_capture_ix = None; let mut name_capture_ix = None; let mut context_capture_ix = None; @@ -1301,7 +1362,7 @@ impl Language { pub fn with_brackets_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut open_capture_ix = None; let mut close_capture_ix = None; get_capture_indices( @@ -1323,7 +1384,7 @@ impl Language { pub fn with_indents_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut indent_capture_ix = None; let mut start_capture_ix = None; let mut end_capture_ix = None; @@ -1351,7 +1412,7 @@ impl Language { pub fn with_injection_query(mut self, source: &str) -> Result { let grammar = self.grammar_mut(); - let query = Query::new(grammar.ts_language, source)?; + let query = Query::new(&grammar.ts_language, source)?; let mut language_capture_ix = None; let mut content_capture_ix = None; get_capture_indices( @@ -1390,7 +1451,7 @@ impl Language { } pub fn with_override_query(mut self, source: &str) -> anyhow::Result { - let query = Query::new(self.grammar_mut().ts_language, source)?; + let query = Query::new(&self.grammar_mut().ts_language, source)?; let mut override_configs_by_id = HashMap::default(); for (ix, name) in query.capture_names().iter().enumerate() { @@ -1701,7 +1762,7 @@ impl Grammar { PARSER.with(|parser| { let mut parser = parser.borrow_mut(); parser - .set_language(self.ts_language) + .set_language(&self.ts_language) .expect("incompatible grammar"); let mut chunks = text.chunks_in_range(0..text.len()); parser diff --git a/crates/language2/src/syntax_map.rs b/crates/language2/src/syntax_map.rs index 18f2e9b264..c22ece48af 100644 --- a/crates/language2/src/syntax_map.rs +++ b/crates/language2/src/syntax_map.rs @@ -7,7 +7,6 @@ use futures::FutureExt; use parking_lot::Mutex; use std::{ borrow::Cow, - cell::RefCell, cmp::{self, Ordering, Reverse}, collections::BinaryHeap, fmt, iter, @@ -16,13 +15,9 @@ use std::{ }; use sum_tree::{Bias, SeekTarget, SumTree}; use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint}; -use tree_sitter::{ - Node, Parser, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree, -}; +use tree_sitter::{Node, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree}; -thread_local! { - static PARSER: RefCell = RefCell::new(Parser::new()); -} +use super::PARSER; static QUERY_CURSORS: Mutex> = Mutex::new(vec![]); @@ -1166,7 +1161,7 @@ fn parse_text( .set_included_ranges(&ranges) .expect("overlapping ranges"); parser - .set_language(grammar.ts_language) + .set_language(&grammar.ts_language) .expect("incompatible grammar"); parser .parse_with( diff --git a/crates/semantic_index/src/parsing.rs b/crates/semantic_index/src/parsing.rs index cb15ca453b..427ac158c1 100644 --- a/crates/semantic_index/src/parsing.rs +++ b/crates/semantic_index/src/parsing.rs @@ -169,7 +169,7 @@ impl CodeContextRetriever { .embedding_config .as_ref() .ok_or_else(|| anyhow!("no embedding queries"))?; - self.parser.set_language(grammar.ts_language).unwrap(); + self.parser.set_language(&grammar.ts_language).unwrap(); let tree = self .parser diff --git a/crates/semantic_index2/src/parsing.rs b/crates/semantic_index2/src/parsing.rs index cb15ca453b..427ac158c1 100644 --- a/crates/semantic_index2/src/parsing.rs +++ b/crates/semantic_index2/src/parsing.rs @@ -169,7 +169,7 @@ impl CodeContextRetriever { .embedding_config .as_ref() .ok_or_else(|| anyhow!("no embedding queries"))?; - self.parser.set_language(grammar.ts_language).unwrap(); + self.parser.set_language(&grammar.ts_language).unwrap(); let tree = self .parser diff --git a/crates/settings/src/settings_store.rs b/crates/settings/src/settings_store.rs index da84074d2a..2fc7ba58ed 100644 --- a/crates/settings/src/settings_store.rs +++ b/crates/settings/src/settings_store.rs @@ -679,14 +679,14 @@ fn replace_value_in_json_text( lazy_static! { static ref PAIR_QUERY: tree_sitter::Query = tree_sitter::Query::new( - tree_sitter_json::language(), + &tree_sitter_json::language(), "(pair key: (string) @key value: (_) @value)", ) .unwrap(); } let mut parser = tree_sitter::Parser::new(); - parser.set_language(tree_sitter_json::language()).unwrap(); + parser.set_language(&tree_sitter_json::language()).unwrap(); let syntax_tree = parser.parse(text, None).unwrap(); let mut cursor = tree_sitter::QueryCursor::new(); diff --git a/crates/settings2/src/settings_store.rs b/crates/settings2/src/settings_store.rs index 5bf10a518d..aa351bc8c2 100644 --- a/crates/settings2/src/settings_store.rs +++ b/crates/settings2/src/settings_store.rs @@ -715,14 +715,14 @@ fn replace_value_in_json_text( lazy_static! { static ref PAIR_QUERY: tree_sitter::Query = tree_sitter::Query::new( - tree_sitter_json::language(), + &tree_sitter_json::language(), "(pair key: (string) @key value: (_) @value)", ) .unwrap(); } let mut parser = tree_sitter::Parser::new(); - parser.set_language(tree_sitter_json::language()).unwrap(); + parser.set_language(&tree_sitter_json::language()).unwrap(); let syntax_tree = parser.parse(text, None).unwrap(); let mut cursor = tree_sitter::QueryCursor::new(); diff --git a/crates/util/src/paths.rs b/crates/util/src/paths.rs index 19b244383f..7abd4a08f4 100644 --- a/crates/util/src/paths.rs +++ b/crates/util/src/paths.rs @@ -10,6 +10,7 @@ lazy_static::lazy_static! { pub static ref EMBEDDINGS_DIR: PathBuf = HOME.join(".config/zed/embeddings"); pub static ref LOGS_DIR: PathBuf = HOME.join("Library/Logs/Zed"); pub static ref SUPPORT_DIR: PathBuf = HOME.join("Library/Application Support/Zed"); + pub static ref PLUGINS_DIR: PathBuf = HOME.join("Library/Application Support/Zed/plugins"); pub static ref LANGUAGES_DIR: PathBuf = HOME.join("Library/Application Support/Zed/languages"); pub static ref COPILOT_DIR: PathBuf = HOME.join("Library/Application Support/Zed/copilot"); pub static ref DEFAULT_PRETTIER_DIR: PathBuf = HOME.join("Library/Application Support/Zed/prettier"); diff --git a/crates/zed2/src/languages.rs b/crates/zed2/src/languages.rs index 129dad8f48..3fdcad46fe 100644 --- a/crates/zed2/src/languages.rs +++ b/crates/zed2/src/languages.rs @@ -5,7 +5,7 @@ use node_runtime::NodeRuntime; use rust_embed::RustEmbed; use settings::Settings; use std::{borrow::Cow, str, sync::Arc}; -use util::asset_str; +use util::{asset_str, paths::PLUGINS_DIR}; use self::elixir::ElixirSettings; @@ -228,6 +228,21 @@ pub fn init( tree_sitter_uiua::language(), vec![Arc::new(uiua::UiuaLanguageServer {})], ); + + if let Ok(children) = std::fs::read_dir(&*PLUGINS_DIR) { + for child in children { + if let Ok(child) = child { + let path = child.path(); + let config_path = path.join("config.toml"); + if let Ok(config) = std::fs::read(&config_path) { + let config: LanguageConfig = toml::from_slice(&config).unwrap(); + if let Some(grammar_name) = config.grammar_name.clone() { + languages.register_wasm(path.into(), grammar_name, config); + } + } + } + } + } } #[cfg(any(test, feature = "test-support"))]