Expanded Semantic language support (#2818)

Expand Language Support within Semantic Search

Release Notes (Preview-only)

- Added semantic search support for lua, ruby, php, svelte, erb, heex,
html and markdown.
- Full details are included here:
https://linear.app/zed-industries/issue/Z-2611/allow-semantic-search-for-all-supported-languages
This commit is contained in:
Kyle Caverly 2023-08-01 11:51:51 -04:00 committed by GitHub
commit ce258dfeb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 751 additions and 141 deletions

232
Cargo.lock generated
View File

@ -141,7 +141,7 @@ source = "git+https://github.com/alacritty/alacritty?rev=7b9f32300ee0a249c087230
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -187,9 +187,9 @@ checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
[[package]]
name = "alsa"
version = "0.7.0"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8512c9117059663fb5606788fbca3619e2a91dac0e3fe516242eab1fa6be5e44"
checksum = "e2562ad8dcf0f789f65c6fdaad8a8a9708ed6b488e649da28c01656ad66b8b47"
dependencies = [
"alsa-sys",
"bitflags 1.3.2",
@ -215,9 +215,9 @@ checksum = "ec8ad6edb4840b78c5c3d88de606b22252d552b55f3a4699fbb10fc070ec3049"
[[package]]
name = "android-activity"
version = "0.4.2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40bc1575e653f158cbdc6ebcd917b9564e66321c5325c232c3591269c257be69"
checksum = "64529721f27c2314ced0890ce45e469574a73e5e6fdd6e9da1860eb29285f5e0"
dependencies = [
"android-properties",
"bitflags 1.3.2",
@ -507,7 +507,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -555,7 +555,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -598,7 +598,7 @@ checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -856,7 +856,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
"syn 2.0.27",
"syn 2.0.28",
"which",
]
@ -1040,7 +1040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05"
dependencies = [
"memchr",
"regex-automata 0.3.3",
"regex-automata 0.3.4",
"serde",
]
@ -1358,7 +1358,7 @@ dependencies = [
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -1425,7 +1425,7 @@ dependencies = [
"sum_tree",
"tempfile",
"thiserror",
"time 0.3.23",
"time 0.3.24",
"tiny_http",
"url",
"util",
@ -1527,7 +1527,7 @@ dependencies = [
"sha-1 0.9.8",
"sqlx",
"theme",
"time 0.3.23",
"time 0.3.24",
"tokio",
"tokio-tungstenite",
"toml 0.5.11",
@ -2042,9 +2042,9 @@ dependencies = [
[[package]]
name = "curl-sys"
version = "0.4.64+curl-8.2.0"
version = "0.4.65+curl-8.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f96069f0b1cb1241c838740659a771ef143363f52772a9ce1bd9c04c75eee0dc"
checksum = "961ba061c9ef2fe34bbd12b807152d96f0badd2bebe7b90ce6c8c8b7572a0986"
dependencies = [
"cc",
"libc",
@ -2124,6 +2124,15 @@ dependencies = [
"byteorder",
]
[[package]]
name = "deranged"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01"
dependencies = [
"serde",
]
[[package]]
name = "dhat"
version = "0.3.2"
@ -2341,7 +2350,7 @@ dependencies = [
"tree-sitter",
"tree-sitter-html",
"tree-sitter-rust",
"tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
"tree-sitter-typescript",
"unindent",
"util",
"workspace",
@ -2425,9 +2434,9 @@ dependencies = [
[[package]]
name = "errno"
version = "0.3.1"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
dependencies = [
"errno-dragonfly",
"libc",
@ -2731,7 +2740,7 @@ dependencies = [
"smol",
"sum_tree",
"tempfile",
"time 0.3.23",
"time 0.3.24",
"util",
]
@ -2881,7 +2890,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -3033,9 +3042,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "globset"
version = "0.4.11"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1391ab1f92ffcc08911957149833e682aa3fe252b9f45f966d2ef972274c97df"
checksum = "aca8bbd8e0707c1887a8bbb7e6b40e228f251ff5d62c8220a4a7a53c73aff006"
dependencies = [
"aho-corasick 1.0.2",
"bstr",
@ -3121,7 +3130,7 @@ dependencies = [
"smol",
"sqlez",
"sum_tree",
"time 0.3.23",
"time 0.3.24",
"tiny-skia",
"usvg",
"util",
@ -3851,7 +3860,7 @@ dependencies = [
"text",
"theme",
"tree-sitter",
"tree-sitter-elixir 0.1.0 (git+https://github.com/elixir-lang/tree-sitter-elixir?rev=a2861e88a730287a60c11ea9299c033c7d076e30)",
"tree-sitter-elixir",
"tree-sitter-embedded-template",
"tree-sitter-heex",
"tree-sitter-html",
@ -3860,7 +3869,7 @@ dependencies = [
"tree-sitter-python",
"tree-sitter-ruby",
"tree-sitter-rust",
"tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
"tree-sitter-typescript",
"unicase",
"unindent",
"util",
@ -4034,9 +4043,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
[[package]]
name = "linux-raw-sys"
version = "0.4.3"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
[[package]]
name = "lipsum"
@ -4744,7 +4753,7 @@ dependencies = [
"proc-macro-crate 1.3.1",
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -4895,7 +4904,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -5134,9 +5143,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
[[package]]
name = "pest"
version = "2.7.1"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d2d1d55045829d65aad9d389139882ad623b33b904e7c9f1b10c5b8927298e5"
checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a"
dependencies = [
"thiserror",
"ucd-trie",
@ -5192,7 +5201,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -5230,7 +5239,7 @@ dependencies = [
"line-wrap",
"quick-xml",
"serde",
"time 0.3.23",
"time 0.3.24",
]
[[package]]
@ -5345,7 +5354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
dependencies = [
"proc-macro2",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -5905,7 +5914,7 @@ checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
dependencies = [
"aho-corasick 1.0.2",
"memchr",
"regex-automata 0.3.3",
"regex-automata 0.3.4",
"regex-syntax 0.7.4",
]
@ -5920,9 +5929,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294"
dependencies = [
"aho-corasick 1.0.2",
"memchr",
@ -6217,7 +6226,7 @@ dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
"syn 2.0.27",
"syn 2.0.28",
"walkdir",
]
@ -6234,13 +6243,12 @@ dependencies = [
[[package]]
name = "rust_decimal"
version = "1.30.0"
version = "1.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0446843641c69436765a35a5a77088e28c2e6a12da93e84aa3ab1cd4aa5a042"
checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea"
dependencies = [
"arrayvec 0.7.4",
"borsh",
"bytecheck",
"byteorder",
"bytes 1.4.0",
"num-traits",
@ -6294,7 +6302,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06"
dependencies = [
"bitflags 1.3.2",
"errno 0.3.1",
"errno 0.3.2",
"io-lifetimes 1.0.11",
"libc",
"linux-raw-sys 0.3.8",
@ -6308,9 +6316,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5"
dependencies = [
"bitflags 2.3.3",
"errno 0.3.1",
"errno 0.3.2",
"libc",
"linux-raw-sys 0.4.3",
"linux-raw-sys 0.4.5",
"windows-sys",
]
@ -6509,7 +6517,7 @@ dependencies = [
"serde_json",
"sqlx",
"thiserror",
"time 0.3.23",
"time 0.3.24",
"tracing",
"url",
"uuid 1.4.1",
@ -6537,7 +6545,7 @@ dependencies = [
"rust_decimal",
"sea-query-derive",
"serde_json",
"time 0.3.23",
"time 0.3.24",
"uuid 1.4.1",
]
@ -6552,7 +6560,7 @@ dependencies = [
"sea-query",
"serde_json",
"sqlx",
"time 0.3.23",
"time 0.3.24",
"uuid 1.4.1",
]
@ -6685,12 +6693,15 @@ dependencies = [
"theme",
"tiktoken-rs 0.5.0",
"tree-sitter",
"tree-sitter-cpp 0.20.2",
"tree-sitter-elixir 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tree-sitter-json 0.19.0",
"tree-sitter-cpp",
"tree-sitter-elixir",
"tree-sitter-json 0.20.0",
"tree-sitter-lua",
"tree-sitter-php",
"tree-sitter-ruby",
"tree-sitter-rust",
"tree-sitter-toml 0.20.0",
"tree-sitter-typescript 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tree-sitter-toml",
"tree-sitter-typescript",
"unindent",
"util",
"workspace",
@ -6722,22 +6733,22 @@ checksum = "5a9f47faea3cad316faa914d013d24f471cd90bfca1a0c70f05a3f42c6441e99"
[[package]]
name = "serde"
version = "1.0.175"
version = "1.0.180"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b"
checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.175"
version = "1.0.180"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4"
checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -6762,9 +6773,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.103"
version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
dependencies = [
"indexmap 2.0.0",
"itoa 1.0.9",
@ -6786,13 +6797,13 @@ dependencies = [
[[package]]
name = "serde_repr"
version = "0.1.15"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e168eaaf71e8f9bd6037feb05190485708e019f4fd87d161b3c0a0d37daf85e5"
checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -7238,7 +7249,7 @@ dependencies = [
"sqlx-rt",
"stringprep",
"thiserror",
"time 0.3.23",
"time 0.3.24",
"tokio-stream",
"url",
"uuid 1.4.1",
@ -7487,9 +7498,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.27"
version = "2.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
dependencies = [
"proc-macro2",
"quote",
@ -7557,9 +7568,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "target-lexicon"
version = "0.12.10"
version = "0.12.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2faeef5759ab89935255b1a4cd98e0baf99d1085e37d36599c625dac49ae8e"
checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a"
[[package]]
name = "tempdir"
@ -7742,7 +7753,7 @@ checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -7815,10 +7826,11 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.23"
version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446"
checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b"
dependencies = [
"deranged",
"itoa 1.0.9",
"serde",
"time-core",
@ -7833,9 +7845,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
[[package]]
name = "time-macros"
version = "0.2.10"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4"
checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd"
dependencies = [
"time-core",
]
@ -7931,7 +7943,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -8153,7 +8165,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]
@ -8255,16 +8267,6 @@ dependencies = [
"tree-sitter",
]
[[package]]
name = "tree-sitter-cpp"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c88fd925d0333e63ac64e521f5bd79c53019e569ffbbccfeef346a326f459e9"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]
name = "tree-sitter-css"
version = "0.19.0"
@ -8274,16 +8276,6 @@ dependencies = [
"tree-sitter",
]
[[package]]
name = "tree-sitter-elixir"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a9916f3e1c80b3c8aab8582604e97e8720cb9b893489b347cf999f80f9d469e"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]
name = "tree-sitter-elixir"
version = "0.1.0"
@ -8471,26 +8463,6 @@ dependencies = [
"tree-sitter",
]
[[package]]
name = "tree-sitter-toml"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca517f578a98b23d20780247cc2688407fa81effad5b627a5a364ec3339b53e8"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]
name = "tree-sitter-typescript"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "079c695c32d39ad089101c66393aeaca30e967fba3486a91f573d2f0e12d290a"
dependencies = [
"cc",
"tree-sitter",
]
[[package]]
name = "tree-sitter-typescript"
version = "0.20.2"
@ -8993,7 +8965,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
"wasm-bindgen-shared",
]
@ -9027,7 +8999,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -9040,9 +9012,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "wasm-encoder"
version = "0.31.0"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06a3d1b4a575ffb873679402b2aedb3117555eb65c27b1b86c8a91e574bc2a2a"
checksum = "41763f20eafed1399fff1afb466496d3a959f58241436cfdc17e3f5ca954de16"
dependencies = [
"leb128",
]
@ -9264,9 +9236,9 @@ dependencies = [
[[package]]
name = "wast"
version = "62.0.0"
version = "62.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7f7ee878019d69436895f019b65f62c33da63595d8e857cbdc87c13ecb29a32"
checksum = "b8ae06f09dbe377b889fbd620ff8fa21e1d49d1d9d364983c0cdbf9870cb9f1f"
dependencies = [
"leb128",
"memchr",
@ -9276,11 +9248,11 @@ dependencies = [
[[package]]
name = "wat"
version = "1.0.68"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "295572bf24aa5b685a971a83ad3e8b6e684aaad8a9be24bc7bf59bed84cc1c08"
checksum = "842e15861d203fb4a96d314b0751cdeaf0f6f8b35e8d81d2953af2af5e44e637"
dependencies = [
"wast 62.0.0",
"wast 62.0.1",
]
[[package]]
@ -9657,9 +9629,9 @@ dependencies = [
[[package]]
name = "winnow"
version = "0.5.1"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25b5872fa2e10bd067ae946f927e726d7d603eaeb6e02fa6a350e0722d2b8c11"
checksum = "8bd122eb777186e60c3fdf765a58ac76e41c582f1f535fbf3314434c6b58f3f7"
dependencies = [
"memchr",
]
@ -9930,9 +9902,9 @@ dependencies = [
"tree-sitter",
"tree-sitter-bash",
"tree-sitter-c",
"tree-sitter-cpp 0.20.0",
"tree-sitter-cpp",
"tree-sitter-css",
"tree-sitter-elixir 0.1.0 (git+https://github.com/elixir-lang/tree-sitter-elixir?rev=a2861e88a730287a60c11ea9299c033c7d076e30)",
"tree-sitter-elixir",
"tree-sitter-elm",
"tree-sitter-embedded-template",
"tree-sitter-glsl",
@ -9950,8 +9922,8 @@ dependencies = [
"tree-sitter-rust",
"tree-sitter-scheme",
"tree-sitter-svelte",
"tree-sitter-toml 0.5.1",
"tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
"tree-sitter-toml",
"tree-sitter-typescript",
"tree-sitter-yaml",
"unindent",
"url",
@ -9988,7 +9960,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.27",
"syn 2.0.28",
]
[[package]]

View File

@ -54,9 +54,12 @@ tempdir.workspace = true
ctor.workspace = true
env_logger.workspace = true
tree-sitter-typescript = "*"
tree-sitter-json = "*"
tree-sitter-rust = "*"
tree-sitter-toml = "*"
tree-sitter-cpp = "*"
tree-sitter-elixir = "*"
tree-sitter-typescript.workspace = true
tree-sitter-json.workspace = true
tree-sitter-rust.workspace = true
tree-sitter-toml.workspace = true
tree-sitter-cpp.workspace = true
tree-sitter-elixir.workspace = true
tree-sitter-lua.workspace = true
tree-sitter-ruby.workspace = true
tree-sitter-php.workspace = true

View File

@ -21,7 +21,9 @@ const CODE_CONTEXT_TEMPLATE: &str =
"The below code snippet is from file '<path>'\n\n```<language>\n<item>\n```";
const ENTIRE_FILE_TEMPLATE: &str =
"The below snippet is from file '<path>'\n\n```<language>\n<item>\n```";
pub const PARSEABLE_ENTIRE_FILE_TYPES: &[&str] = &["TOML", "YAML", "CSS"];
const MARKDOWN_CONTEXT_TEMPLATE: &str = "The below file contents is from file '<path>'\n\n<item>";
pub const PARSEABLE_ENTIRE_FILE_TYPES: &[&str] =
&["TOML", "YAML", "CSS", "HEEX", "ERB", "SVELTE", "HTML"];
pub struct CodeContextRetriever {
pub parser: Parser,
@ -59,7 +61,7 @@ impl CodeContextRetriever {
let document_span = ENTIRE_FILE_TEMPLATE
.replace("<path>", relative_path.to_string_lossy().as_ref())
.replace("<language>", language_name.as_ref())
.replace("item", &content);
.replace("<item>", &content);
Ok(vec![Document {
range: 0..content.len(),
@ -69,6 +71,19 @@ impl CodeContextRetriever {
}])
}
fn parse_markdown_file(&self, relative_path: &Path, content: &str) -> Result<Vec<Document>> {
let document_span = MARKDOWN_CONTEXT_TEMPLATE
.replace("<path>", relative_path.to_string_lossy().as_ref())
.replace("<item>", &content);
Ok(vec![Document {
range: 0..content.len(),
content: document_span,
embedding: Vec::new(),
name: "Markdown".to_string(),
}])
}
fn get_matches_in_file(
&mut self,
content: &str,
@ -135,6 +150,8 @@ impl CodeContextRetriever {
if PARSEABLE_ENTIRE_FILE_TYPES.contains(&language_name.as_ref()) {
return self.parse_entire_file(relative_path, language_name, &content);
} else if &language_name.to_string() == &"Markdown".to_string() {
return self.parse_markdown_file(relative_path, &content);
}
let mut documents = self.parse_file(content, language)?;
@ -200,7 +217,12 @@ impl CodeContextRetriever {
let mut document_content = String::new();
for context_range in &context_match.context_ranges {
document_content.push_str(&content[context_range.clone()]);
add_content_from_range(
&mut document_content,
content,
context_range.clone(),
context_match.start_col,
);
document_content.push_str("\n");
}

View File

@ -612,6 +612,7 @@ impl SemanticIndex {
.await
{
if !PARSEABLE_ENTIRE_FILE_TYPES.contains(&language.name().as_ref())
&& &language.name().as_ref() != &"Markdown"
&& language
.grammar()
.and_then(|grammar| grammar.embedding_config.as_ref())

View File

@ -485,6 +485,79 @@ async fn test_code_context_retrieval_javascript() {
)
}
#[gpui::test]
async fn test_code_context_retrieval_lua() {
let language = lua_lang();
let mut retriever = CodeContextRetriever::new();
let text = r#"
-- Creates a new class
-- @param baseclass The Baseclass of this class, or nil.
-- @return A new class reference.
function classes.class(baseclass)
-- Create the class definition and metatable.
local classdef = {}
-- Find the super class, either Object or user-defined.
baseclass = baseclass or classes.Object
-- If this class definition does not know of a function, it will 'look up' to the Baseclass via the __index of the metatable.
setmetatable(classdef, { __index = baseclass })
-- All class instances have a reference to the class object.
classdef.class = classdef
--- Recursivly allocates the inheritance tree of the instance.
-- @param mastertable The 'root' of the inheritance tree.
-- @return Returns the instance with the allocated inheritance tree.
function classdef.alloc(mastertable)
-- All class instances have a reference to a superclass object.
local instance = { super = baseclass.alloc(mastertable) }
-- Any functions this instance does not know of will 'look up' to the superclass definition.
setmetatable(instance, { __index = classdef, __newindex = mastertable })
return instance
end
end
"#.unindent();
let documents = retriever.parse_file(&text, language.clone()).unwrap();
assert_documents_eq(
&documents,
&[
(r#"
-- Creates a new class
-- @param baseclass The Baseclass of this class, or nil.
-- @return A new class reference.
function classes.class(baseclass)
-- Create the class definition and metatable.
local classdef = {}
-- Find the super class, either Object or user-defined.
baseclass = baseclass or classes.Object
-- If this class definition does not know of a function, it will 'look up' to the Baseclass via the __index of the metatable.
setmetatable(classdef, { __index = baseclass })
-- All class instances have a reference to the class object.
classdef.class = classdef
--- Recursivly allocates the inheritance tree of the instance.
-- @param mastertable The 'root' of the inheritance tree.
-- @return Returns the instance with the allocated inheritance tree.
function classdef.alloc(mastertable)
--[ ... ]--
--[ ... ]--
end
end"#.unindent(),
114),
(r#"
--- Recursivly allocates the inheritance tree of the instance.
-- @param mastertable The 'root' of the inheritance tree.
-- @return Returns the instance with the allocated inheritance tree.
function classdef.alloc(mastertable)
-- All class instances have a reference to a superclass object.
local instance = { super = baseclass.alloc(mastertable) }
-- Any functions this instance does not know of will 'look up' to the superclass definition.
setmetatable(instance, { __index = classdef, __newindex = mastertable })
return instance
end"#.unindent(), 809),
]
);
}
#[gpui::test]
async fn test_code_context_retrieval_elixir() {
let language = elixir_lang();
@ -753,6 +826,346 @@ async fn test_code_context_retrieval_cpp() {
);
}
#[gpui::test]
async fn test_code_context_retrieval_ruby() {
let language = ruby_lang();
let mut retriever = CodeContextRetriever::new();
let text = r#"
# This concern is inspired by "sudo mode" on GitHub. It
# is a way to re-authenticate a user before allowing them
# to see or perform an action.
#
# Add `before_action :require_challenge!` to actions you
# want to protect.
#
# The user will be shown a page to enter the challenge (which
# is either the password, or just the username when no
# password exists). Upon passing, there is a grace period
# during which no challenge will be asked from the user.
#
# Accessing challenge-protected resources during the grace
# period will refresh the grace period.
module ChallengableConcern
extend ActiveSupport::Concern
CHALLENGE_TIMEOUT = 1.hour.freeze
def require_challenge!
return if skip_challenge?
if challenge_passed_recently?
session[:challenge_passed_at] = Time.now.utc
return
end
@challenge = Form::Challenge.new(return_to: request.url)
if params.key?(:form_challenge)
if challenge_passed?
session[:challenge_passed_at] = Time.now.utc
else
flash.now[:alert] = I18n.t('challenge.invalid_password')
render_challenge
end
else
render_challenge
end
end
def challenge_passed?
current_user.valid_password?(challenge_params[:current_password])
end
end
class Animal
include Comparable
attr_reader :legs
def initialize(name, legs)
@name, @legs = name, legs
end
def <=>(other)
legs <=> other.legs
end
end
# Singleton method for car object
def car.wheels
puts "There are four wheels"
end"#
.unindent();
let documents = retriever.parse_file(&text, language.clone()).unwrap();
assert_documents_eq(
&documents,
&[
(
r#"
# This concern is inspired by "sudo mode" on GitHub. It
# is a way to re-authenticate a user before allowing them
# to see or perform an action.
#
# Add `before_action :require_challenge!` to actions you
# want to protect.
#
# The user will be shown a page to enter the challenge (which
# is either the password, or just the username when no
# password exists). Upon passing, there is a grace period
# during which no challenge will be asked from the user.
#
# Accessing challenge-protected resources during the grace
# period will refresh the grace period.
module ChallengableConcern
extend ActiveSupport::Concern
CHALLENGE_TIMEOUT = 1.hour.freeze
def require_challenge!
# ...
end
def challenge_passed?
# ...
end
end"#
.unindent(),
558,
),
(
r#"
def require_challenge!
return if skip_challenge?
if challenge_passed_recently?
session[:challenge_passed_at] = Time.now.utc
return
end
@challenge = Form::Challenge.new(return_to: request.url)
if params.key?(:form_challenge)
if challenge_passed?
session[:challenge_passed_at] = Time.now.utc
else
flash.now[:alert] = I18n.t('challenge.invalid_password')
render_challenge
end
else
render_challenge
end
end"#
.unindent(),
663,
),
(
r#"
def challenge_passed?
current_user.valid_password?(challenge_params[:current_password])
end"#
.unindent(),
1254,
),
(
r#"
class Animal
include Comparable
attr_reader :legs
def initialize(name, legs)
# ...
end
def <=>(other)
# ...
end
end"#
.unindent(),
1363,
),
(
r#"
def initialize(name, legs)
@name, @legs = name, legs
end"#
.unindent(),
1427,
),
(
r#"
def <=>(other)
legs <=> other.legs
end"#
.unindent(),
1501,
),
(
r#"
# Singleton method for car object
def car.wheels
puts "There are four wheels"
end"#
.unindent(),
1591,
),
],
);
}
#[gpui::test]
async fn test_code_context_retrieval_php() {
let language = php_lang();
let mut retriever = CodeContextRetriever::new();
let text = r#"
<?php
namespace LevelUp\Experience\Concerns;
/*
This is a multiple-lines comment block
that spans over multiple
lines
*/
function functionName() {
echo "Hello world!";
}
trait HasAchievements
{
/**
* @throws \Exception
*/
public function grantAchievement(Achievement $achievement, $progress = null): void
{
if ($progress > 100) {
throw new Exception(message: 'Progress cannot be greater than 100');
}
if ($this->achievements()->find($achievement->id)) {
throw new Exception(message: 'User already has this Achievement');
}
$this->achievements()->attach($achievement, [
'progress' => $progress ?? null,
]);
$this->when(value: ($progress === null) || ($progress === 100), callback: fn (): ?array => event(new AchievementAwarded(achievement: $achievement, user: $this)));
}
public function achievements(): BelongsToMany
{
return $this->belongsToMany(related: Achievement::class)
->withPivot(columns: 'progress')
->where('is_secret', false)
->using(AchievementUser::class);
}
}
interface Multiplier
{
public function qualifies(array $data): bool;
public function setMultiplier(): int;
}
enum AuditType: string
{
case Add = 'add';
case Remove = 'remove';
case Reset = 'reset';
case LevelUp = 'level_up';
}
?>"#
.unindent();
let documents = retriever.parse_file(&text, language.clone()).unwrap();
assert_documents_eq(
&documents,
&[
(
r#"
/*
This is a multiple-lines comment block
that spans over multiple
lines
*/
function functionName() {
echo "Hello world!";
}"#
.unindent(),
123,
),
(
r#"
trait HasAchievements
{
/**
* @throws \Exception
*/
public function grantAchievement(Achievement $achievement, $progress = null): void
{/* ... */}
public function achievements(): BelongsToMany
{/* ... */}
}"#
.unindent(),
177,
),
(r#"
/**
* @throws \Exception
*/
public function grantAchievement(Achievement $achievement, $progress = null): void
{
if ($progress > 100) {
throw new Exception(message: 'Progress cannot be greater than 100');
}
if ($this->achievements()->find($achievement->id)) {
throw new Exception(message: 'User already has this Achievement');
}
$this->achievements()->attach($achievement, [
'progress' => $progress ?? null,
]);
$this->when(value: ($progress === null) || ($progress === 100), callback: fn (): ?array => event(new AchievementAwarded(achievement: $achievement, user: $this)));
}"#.unindent(), 245),
(r#"
public function achievements(): BelongsToMany
{
return $this->belongsToMany(related: Achievement::class)
->withPivot(columns: 'progress')
->where('is_secret', false)
->using(AchievementUser::class);
}"#.unindent(), 902),
(r#"
interface Multiplier
{
public function qualifies(array $data): bool;
public function setMultiplier(): int;
}"#.unindent(),
1146),
(r#"
enum AuditType: string
{
case Add = 'add';
case Remove = 'remove';
case Reset = 'reset';
case LevelUp = 'level_up';
}"#.unindent(), 1265)
],
);
}
#[gpui::test]
fn test_dot_product(mut rng: StdRng) {
assert_eq!(dot(&[1., 0., 0., 0., 0.], &[0., 1., 0., 0., 0.]), 0.);
@ -1083,6 +1496,131 @@ fn cpp_lang() -> Arc<Language> {
)
}
fn lua_lang() -> Arc<Language> {
Arc::new(
Language::new(
LanguageConfig {
name: "Lua".into(),
path_suffixes: vec!["lua".into()],
collapsed_placeholder: "--[ ... ]--".to_string(),
..Default::default()
},
Some(tree_sitter_lua::language()),
)
.with_embedding_query(
r#"
(
(comment)* @context
.
(function_declaration
"function" @name
name: (_) @name
(comment)* @collapse
body: (block) @collapse
) @item
)
"#,
)
.unwrap(),
)
}
fn php_lang() -> Arc<Language> {
Arc::new(
Language::new(
LanguageConfig {
name: "PHP".into(),
path_suffixes: vec!["php".into()],
collapsed_placeholder: "/* ... */".into(),
..Default::default()
},
Some(tree_sitter_php::language()),
)
.with_embedding_query(
r#"
(
(comment)* @context
.
[
(function_definition
"function" @name
name: (_) @name
body: (_
"{" @keep
"}" @keep) @collapse
)
(trait_declaration
"trait" @name
name: (_) @name)
(method_declaration
"function" @name
name: (_) @name
body: (_
"{" @keep
"}" @keep) @collapse
)
(interface_declaration
"interface" @name
name: (_) @name
)
(enum_declaration
"enum" @name
name: (_) @name
)
] @item
)
"#,
)
.unwrap(),
)
}
fn ruby_lang() -> Arc<Language> {
Arc::new(
Language::new(
LanguageConfig {
name: "Ruby".into(),
path_suffixes: vec!["rb".into()],
collapsed_placeholder: "# ...".to_string(),
..Default::default()
},
Some(tree_sitter_ruby::language()),
)
.with_embedding_query(
r#"
(
(comment)* @context
.
[
(module
"module" @name
name: (_) @name)
(method
"def" @name
name: (_) @name
body: (body_statement) @collapse)
(class
"class" @name
name: (_) @name)
(singleton_method
"def" @name
object: (_) @name
"." @name
name: (_) @name
body: (body_statement) @collapse)
] @item
)
"#,
)
.unwrap(),
)
}
fn elixir_lang() -> Arc<Language> {
Arc::new(
Language::new(

View File

@ -7,3 +7,4 @@ brackets = [
{ start = "[", end = "]", close = true, newline = true },
{ start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] },
]
collapsed_placeholder = "--[ ... ]--"

View File

@ -0,0 +1,10 @@
(
(comment)* @context
.
(function_declaration
"function" @name
name: (_) @name
(comment)* @collapse
body: (block) @collapse
) @item
)

View File

@ -9,3 +9,4 @@ brackets = [
{ start = "(", end = ")", close = true, newline = true },
{ start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] },
]
collapsed_placeholder = "/* ... */"

View File

@ -0,0 +1,36 @@
(
(comment)* @context
.
[
(function_definition
"function" @name
name: (_) @name
body: (_
"{" @keep
"}" @keep) @collapse
)
(trait_declaration
"trait" @name
name: (_) @name)
(method_declaration
"function" @name
name: (_) @name
body: (_
"{" @keep
"}" @keep) @collapse
)
(interface_declaration
"interface" @name
name: (_) @name
)
(enum_declaration
"enum" @name
name: (_) @name
)
] @item
)

View File

@ -8,8 +8,6 @@
name: (_) @name
) @item
(method_declaration
"function" @context
name: (_) @name
@ -24,3 +22,8 @@
"enum" @context
name: (_) @name
) @item
(trait_declaration
"trait" @context
name: (_) @name
) @item

View File

@ -10,3 +10,4 @@ brackets = [
{ start = "\"", end = "\"", close = true, newline = false, not_in = ["comment", "string"] },
{ start = "'", end = "'", close = true, newline = false, not_in = ["comment", "string"] },
]
collapsed_placeholder = "# ..."

View File

@ -0,0 +1,22 @@
(
(comment)* @context
.
[
(module
"module" @name
name: (_) @name)
(method
"def" @name
name: (_) @name
body: (body_statement) @collapse)
(class
"class" @name
name: (_) @name)
(singleton_method
"def" @name
object: (_) @name
"." @name
name: (_) @name
body: (body_statement) @collapse)
] @item
)