on master help sign in
migrate helix-syntax crate into helix-core and helix-term

helix-syntax mostly existed for the sake of the build task which checks and compiles the submodules. Since we won't be relying on that process anymore, it doesn't end up making much sense to have a very thin crate just for some functions that we could port to helix-core. The remaining build-related code is moved to helix-term which will be able to provide grammar builds through the --build-grammars CLI flag.

commiteeb3f8e9639e68ea2d54a6be26d45543d01057f6 authorMichael Davis <mcarsondavis@gmail.com> dateSun 13 Feb 2022, 10:42 (5 years ago) parentc1f90a1
12 files changed+72−141
Cargo.lock+4−12 Cargo.toml−5 docs/architecture.md+6−7 helix-core/Cargo.toml+3−2 helix-core/src/syntax.rs+33−3 helix-syntax/Cargo.toml−21 helix-syntax/README.md−13 helix-syntax/src/lib.rs−31 helix-term/Cargo.toml+4 helix-term/build.rs+5 helix-syntax/build.rs → helix-term/src/grammars.rs+16−47 helix-term/src/lib.rs+1
Cargo.lock+4−12
@@ -358,11 +358,12 @@ dependencies = [
358358name = "helix-core"
359359version = "0.6.0"
360360dependencies = [
361+ "anyhow",
361362 "arc-swap",
362363 "chrono",
363364 "encoding_rs",
364365 "etcetera",
365− "helix-syntax",
366+ "libloading",
366367 "log",
367368 "once_cell",
368369 "quickcheck",
@@ -415,22 +416,12 @@ dependencies = [
415416 "which",
416417]
417418
418−[[package]]
419−name = "helix-syntax"
420−version = "0.6.0"
421−dependencies = [
422− "anyhow",
423− "cc",
424− "libloading",
425− "threadpool",
426− "tree-sitter",
427−]
428−
429419[[package]]
430420name = "helix-term"
431421version = "0.6.0"
432422dependencies = [
433423 "anyhow",
424+ "cc",
434425 "chrono",
435426 "content_inspector",
436427 "crossterm",
@@ -454,6 +445,7 @@ dependencies = [
454445 "serde_json",
455446 "signal-hook",
456447 "signal-hook-tokio",
448+ "threadpool",
457449 "tokio",
458450 "tokio-stream",
459451 "toml",
Cargo.toml−5
@@ -4,7 +4,6 @@ members = [
44 "helix-view",
55 "helix-term",
66 "helix-tui",
7− "helix-syntax",
87 "helix-lsp",
98 "helix-dap",
109 "xtask",
@@ -14,10 +13,6 @@ default-members = [
1413 "helix-term"
1514]
1615
17−# Build helix-syntax in release mode to make the code path faster in development.
18−# [profile.dev.package."helix-syntax"]
19−# opt-level = 3
20−
2116[profile.dev]
2217split-debuginfo = "unpacked"
2318
docs/architecture.md+6−7
@@ -1,11 +1,10 @@
11
2−| Crate | Description |
3−| ----------- | ----------- |
4−| helix-core | Core editing primitives, functional. |
5−| helix-syntax | Tree-sitter grammars |
6−| helix-lsp | Language server client |
7−| helix-view | UI abstractions for use in backends, imperative shell. |
8−| helix-term | Terminal UI |
2+| Crate | Description |
3+| ----------- | ----------- |
4+| helix-core | Core editing primitives, functional. |
5+| helix-lsp | Language server client |
6+| helix-view | UI abstractions for use in backends, imperative shell. |
7+| helix-term | Terminal UI |
98| helix-tui | TUI primitives, forked from tui-rs, inspired by Cursive |
109
1110
helix-core/Cargo.toml+3−2
@@ -13,8 +13,6 @@ include = ["src/**/*", "README.md"]
1313[features]
1414
1515[dependencies]
16−helix-syntax = { version = "0.6", path = "../helix-syntax" }
17−
1816ropey = "1.3"
1917smallvec = "1.8"
2018smartstring = "1.0.0"
@@ -40,5 +38,8 @@ encoding_rs = "0.8"
4038
4139chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
4240
41+libloading = "0.7"
42+anyhow = "1"
43+
4344[dev-dependencies]
4445quickcheck = { version = "1", default-features = false }
helix-core/src/syntax.rs+33−3
@@ -7,7 +7,9 @@ use crate::{
77 Rope, RopeSlice, Tendril,
88};
99
10−pub use helix_syntax::get_language;
10+use anyhow::{Context, Result};
11+use libloading::{Library, Symbol};
12+use tree_sitter::Language;
1113
1214use arc_swap::{ArcSwap, Guard};
1315use slotmap::{DefaultKey as LayerId, HopSlotMap};
@@ -25,6 +27,34 @@ use std::{
2527use once_cell::sync::{Lazy, OnceCell};
2628use serde::{Deserialize, Serialize};
2729
30+#[cfg(unix)]
31+pub const DYLIB_EXTENSION: &str = "so";
32+
33+#[cfg(windows)]
34+pub const DYLIB_EXTENSION: &str = "dll";
35+
36+fn replace_dashes_with_underscores(name: &str) -> String {
37+ name.replace('-', "_")
38+}
39+
40+pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
41+ let name = name.to_ascii_lowercase();
42+ let mut library_path = runtime_path.join("grammars").join(&name);
43+ library_path.set_extension(DYLIB_EXTENSION);
44+
45+ let library = unsafe { Library::new(&library_path) }
46+ .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
47+ let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
48+ let language = unsafe {
49+ let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
50+ .get(language_fn_name.as_bytes())
51+ .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
52+ language_fn()
53+ };
54+ std::mem::forget(library);
55+ Ok(language)
56+}
57+
2858fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
2959where
3060 D: serde::Deserializer<'de>,
@@ -426,7 +456,7 @@ impl LanguageConfiguration {
426456 &injections_query,
427457 &locals_query,
428458 )
429− .unwrap(); // TODO: avoid panic
459+ .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --fetch-grammars' and 'hx --build-grammars'. This query could not be parsed: {:?}", self.language_id, query_error));
430460
431461 config.configure(scopes);
432462 Some(Arc::new(config))
@@ -2023,7 +2053,7 @@ mod test {
20232053 );
20242054
20252055 let loader = Loader::new(Configuration { language: vec![] });
2026− let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
2056+ let language = get_language("Rust").unwrap();
20272057
20282058 let query = Query::new(language, query_str).unwrap();
20292059 let textobject = TextObjectQuery { query };
helix-syntax/Cargo.tomldeleted−21
@@ -1,21 +0,0 @@
1−[package]
2−name = "helix-syntax"
3−version = "0.6.0"
4−authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
5−edition = "2021"
6−license = "MPL-2.0"
7−description = "Tree-sitter grammars support"
8−categories = ["editor"]
9−repository = "https://github.com/helix-editor/helix"
10−homepage = "https://helix-editor.com"
11−include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
12−
13−[dependencies]
14−tree-sitter = "0.20"
15−libloading = "0.7"
16−anyhow = "1"
17−
18−[build-dependencies]
19−cc = { version = "1" }
20−threadpool = { version = "1.0" }
21−anyhow = "1"
helix-syntax/README.mddeleted−13
@@ -1,13 +0,0 @@
1−helix-syntax
2−============
3−
4−Syntax highlighting for helix, (shallow) submodules resides here.
5−
6−Differences from nvim-treesitter
7−--------------------------------
8−
9−As the syntax are commonly ported from
10−<https://github.com/nvim-treesitter/nvim-treesitter>.
11−
12−Note that we do not support the custom `#any-of` predicate which is
13−supported by neovim so one needs to change it to `#match` with regex.
helix-syntax/src/lib.rsdeleted−31
@@ -1,31 +0,0 @@
1−use anyhow::{Context, Result};
2−use libloading::{Library, Symbol};
3−use tree_sitter::Language;
4−
5−fn replace_dashes_with_underscores(name: &str) -> String {
6− name.replace('-', "_")
7−}
8−#[cfg(unix)]
9−const DYLIB_EXTENSION: &str = "so";
10−
11−#[cfg(windows)]
12−const DYLIB_EXTENSION: &str = "dll";
13−
14−pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
15− let name = name.to_ascii_lowercase();
16− let mut library_path = runtime_path.join("grammars").join(&name);
17− // TODO: duplicated under build
18− library_path.set_extension(DYLIB_EXTENSION);
19−
20− let library = unsafe { Library::new(&library_path) }
21− .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
22− let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
23− let language = unsafe {
24− let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
25− .get(language_fn_name.as_bytes())
26− .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
27− language_fn()
28− };
29− std::mem::forget(library);
30− Ok(language)
31−}
helix-term/Cargo.toml+4
@@ -66,5 +66,9 @@ grep-searcher = "0.1.8"
6666# Remove once retain_mut lands in stable rust
6767retain_mut = "0.1.7"
6868
69+# compiling grammars
70+cc = { version = "1" }
71+threadpool = { version = "1.0" }
72+
6973[target.'cfg(not(windows))'.dependencies] # https://github.com/vorner/signal-hook/issues/100
7074signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
helix-term/build.rs+5
@@ -14,5 +14,10 @@ fn main() {
1414 None => env!("CARGO_PKG_VERSION").into(),
1515 };
1616
17+ println!(
18+ "cargo:rustc-env=BUILD_TARGET={}",
19+ std::env::var("TARGET").unwrap()
20+ );
21+
1722 println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
1823}
helix-syntax/build.rs → helix-term/src/grammars.rsrenamed+16−47
@@ -6,11 +6,13 @@ use std::{
66 process::Command,
77};
88
9−use std::sync::mpsc::channel;
9+use helix_core::syntax::DYLIB_EXTENSION;
1010
11−fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
11+const BUILD_TARGET: &str = env!("BUILD_TARGET");
12+
13+pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
1214 let mut dirs = Vec::new();
13− let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
15+ let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
1416
1517 for entry in fs::read_dir(path)? {
1618 let entry = entry?;
@@ -32,12 +34,6 @@ fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
3234 Ok(dirs)
3335}
3436
35−#[cfg(unix)]
36−const DYLIB_EXTENSION: &str = "so";
37−
38−#[cfg(windows)]
39−const DYLIB_EXTENSION: &str = "dll";
40−
4137fn build_library(src_path: &Path, language: &str) -> Result<()> {
4238 let header_path = src_path;
4339 // let grammar_path = src_path.join("grammar.json");
@@ -65,7 +61,12 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
6561 return Ok(());
6662 }
6763 let mut config = cc::Build::new();
68− config.cpp(true).opt_level(2).cargo_metadata(false);
64+ config
65+ .cpp(true)
66+ .opt_level(2)
67+ .cargo_metadata(false)
68+ .host(BUILD_TARGET)
69+ .target(BUILD_TARGET);
6970 let compiler = config.get_compiler();
7071 let mut command = Command::new(compiler.path());
7172 command.current_dir(src_path);
@@ -148,9 +149,10 @@ fn mtime(path: &Path) -> Result<SystemTime> {
148149 Ok(fs::metadata(path)?.modified()?)
149150}
150151
151−fn build_dir(dir: &str, language: &str) {
152+pub fn build_dir(dir: &str, language: &str) {
152153 println!("Build language {}", language);
153− if PathBuf::from("languages")
154+ if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
155+ .join("../helix-syntax/languages")
154156 .join(dir)
155157 .read_dir()
156158 .unwrap()
@@ -158,49 +160,16 @@ fn build_dir(dir: &str, language: &str) {
158160 .is_none()
159161 {
160162 eprintln!(
161− "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?",
163+ "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
162164 dir
163165 );
164166 std::process::exit(1);
165167 }
166168
167169 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
168− .join("languages")
170+ .join("../helix-syntax/languages")
169171 .join(dir)
170172 .join("src");
171173
172174 build_library(&path, language).unwrap();
173175}
174−
175−fn main() {
176− let ignore = vec![
177− "tree-sitter-typescript".to_string(),
178− "tree-sitter-ocaml".to_string(),
179− ];
180− let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
181−
182− let mut n_jobs = 0;
183− let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
184− let (tx, rx) = channel();
185−
186− for dir in dirs {
187− let tx = tx.clone();
188− n_jobs += 1;
189−
190− pool.execute(move || {
191− let language = &dir.strip_prefix("tree-sitter-").unwrap();
192− build_dir(&dir, language);
193−
194− // report progress
195− tx.send(1).unwrap();
196− });
197− }
198− pool.join();
199− // drop(tx);
200− assert_eq!(rx.try_iter().sum::<usize>(), n_jobs);
201−
202− build_dir("tree-sitter-typescript/tsx", "tsx");
203− build_dir("tree-sitter-typescript/typescript", "typescript");
204− build_dir("tree-sitter-ocaml/ocaml", "ocaml");
205− build_dir("tree-sitter-ocaml/interface", "ocaml-interface")
206−}
helix-term/src/lib.rs+1
@@ -7,6 +7,7 @@ pub mod commands;
77pub mod compositor;
88pub mod config;
99pub mod health;
10+pub mod grammars;
1011pub mod job;
1112pub mod keymap;
1213pub mod ui;