@@ -1,63 +1,155 @@
11use anyhow::{anyhow, Context, Result};
2+use libloading::{Library, Symbol};
3+use serde::{Deserialize, Serialize};
24use std::fs;
35use std::time::SystemTime;
46use std::{
7+ collections::HashSet,
58 path::{Path, PathBuf},
69 process::Command,
710 sync::mpsc::channel,
811};
12+use tree_sitter::Language;
913
10−use helix_core::syntax::{GrammarConfiguration, GrammarSelection, GrammarSource, DYLIB_EXTENSION};
14+#[cfg(unix)]
15+const DYLIB_EXTENSION: &str = "so";
16+
17+#[cfg(windows)]
18+const DYLIB_EXTENSION: &str = "dll";
19+
20+#[derive(Debug, Serialize, Deserialize)]
21+struct Configuration {
22+ #[serde(rename = "use-grammars")]
23+ pub grammar_selection: Option<GrammarSelection>,
24+ pub grammar: Vec<GrammarConfiguration>,
25+}
26+
27+#[derive(Debug, Serialize, Deserialize)]
28+#[serde(rename_all = "lowercase", untagged)]
29+pub enum GrammarSelection {
30+ Only(HashSet<String>),
31+ Except(HashSet<String>),
32+}
33+
34+#[derive(Debug, Serialize, Deserialize)]
35+#[serde(deny_unknown_fields)]
36+pub struct GrammarConfiguration {
37+ #[serde(rename = "name")]
38+ pub grammar_id: String,
39+ pub source: GrammarSource,
40+}
41+
42+#[derive(Debug, Serialize, Deserialize)]
43+#[serde(rename_all = "lowercase", untagged)]
44+pub enum GrammarSource {
45+ Local {
46+ path: String,
47+ },
48+ Git {
49+ #[serde(rename = "git")]
50+ remote: String,
51+ #[serde(rename = "rev")]
52+ revision: String,
53+ subpath: Option<String>,
54+ },
55+}
1156
1257const BUILD_TARGET: &str = env!("BUILD_TARGET");
1358const REMOTE_NAME: &str = "origin";
1459
60+pub fn get_language(name: &str) -> Result<Language> {
61+ let name = name.to_ascii_lowercase();
62+ let mut library_path = crate::runtime_dir().join("grammars").join(&name);
63+ library_path.set_extension(DYLIB_EXTENSION);
64+
65+ let library = unsafe { Library::new(&library_path) }
66+ .with_context(|| format!("Error opening dynamic library {library_path:?}"))?;
67+ let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
68+ let language = unsafe {
69+ let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
70+ .get(language_fn_name.as_bytes())
71+ .with_context(|| format!("Failed to load symbol {language_fn_name}"))?;
72+ language_fn()
73+ };
74+ std::mem::forget(library);
75+ Ok(language)
76+}
77+
1578pub fn fetch_grammars() -> Result<()> {
16− run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
79+ run_parallel(get_grammar_configs()?, fetch_grammar, "fetch")
1780}
1881
1982pub fn build_grammars() -> Result<()> {
20− run_parallel(get_grammar_configs(), build_grammar, "build")
83+ run_parallel(get_grammar_configs()?, build_grammar, "build")
84+}
85+
86+// Returns the set of grammar configurations the user requests.
87+// Grammars are configured in the default and user `languages.toml` and are
88+// merged. The `grammar_selection` key of the config is then used to filter
89+// down all grammars into a subset of the user's choosing.
90+fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
91+ let config: Configuration = crate::user_lang_config()
92+ .context("Could not parse languages.toml")?
93+ .try_into()?;
94+
95+ let grammars = match config.grammar_selection {
96+ Some(GrammarSelection::Only(selections)) => config
97+ .grammar
98+ .into_iter()
99+ .filter(|grammar| selections.contains(&grammar.grammar_id))
100+ .collect(),
101+ Some(GrammarSelection::Except(rejections)) => config
102+ .grammar
103+ .into_iter()
104+ .filter(|grammar| !rejections.contains(&grammar.grammar_id))
105+ .collect(),
106+ None => config.grammar,
107+ };
108+
109+ Ok(grammars)
21110}
22111
23112fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
24113where
25114 F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
26115{
27− let mut n_jobs = 0;
28116 let pool = threadpool::Builder::new().build();
29117 let (tx, rx) = channel();
30118
31119 for grammar in grammars {
32120 let tx = tx.clone();
33− n_jobs += 1;
34121
35122 pool.execute(move || {
36− let grammar_id = grammar.grammar_id.clone();
37− job(grammar).unwrap_or_else(|err| {
38− eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
39− });
40−
41− // report progress
42− tx.send(1).unwrap();
123+ tx.send(job(grammar)).unwrap();
43124 });
44125 }
45126 pool.join();
46127
47− if rx.try_iter().sum::<usize>() == n_jobs {
48− Ok(())
128+ // TODO: print all failures instead of the first one found.
129+ if let Some(failure) = rx.try_iter().find_map(|result| result.err()) {
130+ Err(anyhow!(
131+ "Failed to {} some grammar(s).\n{}",
132+ action,
133+ failure
134+ ))
49135 } else {
50− Err(anyhow!("Failed to {} some grammar(s).", action))
136+ Ok(())
51137 }
52138}
53139
54140fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
55− if let GrammarSource::Git { remote, revision } = grammar.source {
56− let grammar_dir = helix_core::runtime_dir()
141+ if let GrammarSource::Git {
142+ remote, revision, ..
143+ } = grammar.source
144+ {
145+ let grammar_dir = crate::runtime_dir()
57146 .join("grammars/sources")
58− .join(grammar.grammar_id.clone());
147+ .join(&grammar.grammar_id);
59148
60− fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
149+ fs::create_dir_all(&grammar_dir).context(format!(
150+ "Could not create grammar directory {:?}",
151+ grammar_dir
152+ ))?;
61153
62154 // create the grammar dir contains a git directory
63155 if !grammar_dir.join(".git").is_dir() {
@@ -65,12 +157,12 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
65157 }
66158
67159 // ensure the remote matches the configured remote
68− if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
160+ if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) {
69161 set_remote(&grammar_dir, &remote)?;
70162 }
71163
72164 // ensure the revision matches the configured revision
73− if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
165+ if get_revision(&grammar_dir).map_or(true, |s| s != revision) {
74166 // Fetch the exact revision from the remote.
75167 // Supported by server-side git since v2.5.0 (July 2015),
76168 // enabled by default on major git hosts.
@@ -94,33 +186,38 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
94186
95187// Sets the remote for a repository to the given URL, creating the remote if
96188// it does not yet exist.
97−fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
98− git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
99− .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
189+fn set_remote(repository_dir: &Path, remote_url: &str) -> Result<String> {
190+ git(
191+ repository_dir,
192+ ["remote", "set-url", REMOTE_NAME, remote_url],
193+ )
194+ .or_else(|_| git(repository_dir, ["remote", "add", REMOTE_NAME, remote_url]))
100195}
101196
102−fn get_remote_url(repository: &Path) -> Option<String> {
103− git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
197+fn get_remote_url(repository_dir: &Path) -> Option<String> {
198+ git(repository_dir, ["remote", "get-url", REMOTE_NAME]).ok()
104199}
105200
106−fn get_revision(repository: &Path) -> Option<String> {
107− git(repository, ["rev-parse", "HEAD"]).ok()
201+fn get_revision(repository_dir: &Path) -> Option<String> {
202+ git(repository_dir, ["rev-parse", "HEAD"]).ok()
108203}
109204
110205// A wrapper around 'git' commands which returns stdout in success and a
111206// helpful error message showing the command, stdout, and stderr in error.
112−fn git<I, S>(repository: &Path, args: I) -> Result<String>
207+fn git<I, S>(repository_dir: &Path, args: I) -> Result<String>
113208where
114209 I: IntoIterator<Item = S>,
115210 S: AsRef<std::ffi::OsStr>,
116211{
117212 let output = Command::new("git")
118213 .args(args)
119− .current_dir(repository)
214+ .current_dir(repository_dir)
120215 .output()?;
121216
122217 if output.status.success() {
123− Ok(String::from_utf8_lossy(&output.stdout).into_owned())
218+ Ok(String::from_utf8_lossy(&output.stdout)
219+ .trim_end()
220+ .to_owned())
124221 } else {
125222 // TODO: figure out how to display the git command using `args`
126223 Err(anyhow!(
@@ -132,52 +229,37 @@ where
132229}
133230
134231fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
135− let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
136− PathBuf::from(path)
232+ println!("{:#?}", grammar);
233+ let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
234+ PathBuf::from(&path)
137235 } else {
138− helix_core::runtime_dir()
236+ crate::runtime_dir()
139237 .join("grammars/sources")
140− .join(grammar.grammar_id.clone())
238+ .join(&grammar.grammar_id)
141239 };
142240
143− grammar_dir.read_dir().with_context(|| {
144− format!(
145− "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
146− grammar_dir
147− )
241+ let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
242+ format!("Failed to read directory {grammar_dir:?}. Did you use 'hx --fetch-grammars'?")
148243 })?;
149244
150− let path = match grammar.path {
151− Some(ref subpath) => grammar_dir.join(subpath),
152− None => grammar_dir,
245+ if grammar_dir_entries.count() == 0 {
246+ return Err(anyhow!(
247+ "Directory {grammar_dir:?} is empty. Did you use 'hx --fetch-grammars'?"
248+ ));
249+ };
250+
251+ let path = match &grammar.source {
252+ GrammarSource::Git {
253+ subpath: Some(subpath),
254+ ..
255+ } => grammar_dir.join(subpath),
256+ _ => grammar_dir,
153257 }
154258 .join("src");
155259
156260 build_tree_sitter_library(&path, grammar)
157261}
158262
159−// Returns the set of grammar configurations the user requests.
160−// Grammars are configured in the default and user `languages.toml` and are
161−// merged. The `grammar_selection` key of the config is then used to filter
162−// down all grammars into a subset of the user's choosing.
163−fn get_grammar_configs() -> Vec<GrammarConfiguration> {
164− let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
165−
166− match config.grammar_selection {
167− Some(GrammarSelection::Only(selections)) => config
168− .grammar
169− .into_iter()
170− .filter(|grammar| selections.contains(&grammar.grammar_id))
171− .collect(),
172− Some(GrammarSelection::Except(rejections)) => config
173− .grammar
174− .into_iter()
175− .filter(|grammar| !rejections.contains(&grammar.grammar_id))
176− .collect(),
177− None => config.grammar,
178− }
179−}
180−
181263fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
182264 let header_path = src_path;
183265 let parser_path = src_path.join("parser.c");
@@ -193,8 +275,8 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
193275 None
194276 }
195277 };
196− let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
197− let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
278+ let parser_lib_path = crate::runtime_dir().join("grammars");
279+ let mut library_path = parser_lib_path.join(&grammar.grammar_id);
198280 library_path.set_extension(DYLIB_EXTENSION);
199281
200282 let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
@@ -210,7 +292,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
210292 let mut config = cc::Build::new();
211293 config
212294 .cpp(true)
213− .opt_level(2)
295+ .opt_level(3)
214296 .cargo_metadata(false)
215297 .host(BUILD_TARGET)
216298 .target(BUILD_TARGET);
@@ -245,7 +327,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
245327 .arg(header_path)
246328 .arg("-o")
247329 .arg(&library_path)
248− .arg("-O2");
330+ .arg("-O3");
249331 if let Some(scanner_path) = scanner_path.as_ref() {
250332 if scanner_path.extension() == Some("c".as_ref()) {
251333 command.arg("-xc").arg("-std=c99").arg(scanner_path);
@@ -294,3 +376,13 @@ fn needs_recompile(
294376fn mtime(path: &Path) -> Result<SystemTime> {
295377 Ok(fs::metadata(path)?.modified()?)
296378}
379+
380+/// Gives the contents of a file from a language's `runtime/queries/<lang>`
381+/// directory
382+pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
383+ let path = crate::RUNTIME_DIR
384+ .join("queries")
385+ .join(language)
386+ .join(filename);
387+ std::fs::read_to_string(&path)
388+}