git.iko.soy/helix
on master help sign in
implement build_grammars and fetch_grammars
00b2d61Michael Davis
helix-term/src/grammars.rs+157−48
compare with master
unifiedsplit
@@ -4,39 +4,170 @@ use std::time::SystemTime;
4use std::{
4use std::{
5 path::{Path, PathBuf},
5 path::{Path, PathBuf},
6 process::Command,
6 process::Command,
7+ sync::mpsc::channel,
7};
8};
8
9
9−use helix_core::syntax::DYLIB_EXTENSION;
10+use helix_core::syntax::{GrammarConfiguration, GrammarSource, DYLIB_EXTENSION};
10
11
11const BUILD_TARGET: &str = env!("BUILD_TARGET");
12const BUILD_TARGET: &str = env!("BUILD_TARGET");
13+const REMOTE_NAME: &str = "origin";
12
14
13−pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
15+pub fn fetch_grammars() -> Result<()> {
16+ run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
17+}
18+
19+pub fn build_grammars() -> Result<()> {
20+ run_parallel(get_grammar_configs(), build_grammar, "build")
21+}
22+
23+fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
24+where
25+ F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
26+{
14− let mut dirs = Vec::new();
27+ let mut n_jobs = 0;
15− let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
28+ let pool = threadpool::Builder::new().build();
29+ let (tx, rx) = channel();
30+
31+ for grammar in grammars {
32+ let tx = tx.clone();
33+ n_jobs += 1;
34+
35+ pool.execute(move || {
36+ let grammar_id = grammar.grammar_id.clone();
37+ job(grammar).unwrap_or_else(|err| {
38+ eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
39+ });
40+
41+ // report progress
42+ tx.send(1).unwrap();
43+ });
44+ }
45+ pool.join();
46+
47+ if rx.try_iter().sum::<usize>() == n_jobs {
48+ Ok(())
49+ } else {
50+ Err(anyhow!("Failed to {} some grammar(s).", action))
51+ }
52+}
16
53
17− for entry in fs::read_dir(path)? {
54+fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
18− let entry = entry?;
55+ if let GrammarSource::Git { remote, revision } = grammar.source {
19− let path = entry.path();
56+ let grammar_dir = helix_core::runtime_dir()
57+ .join("grammars/sources")
58+ .join(grammar.grammar_id.clone());
20
59
21− if !entry.file_type()?.is_dir() {
60+ fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
22− continue;
61+
62+ // create the grammar dir contains a git directory
63+ if !grammar_dir.join(".git").is_dir() {
64+ git(&grammar_dir, ["init"])?;
65+ }
66+
67+ // ensure the remote matches the configured remote
68+ if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
69+ set_remote(&grammar_dir, &remote)?;
23 }
70 }
24
71
25− let dir = path.file_name().unwrap().to_str().unwrap().to_string();
72+ // ensure the revision matches the configured revision
73+ if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
74+ // Fetch the exact revision from the remote.
75+ // Supported by server-side git since v2.5.0 (July 2015),
76+ // enabled by default on major git hosts.
77+ git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?;
78+ git(&grammar_dir, ["checkout", &revision])?;
26
79
27− // filter ignores
80+ println!(
28− if ignore.contains(&dir) {
81+ "Grammar '{}' checked out at '{}'.",
29− continue;
82+ grammar.grammar_id, revision
83+ );
84+ Ok(())
85+ } else {
86+ println!("Grammar '{}' is already up to date.", grammar.grammar_id);
87+ Ok(())
30 }
88 }
31− dirs.push(dir)
89+ } else {
90+ println!("Skipping local grammar '{}'", grammar.grammar_id);
91+ Ok(())
32 }
92 }
93+}
33
94
34− Ok(dirs)
95+// Sets the remote for a repository to the given URL, creating the remote if
96+// it does not yet exist.
97+fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
98+ git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
99+ .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
35}
100}
36
101
37−fn build_library(src_path: &Path, language: &str) -> Result<()> {
102+fn get_remote_url(repository: &Path) -> Option<String> {
103+ git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
104+}
105+
106+fn get_revision(repository: &Path) -> Option<String> {
107+ git(repository, ["rev-parse", "HEAD"]).ok()
108+}
109+
110+// A wrapper around 'git' commands which returns stdout in success and a
111+// helpful error message showing the command, stdout, and stderr in error.
112+fn git<I, S>(repository: &Path, args: I) -> Result<String>
113+where
114+ I: IntoIterator<Item = S>,
115+ S: AsRef<std::ffi::OsStr>,
116+{
117+ let output = Command::new("git")
118+ .args(args)
119+ .current_dir(repository)
120+ .output()?;
121+
122+ if output.status.success() {
123+ Ok(String::from_utf8_lossy(&output.stdout).into_owned())
124+ } else {
125+ // TODO: figure out how to display the git command using `args`
126+ Err(anyhow!(
127+ "Git command failed.\nStdout: {}\nStderr: {}",
128+ String::from_utf8_lossy(&output.stdout),
129+ String::from_utf8_lossy(&output.stderr),
130+ ))
131+ }
132+}
133+
134+fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
135+ let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
136+ PathBuf::from(path)
137+ } else {
138+ helix_core::runtime_dir()
139+ .join("grammars/sources")
140+ .join(grammar.grammar_id.clone())
141+ };
142+
143+ grammar_dir.read_dir().with_context(|| {
144+ format!(
145+ "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
146+ grammar_dir
147+ )
148+ })?;
149+
150+ let path = match grammar.path {
151+ Some(ref subpath) => grammar_dir.join(subpath),
152+ None => grammar_dir,
153+ }
154+ .join("src");
155+
156+ build_tree_sitter_library(&path, grammar)
157+}
158+
159+// Returns the set of grammar configurations the user requests.
160+// Grammars are configured in the default and user `languages.toml` and are
161+// merged. The `grammar_selection` key of the config is then used to filter
162+// down all grammars into a subset of the user's choosing.
163+fn get_grammar_configs() -> Vec<GrammarConfiguration> {
164+ let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
165+
166+ config.grammar
167+}
168+
169+fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
38 let header_path = src_path;
170 let header_path = src_path;
39− // let grammar_path = src_path.join("grammar.json");
40 let parser_path = src_path.join("parser.c");
171 let parser_path = src_path.join("parser.c");
41 let mut scanner_path = src_path.join("scanner.c");
172 let mut scanner_path = src_path.join("scanner.c");
42
173
@@ -50,16 +181,20 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
50 None
181 None
51 }
182 }
52 };
183 };
53− let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
184+ let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
54− let mut library_path = parser_lib_path.join(language);
185+ let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
55 library_path.set_extension(DYLIB_EXTENSION);
186 library_path.set_extension(DYLIB_EXTENSION);
56
187
57 let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
188 let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
58− .with_context(|| "Failed to compare source and binary timestamps")?;
189+ .context("Failed to compare source and binary timestamps")?;
59
190
60 if !recompile {
191 if !recompile {
192+ println!("Grammar '{}' is already built.", grammar.grammar_id);
61 return Ok(());
193 return Ok(());
62 }
194 }
195+
196+ println!("Building grammar '{}'", grammar.grammar_id);
197+
63 let mut config = cc::Build::new();
198 let mut config = cc::Build::new();
64 config
199 config
65 .cpp(true)
200 .cpp(true)
@@ -112,9 +247,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
112 }
247 }
113 }
248 }
114
249
115− let output = command
116− .output()
117− .with_context(|| "Failed to execute C compiler")?;
250+ let output = command.output().context("Failed to execute C compiler")?;
118 if !output.status.success() {
251 if !output.status.success() {
119 return Err(anyhow!(
252 return Err(anyhow!(
120 "Parser compilation failed.\nStdout: {}\nStderr: {}",
253 "Parser compilation failed.\nStdout: {}\nStderr: {}",
@@ -125,6 +258,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
125
258
126 Ok(())
259 Ok(())
127}
260}
261+
128fn needs_recompile(
262fn needs_recompile(
129 lib_path: &Path,
263 lib_path: &Path,
130 parser_c_path: &Path,
264 parser_c_path: &Path,
@@ -148,28 +282,3 @@ fn needs_recompile(
148fn mtime(path: &Path) -> Result<SystemTime> {
282fn mtime(path: &Path) -> Result<SystemTime> {
149 Ok(fs::metadata(path)?.modified()?)
283 Ok(fs::metadata(path)?.modified()?)
150}
284}
151−
152−pub fn build_dir(dir: &str, language: &str) {
153− println!("Build language {}", language);
154− if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
155− .join("../helix-syntax/languages")
156− .join(dir)
157− .read_dir()
158− .unwrap()
159− .next()
160− .is_none()
161− {
162− eprintln!(
163− "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
164− dir
165− );
166− std::process::exit(1);
167− }
168−
169− let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
170− .join("../helix-syntax/languages")
171− .join(dir)
172− .join("src");
173−
174− build_library(&path, language).unwrap();
175−}
helix-term/src/main.rs+10
@@ -104,6 +104,16 @@ FLAGS:
104 std::process::exit(0);
104 std::process::exit(0);
105 }
105 }
106
106
107+ if args.fetch_grammars {
108+ helix_term::grammars::fetch_grammars()?;
109+ return Ok(0);
110+ }
111+
112+ if args.build_grammars {
113+ helix_term::grammars::build_grammars()?;
114+ return Ok(0);
115+ }
116+
107 let conf_dir = helix_core::config_dir();
117 let conf_dir = helix_core::config_dir();
108 if !conf_dir.exists() {
118 if !conf_dir.exists() {
109 std::fs::create_dir_all(&conf_dir).ok();
119 std::fs::create_dir_all(&conf_dir).ok();