@@ -1,36 +1,9 @@
1−use std::collections::HashMap;
21use std::path::Path;
32use syntect::highlighting::ThemeSet;
4−use syntect::parsing::syntax_definition::{
5− ContextReference, MatchOperation, MatchPattern, Pattern, SyntaxDefinition,
6−};
7−use syntect::parsing::{Scope, SyntaxSet, SyntaxSetBuilder};
3+use syntect::parsing::{SyntaxSet, SyntaxSetBuilder};
84
95use crate::assets::*;
106
11−mod graphviz_utils;
12−
13−type SyntaxName = String;
14−
15−/// Used to look up which [SyntaxDefinition] corresponds to a given [OtherSyntax]
16−type OtherSyntaxLookup<'a> = HashMap<OtherSyntax, &'a SyntaxDefinition>;
17−
18−/// Used to look up what dependencies a given [SyntaxDefinition] has
19−type SyntaxToDependencies = HashMap<SyntaxName, Vec<OtherSyntax>>;
20−
21−/// Used to look up what other [SyntaxDefinition]s depend on a given [SyntaxDefinition]
22−type SyntaxToDependents<'a> = HashMap<SyntaxName, Vec<OtherSyntax>>;
23−
24−/// Represents some other `*.sublime-syntax` file, i.e. another [SyntaxDefinition].
25−#[derive(Debug, Eq, PartialEq, PartialOrd, Ord, Clone, Hash)]
26−pub(crate) enum OtherSyntax {
27− /// By name. Example YAML: `include: C.sublime-syntax` (name is `"C"`)
28− ByName(String),
29−
30− /// By scope. Example YAML: `embed: scope:source.c` (scope is `"source.c"`)
31− ByScope(Scope),
32−}
33−
347pub fn build(
358 source_dir: &Path,
369 include_integrated_assets: bool,
@@ -41,19 +14,11 @@ pub fn build(
4114
4215 let syntax_set_builder = build_syntax_set_builder(source_dir, include_integrated_assets)?;
4316
44− let minimal_syntaxes = build_minimal_syntaxes(&syntax_set_builder, include_integrated_assets)?;
45−
4617 let syntax_set = syntax_set_builder.build();
4718
4819 print_unlinked_contexts(&syntax_set);
4920
50− write_assets(
51− &theme_set,
52− &syntax_set,
53− &minimal_syntaxes,
54− target_dir,
55− current_version,
56− )
21+ write_assets(&theme_set, &syntax_set, target_dir, current_version)
5722}
5823
5924fn build_theme_set(source_dir: &Path, include_integrated_assets: bool) -> ThemeSet {
@@ -122,7 +87,6 @@ fn print_unlinked_contexts(syntax_set: &SyntaxSet) {
12287fn write_assets(
12388 theme_set: &ThemeSet,
12489 syntax_set: &SyntaxSet,
125− minimal_syntaxes: &MinimalSyntaxes,
12690 target_dir: &Path,
12791 current_version: &str,
12892) -> Result<()> {
@@ -139,12 +103,6 @@ fn write_assets(
139103 "syntax set",
140104 COMPRESS_SYNTAXES,
141105 )?;
142− asset_to_cache(
143− minimal_syntaxes,
144− &target_dir.join("minimal_syntaxes.bin"),
145− "minimal syntax sets",
146− COMPRESS_MINIMAL_SYNTAXES,
147− )?;
148106
149107 print!(
150108 "Writing metadata to folder {} ... ",
@@ -156,299 +114,6 @@ fn write_assets(
156114 Ok(())
157115}
158116
159−fn print_syntax_set_names(syntax_set: &SyntaxSet) {
160− let names = syntax_set
161− .syntaxes()
162− .iter()
163− .map(|syntax| &syntax.name)
164− .collect::<Vec<_>>();
165− println!("{:?}", names);
166−}
167−
168−fn build_minimal_syntaxes(
169− syntax_set_builder: &'_ SyntaxSetBuilder,
170− include_integrated_assets: bool,
171−) -> Result<MinimalSyntaxes> {
172− let mut minimal_syntaxes = MinimalSyntaxes {
173− by_name: HashMap::new(),
174− serialized_syntax_sets: vec![],
175− };
176−
177− if include_integrated_assets {
178− // Dependency info is not present in integrated assets, so we can't
179− // calculate minimal syntax sets. Return early without any data filled
180− // in. This means that no minimal syntax sets will be available to use, and
181− // the full, slow-to-deserialize, fallback syntax set will be used instead.
182− return Ok(minimal_syntaxes);
183− }
184−
185− let minimal_syntax_sets_to_serialize = build_minimal_syntax_sets(syntax_set_builder)
186− // For now, only store syntax sets with one syntax, otherwise
187− // the binary grows by several megs
188− .filter(|syntax_set| syntax_set.syntaxes().len() == 1);
189−
190− for minimal_syntax_set in minimal_syntax_sets_to_serialize {
191− // Remember what index it is found at
192− let current_index = minimal_syntaxes.serialized_syntax_sets.len();
193−
194− for syntax in minimal_syntax_set.syntaxes() {
195− minimal_syntaxes
196− .by_name
197− .insert(syntax.name.to_ascii_lowercase().clone(), current_index);
198− }
199−
200− let serialized_syntax_set = asset_to_contents(
201− &minimal_syntax_set,
202− &format!("failed to serialize minimal syntax set {}", current_index),
203− COMPRESS_SERIALIZED_MINIMAL_SYNTAXES,
204− )?;
205−
206− // Add last so that it ends up at `current_index`
207− minimal_syntaxes
208− .serialized_syntax_sets
209− .push(serialized_syntax_set);
210− }
211−
212− Ok(minimal_syntaxes)
213−}
214−
215−/// Analyzes dependencies between syntaxes in a [SyntaxSetBuilder].
216−/// From that, it builds minimal [SyntaxSet]s.
217−fn build_minimal_syntax_sets(
218− syntax_set_builder: &'_ SyntaxSetBuilder,
219−) -> impl Iterator<Item = SyntaxSet> + '_ {
220− let syntaxes = syntax_set_builder.syntaxes();
221−
222− // Build the data structures we need for dependency resolution
223− let (other_syntax_lookup, syntax_to_dependencies, syntax_to_dependents) =
224− generate_maps(syntaxes);
225−
226− maybe_write_syntax_dependencies_to_graphviz_dot_file(
227− &other_syntax_lookup,
228− &syntax_to_dependencies,
229− );
230−
231− // Create one minimal SyntaxSet from each (non-hidden) SyntaxDefinition
232− syntaxes.iter().filter_map(move |syntax| {
233− if syntax.hidden {
234− return None;
235− }
236−
237− let mut builder = SyntaxSetDependencyBuilder::new();
238− builder.add_with_dependencies(
239− syntax,
240− &other_syntax_lookup,
241− &syntax_to_dependencies,
242− &syntax_to_dependents,
243− );
244− let syntax_set = builder.build();
245−
246− if std::env::var("BAT_PRINT_SYNTAX_DEPENDENCIES").is_ok() {
247− // To trigger this code, run:
248− // BAT_PRINT_SYNTAX_DEPENDENCIES=1 cargo run -- cache --build --source assets --blank --target /tmp
249− print_syntax_set_names(&syntax_set);
250− }
251−
252− Some(syntax_set)
253− })
254−}
255−
256−/// In order to analyze dependencies, we need three key pieces of data.
257−///
258−/// * When we have a [OtherSyntax], we need to know what [SyntaxDefinition]
259−/// that corresponds to
260−/// * When we have a [SyntaxDefinition], we need to know what dependencies it
261−/// has
262−/// * When we have a [SyntaxDefinition], we need to know what other syntaxes
263−/// depend on it
264−///
265−/// This functions generates that data for each syntax.
266−fn generate_maps(
267− syntaxes: &[SyntaxDefinition],
268−) -> (OtherSyntaxLookup, SyntaxToDependencies, SyntaxToDependents) {
269− let mut other_syntax_lookup = HashMap::new();
270− let mut syntax_to_dependencies = HashMap::new();
271− let mut syntax_to_dependents = HashMap::new();
272−
273− for syntax in syntaxes {
274− other_syntax_lookup.insert(OtherSyntax::ByName(syntax.name.clone()), syntax);
275− other_syntax_lookup.insert(OtherSyntax::ByScope(syntax.scope), syntax);
276− }
277−
278− for syntax in syntaxes {
279− let dependencies = dependencies_for_syntax(syntax);
280−
281− for dependency in &dependencies {
282− if let Some(dependency) = other_syntax_lookup.get(dependency) {
283− syntax_to_dependents
284− .entry(dependency.name.clone())
285− .or_insert_with(Vec::new)
286− .push(OtherSyntax::ByName(syntax.name.clone()));
287− }
288− }
289−
290− syntax_to_dependencies.insert(syntax.name.clone(), dependencies);
291− }
292−
293− (
294− other_syntax_lookup,
295− syntax_to_dependencies,
296− syntax_to_dependents,
297− )
298−}
299−
300−/// Gets what external dependencies a given [SyntaxDefinition] has.
301−/// An external dependency is another `.sublime-syntax` file.
302−/// It does that by looking for variants of the following YAML patterns:
303−/// - `include: C.sublime-syntax`
304−/// - `embed: scope:source.c`
305−fn dependencies_for_syntax(syntax: &SyntaxDefinition) -> Vec<OtherSyntax> {
306− let mut dependencies: Vec<OtherSyntax> = syntax
307− .contexts
308− .values()
309− .flat_map(|context| &context.patterns)
310− .flat_map(dependencies_from_pattern)
311− .collect();
312−
313− // No need to track a dependency more than once
314− dependencies.sort();
315− dependencies.dedup();
316−
317− dependencies
318−}
319−
320−fn dependencies_from_pattern(pattern: &Pattern) -> Vec<OtherSyntax> {
321− match *pattern {
322− Pattern::Match(MatchPattern {
323− operation: MatchOperation::Push(ref context_references),
324− ..
325− }) => context_references
326− .iter()
327− .map(dependency_from_context_reference)
328− .collect(),
329− Pattern::Include(ref context_reference) => {
330− vec![dependency_from_context_reference(context_reference)]
331− }
332− _ => vec![],
333− }
334− .into_iter()
335− .flatten()
336− .collect()
337−}
338−
339−/// To generate a Graphviz dot file of syntax dependencies, do this:
340−/// ```bash
341−/// sudo apt install graphviz
342−/// BAT_SYNTAX_DEPENDENCIES_TO_GRAPHVIZ_DOT_FILE=/tmp/bat-syntax-dependencies.dot cargo run -- cache --build --source assets --blank --target /tmp
343−/// dot /tmp/bat-syntax-dependencies.dot -Tpng -o /tmp/bat-syntax-dependencies.png
344−/// open /tmp/bat-syntax-dependencies.png
345−/// ```
346−fn maybe_write_syntax_dependencies_to_graphviz_dot_file(
347− other_syntax_lookup: &OtherSyntaxLookup,
348− syntax_to_dependencies: &SyntaxToDependencies,
349−) {
350− if let Ok(dot_file_path) = std::env::var("BAT_SYNTAX_DEPENDENCIES_TO_GRAPHVIZ_DOT_FILE") {
351− graphviz_utils::try_syntax_dependencies_to_graphviz_dot_file(
352− other_syntax_lookup,
353− syntax_to_dependencies,
354− &dot_file_path,
355− );
356− }
357−}
358−
359−/// Removes any context name from the syntax reference.
360−///
361−/// When we track dependencies between syntaxes, we are not interested in
362−/// dependencies on specific contexts inside other syntaxes. We only care about
363−/// the dependency on the syntax itself.
364−///
365−/// For example, if a syntax includes another syntax like this:
366−/// ```yaml
367−/// - include: scope:source.c++#unique-variables
368−/// ```
369−/// we only want to track the dependency on `source.c++`.
370−fn remove_explicit_context(scope: Scope) -> Scope {
371− if let Some(without_context) = scope.build_string().split('#').next() {
372− Scope::new(without_context).expect("removing context reference must never fail")
373− } else {
374− scope
375− }
376−}
377−
378−fn dependency_from_context_reference(context_reference: &ContextReference) -> Option<OtherSyntax> {
379− match &context_reference {
380− ContextReference::File { ref name, .. } => Some(OtherSyntax::ByName(name.clone())),
381− ContextReference::ByScope { ref scope, .. } => {
382− Some(OtherSyntax::ByScope(remove_explicit_context(*scope)))
383− }
384− _ => None,
385− }
386−}
387−
388−/// Helper to construct a [SyntaxSetBuilder] that contains only [SyntaxDefinition]s
389−/// that have dependencies among them.
390−struct SyntaxSetDependencyBuilder {
391− syntax_set_builder: SyntaxSetBuilder,
392−}
393−
394−impl SyntaxSetDependencyBuilder {
395− fn new() -> Self {
396− SyntaxSetDependencyBuilder {
397− syntax_set_builder: SyntaxSetBuilder::new(),
398− }
399− }
400−
401− /// Add a [SyntaxDefinition] to the underlying [SyntaxSetBuilder].
402− /// Also resolve any dependencies it has and add those [SyntaxDefinition]s too.
403− /// This is a recursive process.
404− fn add_with_dependencies(
405− &mut self,
406− syntax: &SyntaxDefinition,
407− other_syntax_lookup: &OtherSyntaxLookup,
408− syntax_to_dependencies: &SyntaxToDependencies,
409− syntax_to_dependents: &SyntaxToDependents,
410− ) {
411− let name = &syntax.name;
412− if self.is_syntax_already_added(name) {
413− return;
414− }
415−
416− self.syntax_set_builder.add(syntax.clone());
417−
418− let mut syntaxes_to_add = vec![];
419− if let Some(dependencies) = syntax_to_dependencies.get(name) {
420− syntaxes_to_add.extend(dependencies);
421− }
422− if let Some(dependents) = syntax_to_dependents.get(name) {
423− // This will later be enabled intelligently
424− if std::env::var("BAT_INCLUDE_SYNTAX_DEPENDENTS").is_ok() {
425− syntaxes_to_add.extend(dependents);
426− }
427− }
428− for syntax_to_add in syntaxes_to_add {
429− if let Some(syntax_to_add) = other_syntax_lookup.get(syntax_to_add) {
430− self.add_with_dependencies(
431− syntax_to_add,
432− other_syntax_lookup,
433− syntax_to_dependencies,
434− syntax_to_dependents,
435− )
436− }
437− }
438− }
439−
440− fn is_syntax_already_added(&self, name: &str) -> bool {
441− self.syntax_set_builder
442− .syntaxes()
443− .iter()
444− .any(|syntax| syntax.name == name)
445− }
446−
447− fn build(self) -> SyntaxSet {
448− self.syntax_set_builder.build()
449− }
450−}
451−
452117fn asset_to_contents<T: serde::Serialize>(
453118 asset: &T,
454119 description: &str,
@@ -485,16 +150,3 @@ fn asset_to_cache<T: serde::Serialize>(
485150 println!("okay");
486151 Ok(())
487152}
488−
489−#[cfg(test)]
490−mod tests {
491− use super::*;
492−
493− #[test]
494− fn remove_explicit_context_sanity() {
495− // Example from Objective-C++.sublime-syntax
496− let scope = Scope::new("source.c++#unique-variables").unwrap();
497− let expected = Scope::new("source.c++").unwrap();
498− assert_eq!(remove_explicit_context(scope), expected);
499− }
500−}