on master help sign in
Don't take a HighlightingAssets detour to build assets (#1802)

Move code to build assets to its own file. That results in better modularity and flexibility. It also allows us to simplify HighlightingAssets a lot, since it will now always be initialized with a SerializedSyntaxSet.

commitf1c0fd7343c6c4c6a5ec198e533a6e8de3369472 authorMartin Nordholts <enselic@gmail.com> dateTue 24 Aug 2021, 07:58 (5 years ago) parent12dfbdc
5 files changed+112−139
CHANGELOG.md+1 src/assets.rs+12−134 src/bin/bat/main.rs+1−2 src/syntax_dependencies.rs → src/build_assets.rs+96−1 src/lib.rs+2−2
CHANGELOG.md+1
@@ -30,6 +30,7 @@
3030## `bat` as a library
3131
3232- Deprecate `HighlightingAssets::syntaxes()` and `HighlightingAssets::syntax_for_file_name()`. Use `HighlightingAssets::get_syntaxes()` and `HighlightingAssets::get_syntax_for_file_name()` instead. They return a `Result` which is needed for upcoming lazy-loading work to improve startup performance. They also return what `SyntaxSet` the returned `SyntaxReference` belongs to. See #1747, #1755 and #1776 (@Enselic)
33+- Remove `HighlightingAssets::from_files` and `HighlightingAssets::save_to_cache`. Instead of calling the former and then the latter you now make a single call to `bat::assets::build`. See #1802 (@Enselic)
3334
3435
3536
src/assets.rs+12−134
@@ -18,7 +18,7 @@ use crate::syntax_mapping::{MappingTarget, SyntaxMapping};
1818#[derive(Debug)]
1919pub struct HighlightingAssets {
2020 syntax_set_cell: LazyCell<SyntaxSet>,
21− serialized_syntax_set: Option<SerializedSyntaxSet>,
21+ serialized_syntax_set: SerializedSyntaxSet,
2222 theme_set: ThemeSet,
2323 fallback_theme: Option<&'static str>,
2424}
@@ -47,20 +47,9 @@ const IGNORED_SUFFIXES: [&str; 10] = [
4747];
4848
4949impl HighlightingAssets {
50− fn new(
51− syntax_set: Option<SyntaxSet>,
52− serialized_syntax_set: Option<SerializedSyntaxSet>,
53− theme_set: ThemeSet,
54− ) -> Self {
55− assert!(syntax_set.is_some() || serialized_syntax_set.is_some());
56−
57− let syntax_set_cell = LazyCell::new();
58− if let Some(syntax_set) = syntax_set {
59− syntax_set_cell.fill(syntax_set).expect("can never fail");
60− }
61−
50+ fn new(serialized_syntax_set: SerializedSyntaxSet, theme_set: ThemeSet) -> Self {
6251 HighlightingAssets {
63− syntax_set_cell,
52+ syntax_set_cell: LazyCell::new(),
6453 serialized_syntax_set,
6554 theme_set,
6655 fallback_theme: None,
@@ -71,127 +60,27 @@ impl HighlightingAssets {
7160 "Monokai Extended"
7261 }
7362
74− #[cfg(feature = "build-assets")]
75− pub fn from_files(source_dir: &Path, include_integrated_assets: bool) -> Result<Self> {
76− let mut theme_set = if include_integrated_assets {
77− get_integrated_themeset()
78− } else {
79− ThemeSet::new()
80− };
81−
82− let theme_dir = source_dir.join("themes");
83− if theme_dir.exists() {
84− let res = theme_set.add_from_folder(&theme_dir);
85− if let Err(err) = res {
86− println!(
87− "Failed to load one or more themes from '{}' (reason: '{}')",
88− theme_dir.to_string_lossy(),
89− err,
90− );
91− }
92− } else {
93− println!(
94− "No themes were found in '{}', using the default set",
95− theme_dir.to_string_lossy()
96− );
97− }
98−
99− let mut syntax_set_builder = if !include_integrated_assets {
100− let mut builder = syntect::parsing::SyntaxSetBuilder::new();
101− builder.add_plain_text_syntax();
102− builder
103− } else {
104− from_binary::<SyntaxSet>(get_serialized_integrated_syntaxset()).into_builder()
105− };
106−
107− let syntax_dir = source_dir.join("syntaxes");
108− if syntax_dir.exists() {
109− syntax_set_builder.add_from_folder(syntax_dir, true)?;
110− } else {
111− println!(
112− "No syntaxes were found in '{}', using the default set.",
113− syntax_dir.to_string_lossy()
114− );
115− }
116−
117− if std::env::var("BAT_PRINT_SYNTAX_DEPENDENCIES").is_ok() {
118− // To trigger this code, run:
119− // BAT_PRINT_SYNTAX_DEPENDENCIES=1 cargo run -- cache --build --source assets --blank --target /tmp
120− crate::syntax_dependencies::print_syntax_dependencies(&syntax_set_builder);
121− }
122−
123− let syntax_set = syntax_set_builder.build();
124− let missing_contexts = syntax_set.find_unlinked_contexts();
125− if !missing_contexts.is_empty() {
126− println!("Some referenced contexts could not be found!");
127− for context in missing_contexts {
128− println!("- {}", context);
129− }
130− }
131−
132− Ok(HighlightingAssets::new(Some(syntax_set), None, theme_set))
133− }
134−
13563 pub fn from_cache(cache_path: &Path) -> Result<Self> {
13664 Ok(HighlightingAssets::new(
137− None,
138− Some(SerializedSyntaxSet::FromFile(
139− cache_path.join("syntaxes.bin"),
140− )),
65+ SerializedSyntaxSet::FromFile(cache_path.join("syntaxes.bin")),
14166 asset_from_cache(&cache_path.join("themes.bin"), "theme set")?,
14267 ))
14368 }
14469
14570 pub fn from_binary() -> Self {
14671 HighlightingAssets::new(
147− None,
148− Some(SerializedSyntaxSet::FromBinary(
149− get_serialized_integrated_syntaxset(),
150− )),
72+ SerializedSyntaxSet::FromBinary(get_serialized_integrated_syntaxset()),
15173 get_integrated_themeset(),
15274 )
15375 }
15476
155− #[cfg(feature = "build-assets")]
156− pub fn save_to_cache(&self, target_dir: &Path, current_version: &str) -> Result<()> {
157− let _ = fs::create_dir_all(target_dir);
158− asset_to_cache(
159− self.get_theme_set(),
160− &target_dir.join("themes.bin"),
161− "theme set",
162− )?;
163− asset_to_cache(
164− self.get_syntax_set()?,
165− &target_dir.join("syntaxes.bin"),
166− "syntax set",
167− )?;
168−
169− print!(
170− "Writing metadata to folder {} ... ",
171− target_dir.to_string_lossy()
172− );
173− crate::assets_metadata::AssetsMetadata::new(current_version).save_to_folder(target_dir)?;
174− println!("okay");
175−
176− Ok(())
177− }
178−
17977 pub fn set_fallback_theme(&mut self, theme: &'static str) {
18078 self.fallback_theme = Some(theme);
18179 }
18280
18381 pub(crate) fn get_syntax_set(&self) -> Result<&SyntaxSet> {
184− if !self.syntax_set_cell.filled() {
185− self.syntax_set_cell.fill(
186− self.serialized_syntax_set
187− .as_ref()
188− .expect("a dev forgot to setup serialized_syntax_set, please report to https://github.com/sharkdp/bat/issues")
189− .deserialize()?
190− ).unwrap();
191− }
192−
193− // It is safe to .unwrap() because we just made sure it was .filled()
194− Ok(self.syntax_set_cell.borrow().unwrap())
82+ self.syntax_set_cell
83+ .try_borrow_with(|| self.serialized_syntax_set.deserialize())
19584 }
19685
19786 /// Use [Self::get_syntaxes] instead
@@ -393,6 +282,9 @@ impl HighlightingAssets {
393282 }
394283}
395284
285+#[cfg(feature = "build-assets")]
286+pub use crate::build_assets::build_assets as build;
287+
396288/// A SyntaxSet in serialized form, i.e. bincoded and flate2 compressed.
397289/// We keep it in this format since we want to load it lazily.
398290#[derive(Debug)]
@@ -413,28 +305,14 @@ impl SerializedSyntaxSet {
413305 }
414306}
415307
416−fn get_serialized_integrated_syntaxset() -> &'static [u8] {
308+pub(crate) fn get_serialized_integrated_syntaxset() -> &'static [u8] {
417309 include_bytes!("../assets/syntaxes.bin")
418310}
419311
420−fn get_integrated_themeset() -> ThemeSet {
312+pub(crate) fn get_integrated_themeset() -> ThemeSet {
421313 from_binary(include_bytes!("../assets/themes.bin"))
422314}
423315
424−#[cfg(feature = "build-assets")]
425−fn asset_to_cache<T: serde::Serialize>(asset: &T, path: &Path, description: &str) -> Result<()> {
426− print!("Writing {} to {} ... ", description, path.to_string_lossy());
427− syntect::dumps::dump_to_file(asset, &path).chain_err(|| {
428− format!(
429− "Could not save {} to {}",
430− description,
431− path.to_string_lossy()
432− )
433− })?;
434− println!("okay");
435− Ok(())
436−}
437−
438316fn asset_from_cache<T: serde::de::DeserializeOwned>(path: &Path, description: &str) -> Result<T> {
439317 let contents = fs::read(path).chain_err(|| {
440318 format!(
src/bin/bat/main.rs+1−2
@@ -50,8 +50,7 @@ fn build_assets(matches: &clap::ArgMatches) -> Result<()> {
5050
5151 let blank = matches.is_present("blank");
5252
53− let assets = bat::assets::HighlightingAssets::from_files(source_dir, !blank)?;
54− assets.save_to_cache(target_dir, clap::crate_version!())
53+ bat::assets::build(source_dir, !blank, target_dir, clap::crate_version!())
5554}
5655
5756fn run_cache_subcommand(matches: &clap::ArgMatches) -> Result<()> {
src/syntax_dependencies.rs → src/build_assets.rsrenamed+96−1
@@ -1,9 +1,14 @@
11use std::collections::HashMap;
2+use std::path::Path;
3+use syntect::dumps::from_binary;
4+use syntect::highlighting::ThemeSet;
25use syntect::parsing::syntax_definition::{
36 ContextReference, MatchOperation, MatchPattern, Pattern, SyntaxDefinition,
47};
58use syntect::parsing::{Scope, SyntaxSet, SyntaxSetBuilder};
69
10+use crate::error::*;
11+
712type SyntaxName = String;
813
914/// Used to look up what dependencies a given [SyntaxDefinition] has
@@ -22,9 +27,86 @@ enum Dependency {
2227 ByScope(Scope),
2328}
2429
30+pub fn build_assets(
31+ source_dir: &Path,
32+ include_integrated_assets: bool,
33+ target_dir: &Path,
34+ current_version: &str,
35+) -> Result<()> {
36+ let mut theme_set = if include_integrated_assets {
37+ crate::assets::get_integrated_themeset()
38+ } else {
39+ ThemeSet::new()
40+ };
41+
42+ let theme_dir = source_dir.join("themes");
43+ if theme_dir.exists() {
44+ let res = theme_set.add_from_folder(&theme_dir);
45+ if let Err(err) = res {
46+ println!(
47+ "Failed to load one or more themes from '{}' (reason: '{}')",
48+ theme_dir.to_string_lossy(),
49+ err,
50+ );
51+ }
52+ } else {
53+ println!(
54+ "No themes were found in '{}', using the default set",
55+ theme_dir.to_string_lossy()
56+ );
57+ }
58+
59+ let mut syntax_set_builder = if !include_integrated_assets {
60+ let mut builder = syntect::parsing::SyntaxSetBuilder::new();
61+ builder.add_plain_text_syntax();
62+ builder
63+ } else {
64+ from_binary::<SyntaxSet>(crate::assets::get_serialized_integrated_syntaxset())
65+ .into_builder()
66+ };
67+
68+ let syntax_dir = source_dir.join("syntaxes");
69+ if syntax_dir.exists() {
70+ syntax_set_builder.add_from_folder(syntax_dir, true)?;
71+ } else {
72+ println!(
73+ "No syntaxes were found in '{}', using the default set.",
74+ syntax_dir.to_string_lossy()
75+ );
76+ }
77+
78+ if std::env::var("BAT_PRINT_SYNTAX_DEPENDENCIES").is_ok() {
79+ // To trigger this code, run:
80+ // BAT_PRINT_SYNTAX_DEPENDENCIES=1 cargo run -- cache --build --source assets --blank --target /tmp
81+ print_syntax_dependencies(&syntax_set_builder);
82+ }
83+
84+ let syntax_set = syntax_set_builder.build();
85+ let missing_contexts = syntax_set.find_unlinked_contexts();
86+ if !missing_contexts.is_empty() {
87+ println!("Some referenced contexts could not be found!");
88+ for context in missing_contexts {
89+ println!("- {}", context);
90+ }
91+ }
92+
93+ let _ = std::fs::create_dir_all(target_dir);
94+ asset_to_cache(&theme_set, &target_dir.join("themes.bin"), "theme set")?;
95+ asset_to_cache(&syntax_set, &target_dir.join("syntaxes.bin"), "syntax set")?;
96+
97+ print!(
98+ "Writing metadata to folder {} ... ",
99+ target_dir.to_string_lossy()
100+ );
101+ crate::assets_metadata::AssetsMetadata::new(current_version).save_to_folder(target_dir)?;
102+ println!("okay");
103+
104+ Ok(())
105+}
106+
25107/// Generates independent [SyntaxSet]s after analyzing dependencies between syntaxes
26108/// in a [SyntaxSetBuilder], and then prints the reults.
27−pub(crate) fn print_syntax_dependencies(syntax_set_builder: &SyntaxSetBuilder) {
109+fn print_syntax_dependencies(syntax_set_builder: &SyntaxSetBuilder) {
28110 println!("Constructing independent SyntaxSets...");
29111 let independent_syntax_sets = build_independent_syntax_sets(syntax_set_builder);
30112
@@ -182,3 +264,16 @@ impl SyntaxSetDependencyBuilder {
182264 self.syntax_set_builder.build()
183265 }
184266}
267+
268+fn asset_to_cache<T: serde::Serialize>(asset: &T, path: &Path, description: &str) -> Result<()> {
269+ print!("Writing {} to {} ... ", description, path.to_string_lossy());
270+ syntect::dumps::dump_to_file(asset, &path).chain_err(|| {
271+ format!(
272+ "Could not save {} to {}",
273+ description,
274+ path.to_string_lossy()
275+ )
276+ })?;
277+ println!("okay");
278+ Ok(())
279+}
src/lib.rs+2−2
@@ -23,6 +23,8 @@ mod macros;
2323
2424pub mod assets;
2525pub mod assets_metadata;
26+#[cfg(feature = "build-assets")]
27+mod build_assets;
2628pub mod config;
2729pub mod controller;
2830mod decorations;
@@ -40,8 +42,6 @@ mod preprocessor;
4042mod pretty_printer;
4143pub(crate) mod printer;
4244pub mod style;
43−#[cfg(feature = "build-assets")]
44−mod syntax_dependencies;
4545pub(crate) mod syntax_mapping;
4646mod terminal;
4747pub(crate) mod wrapping;