87978e7 · bat · git.iko.soy

git.iko.soy/bat

Make asset compression optional at compile time (#1825)

This will be needed to later support zero-copy deserialization of independent syntax sets, but is interesting and useful on its own. Instead of deferring serialization and deserialization to syntect, we implement it ourselves in the same way, but make compression optional.

commit87978e7755d614bbfc6e6939df9ebbcd04925cac authorMartin Nordholts <enselic@gmail.com> dateTue 7 Sep 2021, 17:21 (5 years ago) parentd935ea1

4 files changed+82−14

compare with master

unifiedsplit

Cargo.lock+2 Cargo.toml+3−1 src/assets.rs+37−7 src/build_assets.rs+40−6

Cargo.lock+2

@@ -89,6 +89,7 @@ dependencies = [

8989 "ansi_term 0.12.1",

9090 "assert_cmd",

9191 "atty",

92+ "bincode",

9293 "bugreport",

9394 "clap",

9495 "clircle",

@@ -96,6 +97,7 @@ dependencies = [

9697 "content_inspector",

9798 "dirs-next",

9899 "encoding",

100+ "flate2",

99101 "git2",

100102 "globset",

101103 "grep-cli",

Cargo.toml+3−1

@@ -45,7 +45,9 @@ regex-fancy = ["syntect/regex-fancy"] # Use the rust-only "fancy-regex" engine

4545atty = { version = "0.2.14", optional = true }

4646ansi_term = "^0.12.1"

4747ansi_colours = "^1.0"

48+bincode = "1.0"

4849console = "0.14.1"

50+flate2 = "1.0"

4951lazy_static = { version = "1.4", optional = true }

5052lazycell = "1.0"

5153thiserror = "1.0"

@@ -72,7 +74,7 @@ default-features = false

7274[dependencies.syntect]

7375version = "4.6.0"

7476default-features = false

75−features = ["parsing", "dump-load"]

77+features = ["parsing"]

7678

7779[dependencies.clap]

7880version = "2.33"

src/assets.rs+37−7

@@ -4,7 +4,6 @@ use std::path::{Path, PathBuf};

55use lazycell::LazyCell;

7−use syntect::dumps::{from_binary, from_reader};

87use syntect::highlighting::{Theme, ThemeSet};

98use syntect::parsing::{SyntaxReference, SyntaxSet};

109

@@ -29,6 +28,12 @@ pub struct SyntaxReferenceInSet<'a> {

2928 pub syntax_set: &'a SyntaxSet,

3029}

3130

31+// Compress for size of ~700 kB instead of ~4600 kB at the cost of ~30% longer deserialization time

32+pub(crate) const COMPRESS_SYNTAXES: bool = true;

33+

34+// Compress for size of ~20 kB instead of ~200 kB at the cost of ~30% longer deserialization time

35+pub(crate) const COMPRESS_THEMES: bool = true;

36+

3237const IGNORED_SUFFIXES: [&str; 13] = [

3338 // Editor etc backups

3439 "~",

@@ -66,7 +71,7 @@ impl HighlightingAssets {

6671 pub fn from_cache(cache_path: &Path) -> Result<Self> {

6772 Ok(HighlightingAssets::new(

6873 SerializedSyntaxSet::FromFile(cache_path.join("syntaxes.bin")),

69− asset_from_cache(&cache_path.join("themes.bin"), "theme set")?,

74+ asset_from_cache(&cache_path.join("themes.bin"), "theme set", COMPRESS_THEMES)?,

7075 ))

7176 }

7277

@@ -299,8 +304,10 @@ enum SerializedSyntaxSet {

299304impl SerializedSyntaxSet {

300305 fn deserialize(&self) -> Result<SyntaxSet> {

301306 match self {

302− SerializedSyntaxSet::FromBinary(data) => Ok(from_binary(data)),

303− SerializedSyntaxSet::FromFile(ref path) => asset_from_cache(path, "syntax set"),

307+ SerializedSyntaxSet::FromBinary(data) => Ok(from_binary(data, COMPRESS_SYNTAXES)),

308+ SerializedSyntaxSet::FromFile(ref path) => {

309+ asset_from_cache(path, "syntax set", COMPRESS_SYNTAXES)

310+ }

304311 }

305312 }

306313}

@@ -310,10 +317,32 @@ pub(crate) fn get_serialized_integrated_syntaxset() -> &'static [u8] {

310317}

311318

312319pub(crate) fn get_integrated_themeset() -> ThemeSet {

313− from_binary(include_bytes!("../assets/themes.bin"))

320+ from_binary(include_bytes!("../assets/themes.bin"), COMPRESS_THEMES)

321+}

322+

323+pub(crate) fn from_binary<T: serde::de::DeserializeOwned>(v: &[u8], compressed: bool) -> T {

324+ asset_from_contents(v, "n/a", compressed)

325+ .expect("data integrated in binary is never faulty, but make sure `compressed` is in sync!")

326+}

327+

328+fn asset_from_contents<T: serde::de::DeserializeOwned>(

329+ contents: &[u8],

330+ description: &str,

331+ compressed: bool,

332+) -> Result<T> {

333+ if compressed {

334+ bincode::deserialize_from(flate2::read::ZlibDecoder::new(contents))

335+ } else {

336+ bincode::deserialize_from(contents)

337+ }

338+ .map_err(|_| format!("Could not parse {}", description).into())

314339}

315340

316−fn asset_from_cache<T: serde::de::DeserializeOwned>(path: &Path, description: &str) -> Result<T> {

341+fn asset_from_cache<T: serde::de::DeserializeOwned>(

342+ path: &Path,

343+ description: &str,

344+ compressed: bool,

345+) -> Result<T> {

317346 let contents = fs::read(path).map_err(|_| {

318347 format!(

319348 "Could not load cached {} '{}'",

@@ -321,7 +350,8 @@ fn asset_from_cache<T: serde::de::DeserializeOwned>(path: &Path, description: &s

321350 path.to_string_lossy()

322351 )

323352 })?;

324− from_reader(&contents[..]).map_err(|_| format!("Could not parse cached {}", description).into())

353+ asset_from_contents(&contents[..], description, compressed)

354+ .map_err(|_| format!("Could not parse cached {}", description).into())

325355}

326356

327357#[cfg(test)]

src/build_assets.rs+40−6

@@ -1,12 +1,12 @@

11use std::collections::HashMap;

22use std::path::Path;

3−use syntect::dumps::from_binary;

43use syntect::highlighting::ThemeSet;

54use syntect::parsing::syntax_definition::{

65 ContextReference, MatchOperation, MatchPattern, Pattern, SyntaxDefinition,

76};

87use syntect::parsing::{Scope, SyntaxSet, SyntaxSetBuilder};

9+use crate::assets::*;

1010use crate::error::*;

1111

1212type SyntaxName = String;

@@ -86,7 +86,7 @@ fn build_syntax_set_builder(

8686 builder.add_plain_text_syntax();

8787 builder

8888 } else {

89− from_binary::<SyntaxSet>(crate::assets::get_serialized_integrated_syntaxset())

89+ from_binary::<SyntaxSet>(get_serialized_integrated_syntaxset(), COMPRESS_SYNTAXES)

9090 .into_builder()

9191 };

9292

@@ -120,8 +120,18 @@ fn write_assets(

120120 current_version: &str,

121121) -> Result<()> {

122122 let _ = std::fs::create_dir_all(target_dir);

123− asset_to_cache(theme_set, &target_dir.join("themes.bin"), "theme set")?;

124− asset_to_cache(syntax_set, &target_dir.join("syntaxes.bin"), "syntax set")?;

123+ asset_to_cache(

124+ theme_set,

125+ &target_dir.join("themes.bin"),

126+ "theme set",

127+ COMPRESS_THEMES,

128+ )?;

129+ asset_to_cache(

130+ syntax_set,

131+ &target_dir.join("syntaxes.bin"),

132+ "syntax set",

133+ COMPRESS_SYNTAXES,

134+ )?;

125135

126136 print!(

127137 "Writing metadata to folder {} ... ",

@@ -294,9 +304,33 @@ impl SyntaxSetDependencyBuilder {

294304 }

295305}

296306

297−fn asset_to_cache<T: serde::Serialize>(asset: &T, path: &Path, description: &str) -> Result<()> {

307+fn asset_to_contents<T: serde::Serialize>(

308+ asset: &T,

309+ description: &str,

310+ compressed: bool,

311+) -> Result<Vec<u8>> {

312+ let mut contents = vec![];

313+ if compressed {

314+ bincode::serialize_into(

315+ flate2::write::ZlibEncoder::new(&mut contents, flate2::Compression::best()),

316+ asset,

317+ )

318+ } else {

319+ bincode::serialize_into(&mut contents, asset)

320+ }

321+ .map_err(|_| format!("Could not serialize {}", description))?;

322+ Ok(contents)

323+}

324+

325+fn asset_to_cache<T: serde::Serialize>(

326+ asset: &T,

327+ path: &Path,

328+ description: &str,

329+ compressed: bool,

330+) -> Result<()> {

298331 print!("Writing {} to {} ... ", description, path.to_string_lossy());

299− syntect::dumps::dump_to_file(asset, &path).map_err(|_| {

332+ let contents = asset_to_contents(asset, description, compressed)?;

333+ std::fs::write(path, &contents[..]).map_err(|_| {

300334 format!(

301335 "Could not save {} to {}",

302336 description,