@@ -1,3 +1,4 @@
1+use std::collections::HashMap;
12use std::ffi::OsStr;
23use std::fs;
34use std::path::{Path, PathBuf};
@@ -18,6 +19,14 @@ use crate::syntax_mapping::{MappingTarget, SyntaxMapping};
1819pub struct HighlightingAssets {
1920 syntax_set_cell: LazyCell<SyntaxSet>,
2021 serialized_syntax_set: SerializedSyntaxSet,
22+
23+ minimal_syntaxes: MinimalSyntaxes,
24+
25+ /// Lazily load serialized [SyntaxSet]s from [Self.minimal_syntaxes]. The
26+ /// index in this vec matches the index in
27+ /// [Self.minimal_syntaxes.serialized_syntax_sets]
28+ deserialized_minimal_syntaxes: Vec<LazyCell<SyntaxSet>>,
29+
2130 theme_set: ThemeSet,
2231 fallback_theme: Option<&'static str>,
2332}
@@ -28,12 +37,39 @@ pub struct SyntaxReferenceInSet<'a> {
2837 pub syntax_set: &'a SyntaxSet,
2938}
3039
40+/// Stores and allows lookup of minimal [SyntaxSet]s. The [SyntaxSet]s are
41+/// stored in serialized form, and are deserialized on-demand. This gives good
42+/// startup performance since only the necessary [SyntaxReference]s needs to be
43+/// deserialized.
44+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
45+pub(crate) struct MinimalSyntaxes {
46+ /// Lookup the index into `serialized_syntax_sets` of a [SyntaxSet] by the
47+ /// name of any [SyntaxReference] inside the [SyntaxSet]
48+ /// (We will later add `by_extension`, `by_first_line`, etc.)
49+ pub(crate) by_name: HashMap<String, usize>,
50+
51+ /// Serialized [SyntaxSet]s. Whether or not this data is compressed is
52+ /// decided by [COMPRESS_SERIALIZED_MINIMAL_SYNTAXES]
53+ pub(crate) serialized_syntax_sets: Vec<Vec<u8>>,
54+}
55+
3156// Compress for size of ~700 kB instead of ~4600 kB at the cost of ~30% longer deserialization time
3257pub(crate) const COMPRESS_SYNTAXES: bool = true;
3358
3459// Compress for size of ~20 kB instead of ~200 kB at the cost of ~30% longer deserialization time
3560pub(crate) const COMPRESS_THEMES: bool = true;
3661
62+// Compress for size of ~400 kB instead of ~2100 kB at the cost of ~30% longer deserialization time
63+pub(crate) const COMPRESS_SERIALIZED_MINIMAL_SYNTAXES: bool = true;
64+
65+// Whether or not to compress the serialized form of [MinimalSyntaxes]. Shall
66+// always be `false`, because the data in
67+// [MinimalSyntaxes.serialized_syntax_sets] has already been compressed
68+// (assuming [COMPRESS_SERIALIZED_MINIMAL_SYNTAXES] is `true`). The "outer" data
69+// structures like `by_name` are tiny. If we compress, deserialization can't do
70+// efficient byte-by-byte copy of `serialized_syntax_sets`.
71+pub(crate) const COMPRESS_MINIMAL_SYNTAXES: bool = false;
72+
3773const IGNORED_SUFFIXES: [&str; 13] = [
3874 // Editor etc backups
3975 "~",
@@ -55,10 +91,20 @@ const IGNORED_SUFFIXES: [&str; 13] = [
5591];
5692
5793impl HighlightingAssets {
58− fn new(serialized_syntax_set: SerializedSyntaxSet, theme_set: ThemeSet) -> Self {
94+ fn new(
95+ serialized_syntax_set: SerializedSyntaxSet,
96+ minimal_syntaxes: MinimalSyntaxes,
97+ theme_set: ThemeSet,
98+ ) -> Self {
99+ // Prepare so we can lazily load minimal syntaxes without a mut reference
100+ let deserialized_minimal_syntaxes =
101+ vec![LazyCell::new(); minimal_syntaxes.serialized_syntax_sets.len()];
102+
59103 HighlightingAssets {
60104 syntax_set_cell: LazyCell::new(),
61105 serialized_syntax_set,
106+ deserialized_minimal_syntaxes,
107+ minimal_syntaxes,
62108 theme_set,
63109 fallback_theme: None,
64110 }
@@ -71,6 +117,11 @@ impl HighlightingAssets {
71117 pub fn from_cache(cache_path: &Path) -> Result<Self> {
72118 Ok(HighlightingAssets::new(
73119 SerializedSyntaxSet::FromFile(cache_path.join("syntaxes.bin")),
120+ asset_from_cache(
121+ &cache_path.join("minimal_syntaxes.bin"),
122+ "minimal syntax sets",
123+ COMPRESS_MINIMAL_SYNTAXES,
124+ )?,
74125 asset_from_cache(&cache_path.join("themes.bin"), "theme set", COMPRESS_THEMES)?,
75126 ))
76127 }
@@ -78,6 +129,7 @@ impl HighlightingAssets {
78129 pub fn from_binary() -> Self {
79130 HighlightingAssets::new(
80131 SerializedSyntaxSet::FromBinary(get_serialized_integrated_syntaxset()),
132+ get_integrated_minimal_syntaxes(),
81133 get_integrated_themeset(),
82134 )
83135 }
@@ -111,6 +163,41 @@ impl HighlightingAssets {
111163 self.get_theme_set().themes.keys().map(|s| s.as_ref())
112164 }
113165
166+ /// Finds a [SyntaxSet] that contains a [SyntaxReference] by its name. First
167+ /// tries to find a minimal [SyntaxSet]. If none is found, returns the
168+ /// [SyntaxSet] that contains all syntaxes.
169+ fn get_syntax_set_by_name(&self, name: &str) -> Result<&SyntaxSet> {
170+ let minimal_syntax_set = self
171+ .minimal_syntaxes
172+ .by_name
173+ .get(&name.to_ascii_lowercase())
174+ .and_then(|index| self.get_minimal_syntax_set_with_index(*index));
175+
176+ match minimal_syntax_set {
177+ Some(syntax_set) => Ok(syntax_set),
178+ None => self.get_syntax_set(),
179+ }
180+ }
181+
182+ fn load_minimal_syntax_set_with_index(&self, index: usize) -> Result<SyntaxSet> {
183+ let serialized_syntax_set = &self.minimal_syntaxes.serialized_syntax_sets[index];
184+ asset_from_contents(
185+ &serialized_syntax_set[..],
186+ &format!("minimal syntax set {}", index),
187+ COMPRESS_SERIALIZED_MINIMAL_SYNTAXES,
188+ )
189+ .map_err(|_| format!("Could not parse minimal syntax set {}", index).into())
190+ }
191+
192+ fn get_minimal_syntax_set_with_index(&self, index: usize) -> Option<&SyntaxSet> {
193+ self.deserialized_minimal_syntaxes
194+ .get(index)
195+ .and_then(|cell| {
196+ cell.try_borrow_with(|| self.load_minimal_syntax_set_with_index(index))
197+ .ok()
198+ })
199+ }
200+
114201 /// Use [Self::get_syntax_for_file_name] instead
115202 #[deprecated]
116203 pub fn syntax_for_file_name(
@@ -167,7 +254,7 @@ impl HighlightingAssets {
167254 mapping: &SyntaxMapping,
168255 ) -> Result<SyntaxReferenceInSet> {
169256 if let Some(language) = language {
170− let syntax_set = self.get_syntax_set()?;
257+ let syntax_set = self.get_syntax_set_by_name(language)?;
171258 syntax_set
172259 .find_syntax_by_token(language)
173260 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
@@ -320,6 +407,13 @@ pub(crate) fn get_integrated_themeset() -> ThemeSet {
320407 from_binary(include_bytes!("../assets/themes.bin"), COMPRESS_THEMES)
321408}
322409
410+fn get_integrated_minimal_syntaxes() -> MinimalSyntaxes {
411+ from_binary(
412+ include_bytes!("../assets/minimal_syntaxes.bin"),
413+ COMPRESS_MINIMAL_SYNTAXES,
414+ )
415+}
416+
323417pub(crate) fn from_binary<T: serde::de::DeserializeOwned>(v: &[u8], compressed: bool) -> T {
324418 asset_from_contents(v, "n/a", compressed)
325419 .expect("data integrated in binary is never faulty, but make sure `compressed` is in sync!")