git.iko.soy/bat
on 448bfaf help sign in
bat/src/assets.rs
632 lines 22 kB history raw
1use std::ffi::OsStr;
2use std::fs;
3use std::path::Path;
5use once_cell::unsync::OnceCell;
7use syntect::highlighting::Theme;
8use syntect::parsing::{SyntaxReference, SyntaxSet};
10use path_abs::PathAbs;
12use crate::error::*;
13use crate::input::{InputReader, OpenedInput};
14use crate::syntax_mapping::ignored_suffixes::IgnoredSuffixes;
15use crate::syntax_mapping::MappingTarget;
16use crate::{bat_warning, SyntaxMapping};
18use lazy_theme_set::LazyThemeSet;
20use serialized_syntax_set::*;
22#[cfg(feature = "build-assets")]
23pub use crate::assets::build_assets::*;
25pub(crate) mod assets_metadata;
26#[cfg(feature = "build-assets")]
27mod build_assets;
28mod lazy_theme_set;
29mod serialized_syntax_set;
31#[derive(Debug)]
32pub struct HighlightingAssets {
33 syntax_set_cell: OnceCell<SyntaxSet>,
34 serialized_syntax_set: SerializedSyntaxSet,
36 theme_set: LazyThemeSet,
37 fallback_theme: Option<&'static str>,
40#[derive(Debug)]
41pub struct SyntaxReferenceInSet<'a> {
42 pub syntax: &'a SyntaxReference,
43 pub syntax_set: &'a SyntaxSet,
46/// Compress for size of ~700 kB instead of ~4600 kB at the cost of ~30% longer deserialization time
47pub(crate) const COMPRESS_SYNTAXES: bool = true;
49/// We don't want to compress our [LazyThemeSet] since the lazy-loaded themes
50/// within it are already compressed, and compressing another time just makes
51/// performance suffer
52pub(crate) const COMPRESS_THEMES: bool = false;
54/// Compress for size of ~40 kB instead of ~200 kB without much difference in
55/// performance due to lazy-loading
56pub(crate) const COMPRESS_LAZY_THEMES: bool = true;
58/// Compress for size of ~10 kB instead of ~120 kB
59pub(crate) const COMPRESS_ACKNOWLEDGEMENTS: bool = true;
61impl HighlightingAssets {
62 fn new(serialized_syntax_set: SerializedSyntaxSet, theme_set: LazyThemeSet) -> Self {
63 HighlightingAssets {
64 syntax_set_cell: OnceCell::new(),
65 serialized_syntax_set,
66 theme_set,
67 fallback_theme: None,
68 }
69 }
71 pub fn default_theme() -> &'static str {
72 "Monokai Extended"
73 }
75 pub fn from_cache(cache_path: &Path) -> Result<Self> {
76 Ok(HighlightingAssets::new(
77 SerializedSyntaxSet::FromFile(cache_path.join("syntaxes.bin")),
78 asset_from_cache(&cache_path.join("themes.bin"), "theme set", COMPRESS_THEMES)?,
79 ))
80 }
82 pub fn from_binary() -> Self {
83 HighlightingAssets::new(
84 SerializedSyntaxSet::FromBinary(get_serialized_integrated_syntaxset()),
85 get_integrated_themeset(),
86 )
87 }
89 pub fn set_fallback_theme(&mut self, theme: &'static str) {
90 self.fallback_theme = Some(theme);
91 }
93 /// Return the collection of syntect syntax definitions.
94 pub fn get_syntax_set(&self) -> Result<&SyntaxSet> {
95 self.syntax_set_cell
96 .get_or_try_init(|| self.serialized_syntax_set.deserialize())
97 }
99 /// Use [Self::get_syntaxes] instead
100 #[deprecated]
101 pub fn syntaxes(&self) -> &[SyntaxReference] {
102 self.get_syntax_set()
103 .expect(".syntaxes() is deprecated, use .get_syntaxes() instead")
104 .syntaxes()
105 }
107 pub fn get_syntaxes(&self) -> Result<&[SyntaxReference]> {
108 Ok(self.get_syntax_set()?.syntaxes())
109 }
111 fn get_theme_set(&self) -> &LazyThemeSet {
112 &self.theme_set
113 }
115 pub fn themes(&self) -> impl Iterator<Item = &str> {
116 self.get_theme_set().themes()
117 }
119 /// Use [Self::get_syntax_for_path] instead
120 #[deprecated]
121 pub fn syntax_for_file_name(
122 &self,
123 file_name: impl AsRef<Path>,
124 mapping: &SyntaxMapping,
125 ) -> Option<&SyntaxReference> {
126 self.get_syntax_for_path(file_name, mapping)
127 .ok()
128 .map(|syntax_in_set| syntax_in_set.syntax)
129 }
131 /// Detect the syntax based on, in order:
132 /// 1. Syntax mappings with [MappingTarget::MapTo] and [MappingTarget::MapToUnknown]
133 /// (e.g. `/etc/profile` -> `Bourne Again Shell (bash)`)
134 /// 2. The file name (e.g. `Dockerfile`)
135 /// 3. Syntax mappings with [MappingTarget::MapExtensionToUnknown]
136 /// (e.g. `*.conf`)
137 /// 4. The file name extension (e.g. `.rs`)
138 ///
139 /// When detecting syntax based on syntax mappings, the full path is taken
140 /// into account. When detecting syntax based on file name, no regard is
141 /// taken to the path of the file. Only the file name itself matters. When
142 /// detecting syntax based on file name extension, only the file name
143 /// extension itself matters.
144 ///
145 /// Returns [Error::UndetectedSyntax] if it was not possible detect syntax
146 /// based on path/file name/extension (or if the path was mapped to
147 /// [MappingTarget::MapToUnknown] or [MappingTarget::MapExtensionToUnknown]).
148 /// In this case it is appropriate to fall back to other methods to detect
149 /// syntax. Such as using the contents of the first line of the file.
150 ///
151 /// Returns [Error::UnknownSyntax] if a syntax mapping exist, but the mapped
152 /// syntax does not exist.
153 pub fn get_syntax_for_path(
154 &self,
155 path: impl AsRef<Path>,
156 mapping: &SyntaxMapping,
157 ) -> Result<SyntaxReferenceInSet> {
158 let path = path.as_ref();
160 let syntax_match = mapping.get_syntax_for(path);
162 if let Some(MappingTarget::MapToUnknown) = syntax_match {
163 return Err(Error::UndetectedSyntax(path.to_string_lossy().into()));
164 }
166 if let Some(MappingTarget::MapTo(syntax_name)) = syntax_match {
167 return self
168 .find_syntax_by_name(syntax_name)?
169 .ok_or_else(|| Error::UnknownSyntax(syntax_name.to_owned()));
170 }
172 let file_name = path.file_name().unwrap_or_default();
174 match (
175 self.get_syntax_for_file_name(file_name, &mapping.ignored_suffixes)?,
176 syntax_match,
177 ) {
178 (Some(syntax), _) => Ok(syntax),
180 (_, Some(MappingTarget::MapExtensionToUnknown)) => {
181 Err(Error::UndetectedSyntax(path.to_string_lossy().into()))
182 }
184 _ => self
185 .get_syntax_for_file_extension(file_name, &mapping.ignored_suffixes)?
186 .ok_or_else(|| Error::UndetectedSyntax(path.to_string_lossy().into())),
187 }
188 }
190 /// Look up a syntect theme by name.
191 pub fn get_theme(&self, theme: &str) -> &Theme {
192 match self.get_theme_set().get(theme) {
193 Some(theme) => theme,
194 None => {
195 if theme == "ansi-light" || theme == "ansi-dark" {
196 bat_warning!("Theme '{}' is deprecated, using 'ansi' instead.", theme);
197 return self.get_theme("ansi");
198 }
199 if !theme.is_empty() {
200 bat_warning!("Unknown theme '{}', using default.", theme)
201 }
202 self.get_theme_set()
203 .get(self.fallback_theme.unwrap_or_else(Self::default_theme))
204 .expect("something is very wrong if the default theme is missing")
205 }
206 }
207 }
209 pub(crate) fn get_syntax(
210 &self,
211 language: Option<&str>,
212 input: &mut OpenedInput,
213 mapping: &SyntaxMapping,
214 ) -> Result<SyntaxReferenceInSet> {
215 if let Some(language) = language {
216 let syntax_set = self.get_syntax_set()?;
217 return syntax_set
218 .find_syntax_by_token(language)
219 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
220 .ok_or_else(|| Error::UnknownSyntax(language.to_owned()));
221 }
223 let path = input.path();
224 let path_syntax = if let Some(path) = path {
225 self.get_syntax_for_path(
226 PathAbs::new(path).map_or_else(|_| path.to_owned(), |p| p.as_path().to_path_buf()),
227 mapping,
228 )
229 } else {
230 Err(Error::UndetectedSyntax("[unknown]".into()))
231 };
233 match path_syntax {
234 // If a path wasn't provided, or if path based syntax detection
235 // above failed, we fall back to first-line syntax detection.
236 Err(Error::UndetectedSyntax(path)) => self
237 .get_first_line_syntax(&mut input.reader)?
238 .ok_or(Error::UndetectedSyntax(path)),
239 _ => path_syntax,
240 }
241 }
243 pub(crate) fn find_syntax_by_name(
244 &self,
245 syntax_name: &str,
246 ) -> Result<Option<SyntaxReferenceInSet>> {
247 let syntax_set = self.get_syntax_set()?;
248 Ok(syntax_set
249 .find_syntax_by_name(syntax_name)
250 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
251 }
253 fn find_syntax_by_extension(&self, e: Option<&OsStr>) -> Result<Option<SyntaxReferenceInSet>> {
254 let syntax_set = self.get_syntax_set()?;
255 let extension = e.and_then(|x| x.to_str()).unwrap_or_default();
256 Ok(syntax_set
257 .find_syntax_by_extension(extension)
258 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
259 }
261 fn get_syntax_for_file_name(
262 &self,
263 file_name: &OsStr,
264 ignored_suffixes: &IgnoredSuffixes,
265 ) -> Result<Option<SyntaxReferenceInSet>> {
266 let mut syntax = self.find_syntax_by_extension(Some(file_name))?;
267 if syntax.is_none() {
268 syntax =
269 ignored_suffixes.try_with_stripped_suffix(file_name, |stripped_file_name| {
270 // Note: recursion
271 self.get_syntax_for_file_name(stripped_file_name, ignored_suffixes)
272 })?;
273 }
274 Ok(syntax)
275 }
277 fn get_syntax_for_file_extension(
278 &self,
279 file_name: &OsStr,
280 ignored_suffixes: &IgnoredSuffixes,
281 ) -> Result<Option<SyntaxReferenceInSet>> {
282 let mut syntax = self.find_syntax_by_extension(Path::new(file_name).extension())?;
283 if syntax.is_none() {
284 syntax =
285 ignored_suffixes.try_with_stripped_suffix(file_name, |stripped_file_name| {
286 // Note: recursion
287 self.get_syntax_for_file_extension(stripped_file_name, ignored_suffixes)
288 })?;
289 }
290 Ok(syntax)
291 }
293 fn get_first_line_syntax(
294 &self,
295 reader: &mut InputReader,
296 ) -> Result<Option<SyntaxReferenceInSet>> {
297 let syntax_set = self.get_syntax_set()?;
298 Ok(String::from_utf8(reader.first_line.clone())
299 .ok()
300 .and_then(|l| syntax_set.find_syntax_by_first_line(&l))
301 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
302 }
305pub(crate) fn get_serialized_integrated_syntaxset() -> &'static [u8] {
306 include_bytes!("../assets/syntaxes.bin")
309pub(crate) fn get_integrated_themeset() -> LazyThemeSet {
310 from_binary(include_bytes!("../assets/themes.bin"), COMPRESS_THEMES)
313pub fn get_acknowledgements() -> String {
314 from_binary(
315 include_bytes!("../assets/acknowledgements.bin"),
316 COMPRESS_ACKNOWLEDGEMENTS,
317 )
320pub(crate) fn from_binary<T: serde::de::DeserializeOwned>(v: &[u8], compressed: bool) -> T {
321 asset_from_contents(v, "n/a", compressed)
322 .expect("data integrated in binary is never faulty, but make sure `compressed` is in sync!")
325fn asset_from_contents<T: serde::de::DeserializeOwned>(
326 contents: &[u8],
327 description: &str,
328 compressed: bool,
329) -> Result<T> {
330 if compressed {
331 bincode::deserialize_from(flate2::read::ZlibDecoder::new(contents))
332 } else {
333 bincode::deserialize_from(contents)
334 }
335 .map_err(|_| format!("Could not parse {}", description).into())
338fn asset_from_cache<T: serde::de::DeserializeOwned>(
339 path: &Path,
340 description: &str,
341 compressed: bool,
342) -> Result<T> {
343 let contents = fs::read(path).map_err(|_| {
344 format!(
345 "Could not load cached {} '{}'",
346 description,
347 path.to_string_lossy()
348 )
349 })?;
350 asset_from_contents(&contents[..], description, compressed)
351 .map_err(|_| format!("Could not parse cached {}", description).into())
354#[cfg(test)]
355mod tests {
356 use super::*;
358 use std::ffi::OsStr;
360 use std::fs::File;
361 use std::io::{BufReader, Write};
362 use tempfile::TempDir;
364 use crate::input::Input;
366 struct SyntaxDetectionTest<'a> {
367 assets: HighlightingAssets,
368 pub syntax_mapping: SyntaxMapping<'a>,
369 pub temp_dir: TempDir,
370 }
372 impl<'a> SyntaxDetectionTest<'a> {
373 fn new() -> Self {
374 SyntaxDetectionTest {
375 assets: HighlightingAssets::from_binary(),
376 syntax_mapping: SyntaxMapping::builtin(),
377 temp_dir: TempDir::new().expect("creation of temporary directory"),
378 }
379 }
381 fn get_syntax_name(
382 &self,
383 language: Option<&str>,
384 input: &mut OpenedInput,
385 mapping: &SyntaxMapping,
386 ) -> String {
387 self.assets
388 .get_syntax(language, input, mapping)
389 .map(|syntax_in_set| syntax_in_set.syntax.name.clone())
390 .unwrap_or_else(|_| "!no syntax!".to_owned())
391 }
393 fn syntax_for_real_file_with_content_os(
394 &self,
395 file_name: &OsStr,
396 first_line: &str,
397 ) -> String {
398 let file_path = self.temp_dir.path().join(file_name);
399 {
400 let mut temp_file = File::create(&file_path).unwrap();
401 writeln!(temp_file, "{}", first_line).unwrap();
402 }
404 let input = Input::ordinary_file(&file_path);
405 let dummy_stdin: &[u8] = &[];
406 let mut opened_input = input.open(dummy_stdin, None).unwrap();
408 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
409 }
411 fn syntax_for_file_with_content_os(&self, file_name: &OsStr, first_line: &str) -> String {
412 let file_path = self.temp_dir.path().join(file_name);
413 let input = Input::from_reader(Box::new(BufReader::new(first_line.as_bytes())))
414 .with_name(Some(&file_path));
415 let dummy_stdin: &[u8] = &[];
416 let mut opened_input = input.open(dummy_stdin, None).unwrap();
418 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
419 }
421 #[cfg(unix)]
422 fn syntax_for_file_os(&self, file_name: &OsStr) -> String {
423 self.syntax_for_file_with_content_os(file_name, "")
424 }
426 fn syntax_for_file_with_content(&self, file_name: &str, first_line: &str) -> String {
427 self.syntax_for_file_with_content_os(OsStr::new(file_name), first_line)
428 }
430 fn syntax_for_file(&self, file_name: &str) -> String {
431 self.syntax_for_file_with_content(file_name, "")
432 }
434 fn syntax_for_stdin_with_content(&self, file_name: &str, content: &[u8]) -> String {
435 let input = Input::stdin().with_name(Some(file_name));
436 let mut opened_input = input.open(content, None).unwrap();
438 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
439 }
441 fn syntax_is_same_for_inputkinds(&self, file_name: &str, content: &str) -> bool {
442 let as_file = self.syntax_for_real_file_with_content_os(file_name.as_ref(), content);
443 let as_reader = self.syntax_for_file_with_content_os(file_name.as_ref(), content);
444 let consistent = as_file == as_reader;
445 // TODO: Compare StdIn somehow?
447 if !consistent {
448 eprintln!(
449 "Inconsistent syntax detection:\nFor File: {}\nFor Reader: {}",
450 as_file, as_reader
451 )
452 }
454 consistent
455 }
456 }
458 #[test]
459 fn syntax_detection_basic() {
460 let test = SyntaxDetectionTest::new();
462 assert_eq!(test.syntax_for_file("test.rs"), "Rust");
463 assert_eq!(test.syntax_for_file("test.cpp"), "C++");
464 assert_eq!(test.syntax_for_file("test.build"), "NAnt Build File");
465 assert_eq!(
466 test.syntax_for_file("PKGBUILD"),
467 "Bourne Again Shell (bash)"
468 );
469 assert_eq!(test.syntax_for_file(".bashrc"), "Bourne Again Shell (bash)");
470 assert_eq!(test.syntax_for_file("Makefile"), "Makefile");
471 }
473 #[cfg(unix)]
474 #[test]
475 fn syntax_detection_invalid_utf8() {
476 use std::os::unix::ffi::OsStrExt;
478 let test = SyntaxDetectionTest::new();
480 assert_eq!(
481 test.syntax_for_file_os(OsStr::from_bytes(b"invalid_\xFEutf8_filename.rs")),
482 "Rust"
483 );
484 }
486 #[test]
487 fn syntax_detection_same_for_inputkinds() {
488 let mut test = SyntaxDetectionTest::new();
490 test.syntax_mapping
491 .insert("*.myext", MappingTarget::MapTo("C"))
492 .ok();
493 test.syntax_mapping
494 .insert("MY_FILE", MappingTarget::MapTo("Markdown"))
495 .ok();
497 assert!(test.syntax_is_same_for_inputkinds("Test.md", ""));
498 assert!(test.syntax_is_same_for_inputkinds("Test.txt", "#!/bin/bash"));
499 assert!(test.syntax_is_same_for_inputkinds(".bashrc", ""));
500 assert!(test.syntax_is_same_for_inputkinds("test.h", ""));
501 assert!(test.syntax_is_same_for_inputkinds("test.js", "#!/bin/bash"));
502 assert!(test.syntax_is_same_for_inputkinds("test.myext", ""));
503 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", ""));
504 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", "<?php"));
505 }
507 #[test]
508 fn syntax_detection_well_defined_mapping_for_duplicate_extensions() {
509 let test = SyntaxDetectionTest::new();
511 assert_eq!(test.syntax_for_file("test.h"), "C++");
512 assert_eq!(test.syntax_for_file("test.sass"), "Sass");
513 assert_eq!(test.syntax_for_file("test.js"), "JavaScript (Babel)");
514 assert_eq!(test.syntax_for_file("test.fs"), "F#");
515 assert_eq!(test.syntax_for_file("test.v"), "Verilog");
516 }
518 #[test]
519 fn syntax_detection_first_line() {
520 let test = SyntaxDetectionTest::new();
522 assert_eq!(
523 test.syntax_for_file_with_content("my_script", "#!/bin/bash"),
524 "Bourne Again Shell (bash)"
525 );
526 assert_eq!(
527 test.syntax_for_file_with_content("build", "#!/bin/bash"),
528 "Bourne Again Shell (bash)"
529 );
530 assert_eq!(
531 test.syntax_for_file_with_content("my_script", "<?php"),
532 "PHP"
533 );
534 }
536 #[test]
537 fn syntax_detection_with_custom_mapping() {
538 let mut test = SyntaxDetectionTest::new();
540 assert_eq!(test.syntax_for_file("test.h"), "C++");
541 test.syntax_mapping
542 .insert("*.h", MappingTarget::MapTo("C"))
543 .ok();
544 assert_eq!(test.syntax_for_file("test.h"), "C");
545 }
547 #[test]
548 fn syntax_detection_with_extension_mapping_to_unknown() {
549 let mut test = SyntaxDetectionTest::new();
551 // Normally, a CMakeLists.txt file shall use the CMake syntax, even if it is
552 // a bash script in disguise
553 assert_eq!(
554 test.syntax_for_file_with_content("CMakeLists.txt", "#!/bin/bash"),
555 "CMake"
556 );
558 // Other .txt files shall use the Plain Text syntax
559 assert_eq!(
560 test.syntax_for_file_with_content("some-other.txt", "#!/bin/bash"),
561 "Plain Text"
562 );
564 // If we setup MapExtensionToUnknown on *.txt, the match on the full
565 // file name of "CMakeLists.txt" shall have higher prio, and CMake shall
566 // still be used for it
567 test.syntax_mapping
568 .insert("*.txt", MappingTarget::MapExtensionToUnknown)
569 .ok();
570 assert_eq!(
571 test.syntax_for_file_with_content("CMakeLists.txt", "#!/bin/bash"),
572 "CMake"
573 );
575 // However, for *other* files with a .txt extension, first-line fallback
576 // shall now be used
577 assert_eq!(
578 test.syntax_for_file_with_content("some-other.txt", "#!/bin/bash"),
579 "Bourne Again Shell (bash)"
580 );
581 }
583 #[test]
584 fn syntax_detection_is_case_sensitive() {
585 let mut test = SyntaxDetectionTest::new();
587 assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
588 test.syntax_mapping
589 .insert("*.MD", MappingTarget::MapTo("Markdown"))
590 .ok();
591 assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
592 }
594 #[test]
595 fn syntax_detection_stdin_filename() {
596 let test = SyntaxDetectionTest::new();
598 // from file extension
599 assert_eq!(test.syntax_for_stdin_with_content("test.cpp", b"a"), "C++");
600 // from first line (fallback)
601 assert_eq!(
602 test.syntax_for_stdin_with_content("my_script", b"#!/bin/bash"),
603 "Bourne Again Shell (bash)"
604 );
605 }
607 #[cfg(unix)]
608 #[test]
609 fn syntax_detection_for_symlinked_file() {
610 use std::os::unix::fs::symlink;
612 let test = SyntaxDetectionTest::new();
613 let file_path = test.temp_dir.path().join("my_ssh_config_filename");
614 {
615 File::create(&file_path).unwrap();
616 }
617 let file_path_symlink = test.temp_dir.path().join(".ssh").join("config");
619 std::fs::create_dir(test.temp_dir.path().join(".ssh"))
620 .expect("creation of directory succeeds");
621 symlink(&file_path, &file_path_symlink).expect("creation of symbolic link succeeds");
623 let input = Input::ordinary_file(&file_path_symlink);
624 let dummy_stdin: &[u8] = &[];
625 let mut opened_input = input.open(dummy_stdin, None).unwrap();
627 assert_eq!(
628 test.get_syntax_name(None, &mut opened_input, &test.syntax_mapping),
629 "SSH Config"
630 );
631 }