git.iko.soy/bat
on 4236fa9 help sign in
bat/src/assets.rs
610 lines 21 kB history raw
1use std::ffi::OsStr;
2use std::fs;
3use std::path::Path;
5use once_cell::unsync::OnceCell;
7use syntect::highlighting::{Theme, ThemeSet};
8use syntect::parsing::{SyntaxReference, SyntaxSet};
10use path_abs::PathAbs;
12use crate::error::*;
13use crate::input::{InputReader, OpenedInput};
14use crate::syntax_mapping::ignored_suffixes::IgnoredSuffixes;
15use crate::syntax_mapping::MappingTarget;
16use crate::{bat_warning, SyntaxMapping};
18use serialized_syntax_set::*;
20#[cfg(feature = "build-assets")]
21pub use crate::assets::build_assets::*;
23pub(crate) mod assets_metadata;
24#[cfg(feature = "build-assets")]
25mod build_assets;
26mod serialized_syntax_set;
28#[derive(Debug)]
29pub struct HighlightingAssets {
30 syntax_set_cell: OnceCell<SyntaxSet>,
31 serialized_syntax_set: SerializedSyntaxSet,
33 theme_set: ThemeSet,
34 fallback_theme: Option<&'static str>,
37#[derive(Debug)]
38pub struct SyntaxReferenceInSet<'a> {
39 pub syntax: &'a SyntaxReference,
40 pub syntax_set: &'a SyntaxSet,
43/// Compress for size of ~700 kB instead of ~4600 kB at the cost of ~30% longer deserialization time
44pub(crate) const COMPRESS_SYNTAXES: bool = true;
46/// Compress for size of ~20 kB instead of ~200 kB at the cost of ~30% longer deserialization time
47pub(crate) const COMPRESS_THEMES: bool = true;
49impl HighlightingAssets {
50 fn new(serialized_syntax_set: SerializedSyntaxSet, theme_set: ThemeSet) -> Self {
51 HighlightingAssets {
52 syntax_set_cell: OnceCell::new(),
53 serialized_syntax_set,
54 theme_set,
55 fallback_theme: None,
56 }
57 }
59 pub fn default_theme() -> &'static str {
60 "Monokai Extended"
61 }
63 pub fn from_cache(cache_path: &Path) -> Result<Self> {
64 Ok(HighlightingAssets::new(
65 SerializedSyntaxSet::FromFile(cache_path.join("syntaxes.bin")),
66 asset_from_cache(&cache_path.join("themes.bin"), "theme set", COMPRESS_THEMES)?,
67 ))
68 }
70 pub fn from_binary() -> Self {
71 HighlightingAssets::new(
72 SerializedSyntaxSet::FromBinary(get_serialized_integrated_syntaxset()),
73 get_integrated_themeset(),
74 )
75 }
77 pub fn set_fallback_theme(&mut self, theme: &'static str) {
78 self.fallback_theme = Some(theme);
79 }
81 fn get_syntax_set(&self) -> Result<&SyntaxSet> {
82 self.syntax_set_cell
83 .get_or_try_init(|| self.serialized_syntax_set.deserialize())
84 }
86 /// Use [Self::get_syntaxes] instead
87 #[deprecated]
88 pub fn syntaxes(&self) -> &[SyntaxReference] {
89 self.get_syntax_set()
90 .expect(".syntaxes() is deprecated, use .get_syntaxes() instead")
91 .syntaxes()
92 }
94 pub fn get_syntaxes(&self) -> Result<&[SyntaxReference]> {
95 Ok(self.get_syntax_set()?.syntaxes())
96 }
98 fn get_theme_set(&self) -> &ThemeSet {
99 &self.theme_set
100 }
102 pub fn themes(&self) -> impl Iterator<Item = &str> {
103 self.get_theme_set().themes.keys().map(|s| s.as_ref())
104 }
106 /// Use [Self::get_syntax_for_path] instead
107 #[deprecated]
108 pub fn syntax_for_file_name(
109 &self,
110 file_name: impl AsRef<Path>,
111 mapping: &SyntaxMapping,
112 ) -> Option<&SyntaxReference> {
113 self.get_syntax_for_path(file_name, mapping)
114 .ok()
115 .map(|syntax_in_set| syntax_in_set.syntax)
116 }
118 /// Detect the syntax based on, in order:
119 /// 1. Syntax mappings with [MappingTarget::MapTo] and [MappingTarget::MapToUnknown]
120 /// (e.g. `/etc/profile` -> `Bourne Again Shell (bash)`)
121 /// 2. The file name (e.g. `Dockerfile`)
122 /// 3. Syntax mappings with [MappingTarget::MapExtensionToUnknown]
123 /// (e.g. `*.conf`)
124 /// 4. The file name extension (e.g. `.rs`)
125 ///
126 /// When detecting syntax based on syntax mappings, the full path is taken
127 /// into account. When detecting syntax based on file name, no regard is
128 /// taken to the path of the file. Only the file name itself matters. When
129 /// detecting syntax based on file name extension, only the file name
130 /// extension itself matters.
131 ///
132 /// Returns [Error::UndetectedSyntax] if it was not possible detect syntax
133 /// based on path/file name/extension (or if the path was mapped to
134 /// [MappingTarget::MapToUnknown] or [MappingTarget::MapExtensionToUnknown]).
135 /// In this case it is appropriate to fall back to other methods to detect
136 /// syntax. Such as using the contents of the first line of the file.
137 ///
138 /// Returns [Error::UnknownSyntax] if a syntax mapping exist, but the mapped
139 /// syntax does not exist.
140 pub fn get_syntax_for_path(
141 &self,
142 path: impl AsRef<Path>,
143 mapping: &SyntaxMapping,
144 ) -> Result<SyntaxReferenceInSet> {
145 let path = path.as_ref();
147 let syntax_match = mapping.get_syntax_for(path);
149 if let Some(MappingTarget::MapToUnknown) = syntax_match {
150 return Err(Error::UndetectedSyntax(path.to_string_lossy().into()));
151 }
153 if let Some(MappingTarget::MapTo(syntax_name)) = syntax_match {
154 return self
155 .find_syntax_by_name(syntax_name)?
156 .ok_or_else(|| Error::UnknownSyntax(syntax_name.to_owned()));
157 }
159 let file_name = path.file_name().unwrap_or_default();
161 match (
162 self.get_syntax_for_file_name(file_name, &mapping.ignored_suffixes)?,
163 syntax_match,
164 ) {
165 (Some(syntax), _) => Ok(syntax),
167 (_, Some(MappingTarget::MapExtensionToUnknown)) => {
168 Err(Error::UndetectedSyntax(path.to_string_lossy().into()))
169 }
171 _ => self
172 .get_syntax_for_file_extension(file_name, &mapping.ignored_suffixes)?
173 .ok_or_else(|| Error::UndetectedSyntax(path.to_string_lossy().into())),
174 }
175 }
177 pub(crate) fn get_theme(&self, theme: &str) -> &Theme {
178 match self.get_theme_set().themes.get(theme) {
179 Some(theme) => theme,
180 None => {
181 if theme == "ansi-light" || theme == "ansi-dark" {
182 bat_warning!("Theme '{}' is deprecated, using 'ansi' instead.", theme);
183 return self.get_theme("ansi");
184 }
185 if !theme.is_empty() {
186 bat_warning!("Unknown theme '{}', using default.", theme)
187 }
188 &self.get_theme_set().themes
189 [self.fallback_theme.unwrap_or_else(|| Self::default_theme())]
190 }
191 }
192 }
194 pub(crate) fn get_syntax(
195 &self,
196 language: Option<&str>,
197 input: &mut OpenedInput,
198 mapping: &SyntaxMapping,
199 ) -> Result<SyntaxReferenceInSet> {
200 if let Some(language) = language {
201 let syntax_set = self.get_syntax_set()?;
202 return syntax_set
203 .find_syntax_by_token(language)
204 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
205 .ok_or_else(|| Error::UnknownSyntax(language.to_owned()));
206 }
208 let path = input.path();
209 let path_syntax = if let Some(path) = path {
210 self.get_syntax_for_path(
211 PathAbs::new(path).map_or_else(|_| path.to_owned(), |p| p.as_path().to_path_buf()),
212 mapping,
213 )
214 } else {
215 Err(Error::UndetectedSyntax("[unknown]".into()))
216 };
218 match path_syntax {
219 // If a path wasn't provided, or if path based syntax detection
220 // above failed, we fall back to first-line syntax detection.
221 Err(Error::UndetectedSyntax(path)) => self
222 .get_first_line_syntax(&mut input.reader)?
223 .ok_or(Error::UndetectedSyntax(path)),
224 _ => path_syntax,
225 }
226 }
228 pub(crate) fn find_syntax_by_name(
229 &self,
230 syntax_name: &str,
231 ) -> Result<Option<SyntaxReferenceInSet>> {
232 let syntax_set = self.get_syntax_set()?;
233 Ok(syntax_set
234 .find_syntax_by_name(syntax_name)
235 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
236 }
238 fn find_syntax_by_extension(&self, e: Option<&OsStr>) -> Result<Option<SyntaxReferenceInSet>> {
239 let syntax_set = self.get_syntax_set()?;
240 let extension = e.and_then(|x| x.to_str()).unwrap_or_default();
241 Ok(syntax_set
242 .find_syntax_by_extension(extension)
243 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
244 }
246 fn get_syntax_for_file_name(
247 &self,
248 file_name: &OsStr,
249 ignored_suffixes: &IgnoredSuffixes,
250 ) -> Result<Option<SyntaxReferenceInSet>> {
251 let mut syntax = self.find_syntax_by_extension(Some(file_name))?;
252 if syntax.is_none() {
253 syntax =
254 ignored_suffixes.try_with_stripped_suffix(file_name, |stripped_file_name| {
255 // Note: recursion
256 self.get_syntax_for_file_name(stripped_file_name, ignored_suffixes)
257 })?;
258 }
259 Ok(syntax)
260 }
262 fn get_syntax_for_file_extension(
263 &self,
264 file_name: &OsStr,
265 ignored_suffixes: &IgnoredSuffixes,
266 ) -> Result<Option<SyntaxReferenceInSet>> {
267 let mut syntax = self.find_syntax_by_extension(Path::new(file_name).extension())?;
268 if syntax.is_none() {
269 syntax =
270 ignored_suffixes.try_with_stripped_suffix(file_name, |stripped_file_name| {
271 // Note: recursion
272 self.get_syntax_for_file_extension(stripped_file_name, ignored_suffixes)
273 })?;
274 }
275 Ok(syntax)
276 }
278 fn get_first_line_syntax(
279 &self,
280 reader: &mut InputReader,
281 ) -> Result<Option<SyntaxReferenceInSet>> {
282 let syntax_set = self.get_syntax_set()?;
283 Ok(String::from_utf8(reader.first_line.clone())
284 .ok()
285 .and_then(|l| syntax_set.find_syntax_by_first_line(&l))
286 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
287 }
290pub(crate) fn get_serialized_integrated_syntaxset() -> &'static [u8] {
291 include_bytes!("../assets/syntaxes.bin")
294pub(crate) fn get_integrated_themeset() -> ThemeSet {
295 from_binary(include_bytes!("../assets/themes.bin"), COMPRESS_THEMES)
298pub(crate) fn from_binary<T: serde::de::DeserializeOwned>(v: &[u8], compressed: bool) -> T {
299 asset_from_contents(v, "n/a", compressed)
300 .expect("data integrated in binary is never faulty, but make sure `compressed` is in sync!")
303fn asset_from_contents<T: serde::de::DeserializeOwned>(
304 contents: &[u8],
305 description: &str,
306 compressed: bool,
307) -> Result<T> {
308 if compressed {
309 bincode::deserialize_from(flate2::read::ZlibDecoder::new(contents))
310 } else {
311 bincode::deserialize_from(contents)
312 }
313 .map_err(|_| format!("Could not parse {}", description).into())
316fn asset_from_cache<T: serde::de::DeserializeOwned>(
317 path: &Path,
318 description: &str,
319 compressed: bool,
320) -> Result<T> {
321 let contents = fs::read(path).map_err(|_| {
322 format!(
323 "Could not load cached {} '{}'",
324 description,
325 path.to_string_lossy()
326 )
327 })?;
328 asset_from_contents(&contents[..], description, compressed)
329 .map_err(|_| format!("Could not parse cached {}", description).into())
332#[cfg(test)]
333mod tests {
334 use super::*;
336 use std::ffi::OsStr;
338 use std::fs::File;
339 use std::io::{BufReader, Write};
340 use tempfile::TempDir;
342 use crate::input::Input;
344 struct SyntaxDetectionTest<'a> {
345 assets: HighlightingAssets,
346 pub syntax_mapping: SyntaxMapping<'a>,
347 pub temp_dir: TempDir,
348 }
350 impl<'a> SyntaxDetectionTest<'a> {
351 fn new() -> Self {
352 SyntaxDetectionTest {
353 assets: HighlightingAssets::from_binary(),
354 syntax_mapping: SyntaxMapping::builtin(),
355 temp_dir: TempDir::new().expect("creation of temporary directory"),
356 }
357 }
359 fn get_syntax_name(
360 &self,
361 language: Option<&str>,
362 input: &mut OpenedInput,
363 mapping: &SyntaxMapping,
364 ) -> String {
365 self.assets
366 .get_syntax(language, input, mapping)
367 .map(|syntax_in_set| syntax_in_set.syntax.name.clone())
368 .unwrap_or_else(|_| "!no syntax!".to_owned())
369 }
371 fn syntax_for_real_file_with_content_os(
372 &self,
373 file_name: &OsStr,
374 first_line: &str,
375 ) -> String {
376 let file_path = self.temp_dir.path().join(file_name);
377 {
378 let mut temp_file = File::create(&file_path).unwrap();
379 writeln!(temp_file, "{}", first_line).unwrap();
380 }
382 let input = Input::ordinary_file(&file_path);
383 let dummy_stdin: &[u8] = &[];
384 let mut opened_input = input.open(dummy_stdin, None).unwrap();
386 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
387 }
389 fn syntax_for_file_with_content_os(&self, file_name: &OsStr, first_line: &str) -> String {
390 let file_path = self.temp_dir.path().join(file_name);
391 let input = Input::from_reader(Box::new(BufReader::new(first_line.as_bytes())))
392 .with_name(Some(&file_path));
393 let dummy_stdin: &[u8] = &[];
394 let mut opened_input = input.open(dummy_stdin, None).unwrap();
396 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
397 }
399 #[cfg(unix)]
400 fn syntax_for_file_os(&self, file_name: &OsStr) -> String {
401 self.syntax_for_file_with_content_os(file_name, "")
402 }
404 fn syntax_for_file_with_content(&self, file_name: &str, first_line: &str) -> String {
405 self.syntax_for_file_with_content_os(OsStr::new(file_name), first_line)
406 }
408 fn syntax_for_file(&self, file_name: &str) -> String {
409 self.syntax_for_file_with_content(file_name, "")
410 }
412 fn syntax_for_stdin_with_content(&self, file_name: &str, content: &[u8]) -> String {
413 let input = Input::stdin().with_name(Some(file_name));
414 let mut opened_input = input.open(content, None).unwrap();
416 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
417 }
419 fn syntax_is_same_for_inputkinds(&self, file_name: &str, content: &str) -> bool {
420 let as_file = self.syntax_for_real_file_with_content_os(file_name.as_ref(), content);
421 let as_reader = self.syntax_for_file_with_content_os(file_name.as_ref(), content);
422 let consistent = as_file == as_reader;
423 // TODO: Compare StdIn somehow?
425 if !consistent {
426 eprintln!(
427 "Inconsistent syntax detection:\nFor File: {}\nFor Reader: {}",
428 as_file, as_reader
429 )
430 }
432 consistent
433 }
434 }
436 #[test]
437 fn syntax_detection_basic() {
438 let test = SyntaxDetectionTest::new();
440 assert_eq!(test.syntax_for_file("test.rs"), "Rust");
441 assert_eq!(test.syntax_for_file("test.cpp"), "C++");
442 assert_eq!(test.syntax_for_file("test.build"), "NAnt Build File");
443 assert_eq!(
444 test.syntax_for_file("PKGBUILD"),
445 "Bourne Again Shell (bash)"
446 );
447 assert_eq!(test.syntax_for_file(".bashrc"), "Bourne Again Shell (bash)");
448 assert_eq!(test.syntax_for_file("Makefile"), "Makefile");
449 }
451 #[cfg(unix)]
452 #[test]
453 fn syntax_detection_invalid_utf8() {
454 use std::os::unix::ffi::OsStrExt;
456 let test = SyntaxDetectionTest::new();
458 assert_eq!(
459 test.syntax_for_file_os(OsStr::from_bytes(b"invalid_\xFEutf8_filename.rs")),
460 "Rust"
461 );
462 }
464 #[test]
465 fn syntax_detection_same_for_inputkinds() {
466 let mut test = SyntaxDetectionTest::new();
468 test.syntax_mapping
469 .insert("*.myext", MappingTarget::MapTo("C"))
470 .ok();
471 test.syntax_mapping
472 .insert("MY_FILE", MappingTarget::MapTo("Markdown"))
473 .ok();
475 assert!(test.syntax_is_same_for_inputkinds("Test.md", ""));
476 assert!(test.syntax_is_same_for_inputkinds("Test.txt", "#!/bin/bash"));
477 assert!(test.syntax_is_same_for_inputkinds(".bashrc", ""));
478 assert!(test.syntax_is_same_for_inputkinds("test.h", ""));
479 assert!(test.syntax_is_same_for_inputkinds("test.js", "#!/bin/bash"));
480 assert!(test.syntax_is_same_for_inputkinds("test.myext", ""));
481 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", ""));
482 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", "<?php"));
483 }
485 #[test]
486 fn syntax_detection_well_defined_mapping_for_duplicate_extensions() {
487 let test = SyntaxDetectionTest::new();
489 assert_eq!(test.syntax_for_file("test.h"), "C++");
490 assert_eq!(test.syntax_for_file("test.sass"), "Sass");
491 assert_eq!(test.syntax_for_file("test.js"), "JavaScript (Babel)");
492 assert_eq!(test.syntax_for_file("test.fs"), "F#");
493 assert_eq!(test.syntax_for_file("test.v"), "Verilog");
494 }
496 #[test]
497 fn syntax_detection_first_line() {
498 let test = SyntaxDetectionTest::new();
500 assert_eq!(
501 test.syntax_for_file_with_content("my_script", "#!/bin/bash"),
502 "Bourne Again Shell (bash)"
503 );
504 assert_eq!(
505 test.syntax_for_file_with_content("build", "#!/bin/bash"),
506 "Bourne Again Shell (bash)"
507 );
508 assert_eq!(
509 test.syntax_for_file_with_content("my_script", "<?php"),
510 "PHP"
511 );
512 }
514 #[test]
515 fn syntax_detection_with_custom_mapping() {
516 let mut test = SyntaxDetectionTest::new();
518 assert_eq!(test.syntax_for_file("test.h"), "C++");
519 test.syntax_mapping
520 .insert("*.h", MappingTarget::MapTo("C"))
521 .ok();
522 assert_eq!(test.syntax_for_file("test.h"), "C");
523 }
525 #[test]
526 fn syntax_detection_with_extension_mapping_to_unknown() {
527 let mut test = SyntaxDetectionTest::new();
529 // Normally, a CMakeLists.txt file shall use the CMake syntax, even if it is
530 // a bash script in disguise
531 assert_eq!(
532 test.syntax_for_file_with_content("CMakeLists.txt", "#!/bin/bash"),
533 "CMake"
534 );
536 // Other .txt files shall use the Plain Text syntax
537 assert_eq!(
538 test.syntax_for_file_with_content("some-other.txt", "#!/bin/bash"),
539 "Plain Text"
540 );
542 // If we setup MapExtensionToUnknown on *.txt, the match on the full
543 // file name of "CMakeLists.txt" shall have higher prio, and CMake shall
544 // still be used for it
545 test.syntax_mapping
546 .insert("*.txt", MappingTarget::MapExtensionToUnknown)
547 .ok();
548 assert_eq!(
549 test.syntax_for_file_with_content("CMakeLists.txt", "#!/bin/bash"),
550 "CMake"
551 );
553 // However, for *other* files with a .txt extension, first-line fallback
554 // shall now be used
555 assert_eq!(
556 test.syntax_for_file_with_content("some-other.txt", "#!/bin/bash"),
557 "Bourne Again Shell (bash)"
558 );
559 }
561 #[test]
562 fn syntax_detection_is_case_sensitive() {
563 let mut test = SyntaxDetectionTest::new();
565 assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
566 test.syntax_mapping
567 .insert("*.MD", MappingTarget::MapTo("Markdown"))
568 .ok();
569 assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
570 }
572 #[test]
573 fn syntax_detection_stdin_filename() {
574 let test = SyntaxDetectionTest::new();
576 // from file extension
577 assert_eq!(test.syntax_for_stdin_with_content("test.cpp", b"a"), "C++");
578 // from first line (fallback)
579 assert_eq!(
580 test.syntax_for_stdin_with_content("my_script", b"#!/bin/bash"),
581 "Bourne Again Shell (bash)"
582 );
583 }
585 #[cfg(unix)]
586 #[test]
587 fn syntax_detection_for_symlinked_file() {
588 use std::os::unix::fs::symlink;
590 let test = SyntaxDetectionTest::new();
591 let file_path = test.temp_dir.path().join("my_ssh_config_filename");
592 {
593 File::create(&file_path).unwrap();
594 }
595 let file_path_symlink = test.temp_dir.path().join(".ssh").join("config");
597 std::fs::create_dir(test.temp_dir.path().join(".ssh"))
598 .expect("creation of directory succeeds");
599 symlink(&file_path, &file_path_symlink).expect("creation of symbolic link succeeds");
601 let input = Input::ordinary_file(&file_path_symlink);
602 let dummy_stdin: &[u8] = &[];
603 let mut opened_input = input.open(dummy_stdin, None).unwrap();
605 assert_eq!(
606 test.get_syntax_name(None, &mut opened_input, &test.syntax_mapping),
607 "SSH Config"
608 );
609 }