git.iko.soy/bat
on 7c41bd7 help sign in
bat/src/assets.rs
569 lines 20 kB history raw
1use std::ffi::OsStr;
2use std::fs;
3use std::path::{Path, PathBuf};
5use lazycell::LazyCell;
7use syntect::dumps::{from_binary, from_reader};
8use syntect::highlighting::{Theme, ThemeSet};
9use syntect::parsing::{SyntaxReference, SyntaxSet};
11use path_abs::PathAbs;
13use crate::bat_warning;
14use crate::error::*;
15use crate::input::{InputReader, OpenedInput, OpenedInputKind};
16use crate::syntax_mapping::{MappingTarget, SyntaxMapping};
18#[derive(Debug)]
19pub struct HighlightingAssets {
20 syntax_set_cell: LazyCell<SyntaxSet>,
21 serialized_syntax_set: SerializedSyntaxSet,
22 theme_set: ThemeSet,
23 fallback_theme: Option<&'static str>,
26#[derive(Debug)]
27pub struct SyntaxReferenceInSet<'a> {
28 pub syntax: &'a SyntaxReference,
29 pub syntax_set: &'a SyntaxSet,
32const IGNORED_SUFFIXES: [&str; 13] = [
33 // Editor etc backups
34 "~",
35 ".bak",
36 ".old",
37 ".orig",
38 // Debian and derivatives apt/dpkg/ucf backups
39 ".dpkg-dist",
40 ".dpkg-old",
41 ".ucf-dist",
42 ".ucf-new",
43 ".ucf-old",
44 // Red Hat and derivatives rpm backups
45 ".rpmnew",
46 ".rpmorig",
47 ".rpmsave",
48 // Build system input/template files
49 ".in",
50];
52impl HighlightingAssets {
53 fn new(serialized_syntax_set: SerializedSyntaxSet, theme_set: ThemeSet) -> Self {
54 HighlightingAssets {
55 syntax_set_cell: LazyCell::new(),
56 serialized_syntax_set,
57 theme_set,
58 fallback_theme: None,
59 }
60 }
62 pub fn default_theme() -> &'static str {
63 "Monokai Extended"
64 }
66 pub fn from_cache(cache_path: &Path) -> Result<Self> {
67 Ok(HighlightingAssets::new(
68 SerializedSyntaxSet::FromFile(cache_path.join("syntaxes.bin")),
69 asset_from_cache(&cache_path.join("themes.bin"), "theme set")?,
70 ))
71 }
73 pub fn from_binary() -> Self {
74 HighlightingAssets::new(
75 SerializedSyntaxSet::FromBinary(get_serialized_integrated_syntaxset()),
76 get_integrated_themeset(),
77 )
78 }
80 pub fn set_fallback_theme(&mut self, theme: &'static str) {
81 self.fallback_theme = Some(theme);
82 }
84 pub(crate) fn get_syntax_set(&self) -> Result<&SyntaxSet> {
85 self.syntax_set_cell
86 .try_borrow_with(|| self.serialized_syntax_set.deserialize())
87 }
89 /// Use [Self::get_syntaxes] instead
90 #[deprecated]
91 pub fn syntaxes(&self) -> &[SyntaxReference] {
92 self.get_syntax_set()
93 .expect(".syntaxes() is deprecated, use .get_syntaxes() instead")
94 .syntaxes()
95 }
97 pub fn get_syntaxes(&self) -> Result<&[SyntaxReference]> {
98 Ok(self.get_syntax_set()?.syntaxes())
99 }
101 fn get_theme_set(&self) -> &ThemeSet {
102 &self.theme_set
103 }
105 pub fn themes(&self) -> impl Iterator<Item = &str> {
106 self.get_theme_set().themes.keys().map(|s| s.as_ref())
107 }
109 /// Use [Self::get_syntax_for_file_name] instead
110 #[deprecated]
111 pub fn syntax_for_file_name(
112 &self,
113 file_name: impl AsRef<Path>,
114 mapping: &SyntaxMapping,
115 ) -> Option<&SyntaxReference> {
116 self.get_syntax_for_file_name(file_name, mapping)
117 .expect(
118 ".syntax_for_file_name() is deprecated, use .get_syntax_for_file_name() instead",
119 )
120 .map(|syntax_in_set| syntax_in_set.syntax)
121 }
123 pub fn get_syntax_for_file_name(
124 &self,
125 file_name: impl AsRef<Path>,
126 mapping: &SyntaxMapping,
127 ) -> Result<Option<SyntaxReferenceInSet>> {
128 let file_name = file_name.as_ref();
129 Ok(match mapping.get_syntax_for(file_name) {
130 Some(MappingTarget::MapToUnknown) => None,
131 Some(MappingTarget::MapTo(syntax_name)) => {
132 let syntax_set = self.get_syntax_set()?;
133 syntax_set
134 .find_syntax_by_name(syntax_name)
135 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
136 }
137 None => self.get_extension_syntax(file_name.as_os_str())?,
138 })
139 }
141 pub(crate) fn get_theme(&self, theme: &str) -> &Theme {
142 match self.get_theme_set().themes.get(theme) {
143 Some(theme) => theme,
144 None => {
145 if theme == "ansi-light" || theme == "ansi-dark" {
146 bat_warning!("Theme '{}' is deprecated, using 'ansi' instead.", theme);
147 return self.get_theme("ansi");
148 }
149 if !theme.is_empty() {
150 bat_warning!("Unknown theme '{}', using default.", theme)
151 }
152 &self.get_theme_set().themes
153 [self.fallback_theme.unwrap_or_else(|| Self::default_theme())]
154 }
155 }
156 }
158 pub(crate) fn get_syntax(
159 &self,
160 language: Option<&str>,
161 input: &mut OpenedInput,
162 mapping: &SyntaxMapping,
163 ) -> Result<SyntaxReferenceInSet> {
164 if let Some(language) = language {
165 let syntax_set = self.get_syntax_set()?;
166 syntax_set
167 .find_syntax_by_token(language)
168 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
169 .ok_or_else(|| Error::UnknownSyntax(language.to_owned()))
170 } else {
171 let line_syntax = self.get_first_line_syntax(&mut input.reader)?;
173 // Get the path of the file:
174 // If this was set by the metadata, that will take priority.
175 // If it wasn't, it will use the real file path (if available).
176 let path_str =
177 input
178 .metadata
179 .user_provided_name
180 .as_ref()
181 .or_else(|| match input.kind {
182 OpenedInputKind::OrdinaryFile(ref path) => Some(path),
183 _ => None,
184 });
186 if let Some(path_str) = path_str {
187 // If a path was provided, we try and detect the syntax based on extension mappings.
188 let path = Path::new(path_str);
189 let absolute_path = PathAbs::new(path)
190 .ok()
191 .map(|p| p.as_path().to_path_buf())
192 .unwrap_or_else(|| path.to_owned());
194 match mapping.get_syntax_for(absolute_path) {
195 Some(MappingTarget::MapToUnknown) => line_syntax
196 .ok_or_else(|| Error::UndetectedSyntax(path.to_string_lossy().into())),
198 Some(MappingTarget::MapTo(syntax_name)) => {
199 let syntax_set = self.get_syntax_set()?;
200 syntax_set
201 .find_syntax_by_name(syntax_name)
202 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
203 .ok_or_else(|| Error::UnknownSyntax(syntax_name.to_owned()))
204 }
206 None => {
207 let file_name = path.file_name().unwrap_or_default();
208 self.get_extension_syntax(file_name)?
209 .or(line_syntax)
210 .ok_or_else(|| Error::UndetectedSyntax(path.to_string_lossy().into()))
211 }
212 }
213 } else {
214 // If a path wasn't provided, we fall back to the detect first-line syntax.
215 line_syntax.ok_or_else(|| Error::UndetectedSyntax("[unknown]".into()))
216 }
217 }
218 }
220 fn get_extension_syntax(&self, file_name: &OsStr) -> Result<Option<SyntaxReferenceInSet>> {
221 let mut syntax = self.find_syntax_by_file_name(file_name)?;
222 if syntax.is_none() {
223 syntax = self.find_syntax_by_file_name_extension(file_name)?;
224 }
225 if syntax.is_none() {
226 syntax = self.get_extension_syntax_with_stripped_suffix(file_name)?;
227 }
228 Ok(syntax)
229 }
231 fn find_syntax_by_file_name(&self, file_name: &OsStr) -> Result<Option<SyntaxReferenceInSet>> {
232 let syntax_set = self.get_syntax_set()?;
233 Ok(syntax_set
234 .find_syntax_by_extension(file_name.to_str().unwrap_or_default())
235 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
236 }
238 fn find_syntax_by_file_name_extension(
239 &self,
240 file_name: &OsStr,
241 ) -> Result<Option<SyntaxReferenceInSet>> {
242 let file_path = Path::new(file_name);
243 let syntax_set = self.get_syntax_set()?;
244 Ok(syntax_set
245 .find_syntax_by_extension(
246 file_path
247 .extension()
248 .and_then(|x| x.to_str())
249 .unwrap_or_default(),
250 )
251 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
252 }
254 /// If we find an ignored suffix on the file name, e.g. '~', we strip it and
255 /// then try again to find a syntax without it. Note that we do this recursively.
256 fn get_extension_syntax_with_stripped_suffix(
257 &self,
258 file_name: &OsStr,
259 ) -> Result<Option<SyntaxReferenceInSet>> {
260 let file_path = Path::new(file_name);
261 let mut syntax = None;
262 if let Some(file_str) = file_path.to_str() {
263 for suffix in IGNORED_SUFFIXES.iter() {
264 if let Some(stripped_filename) = file_str.strip_suffix(suffix) {
265 syntax = self.get_extension_syntax(OsStr::new(stripped_filename))?;
266 break;
267 }
268 }
269 }
270 Ok(syntax)
271 }
273 fn get_first_line_syntax(
274 &self,
275 reader: &mut InputReader,
276 ) -> Result<Option<SyntaxReferenceInSet>> {
277 let syntax_set = self.get_syntax_set()?;
278 Ok(String::from_utf8(reader.first_line.clone())
279 .ok()
280 .and_then(|l| syntax_set.find_syntax_by_first_line(&l))
281 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
282 }
285#[cfg(feature = "build-assets")]
286pub use crate::build_assets::build_assets as build;
288/// A SyntaxSet in serialized form, i.e. bincoded and flate2 compressed.
289/// We keep it in this format since we want to load it lazily.
290#[derive(Debug)]
291enum SerializedSyntaxSet {
292 /// The data comes from a user-generated cache file.
293 FromFile(PathBuf),
295 /// The data to use is embedded into the bat binary.
296 FromBinary(&'static [u8]),
299impl SerializedSyntaxSet {
300 fn deserialize(&self) -> Result<SyntaxSet> {
301 match self {
302 SerializedSyntaxSet::FromBinary(data) => Ok(from_binary(data)),
303 SerializedSyntaxSet::FromFile(ref path) => asset_from_cache(path, "syntax set"),
304 }
305 }
308pub(crate) fn get_serialized_integrated_syntaxset() -> &'static [u8] {
309 include_bytes!("../assets/syntaxes.bin")
312pub(crate) fn get_integrated_themeset() -> ThemeSet {
313 from_binary(include_bytes!("../assets/themes.bin"))
316fn asset_from_cache<T: serde::de::DeserializeOwned>(path: &Path, description: &str) -> Result<T> {
317 let contents = fs::read(path).map_err(|_| {
318 format!(
319 "Could not load cached {} '{}'",
320 description,
321 path.to_string_lossy()
322 )
323 })?;
324 from_reader(&contents[..]).map_err(|_| format!("Could not parse cached {}", description).into())
327#[cfg(test)]
328mod tests {
329 use super::*;
331 use std::ffi::OsStr;
333 use std::fs::File;
334 use std::io::{BufReader, Write};
335 use tempfile::TempDir;
337 use crate::input::Input;
339 struct SyntaxDetectionTest<'a> {
340 assets: HighlightingAssets,
341 pub syntax_mapping: SyntaxMapping<'a>,
342 pub temp_dir: TempDir,
343 }
345 impl<'a> SyntaxDetectionTest<'a> {
346 fn new() -> Self {
347 SyntaxDetectionTest {
348 assets: HighlightingAssets::from_binary(),
349 syntax_mapping: SyntaxMapping::builtin(),
350 temp_dir: TempDir::new().expect("creation of temporary directory"),
351 }
352 }
354 fn get_syntax_name(
355 &self,
356 language: Option<&str>,
357 input: &mut OpenedInput,
358 mapping: &SyntaxMapping,
359 ) -> String {
360 self.assets
361 .get_syntax(language, input, mapping)
362 .map(|syntax_in_set| syntax_in_set.syntax.name.clone())
363 .unwrap_or_else(|_| "!no syntax!".to_owned())
364 }
366 fn syntax_for_real_file_with_content_os(
367 &self,
368 file_name: &OsStr,
369 first_line: &str,
370 ) -> String {
371 let file_path = self.temp_dir.path().join(file_name);
372 {
373 let mut temp_file = File::create(&file_path).unwrap();
374 writeln!(temp_file, "{}", first_line).unwrap();
375 }
377 let input = Input::ordinary_file(&file_path);
378 let dummy_stdin: &[u8] = &[];
379 let mut opened_input = input.open(dummy_stdin, None).unwrap();
381 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
382 }
384 fn syntax_for_file_with_content_os(&self, file_name: &OsStr, first_line: &str) -> String {
385 let file_path = self.temp_dir.path().join(file_name);
386 let input = Input::from_reader(Box::new(BufReader::new(first_line.as_bytes())))
387 .with_name(Some(&file_path));
388 let dummy_stdin: &[u8] = &[];
389 let mut opened_input = input.open(dummy_stdin, None).unwrap();
391 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
392 }
394 #[cfg(unix)]
395 fn syntax_for_file_os(&self, file_name: &OsStr) -> String {
396 self.syntax_for_file_with_content_os(file_name, "")
397 }
399 fn syntax_for_file_with_content(&self, file_name: &str, first_line: &str) -> String {
400 self.syntax_for_file_with_content_os(OsStr::new(file_name), first_line)
401 }
403 fn syntax_for_file(&self, file_name: &str) -> String {
404 self.syntax_for_file_with_content(file_name, "")
405 }
407 fn syntax_for_stdin_with_content(&self, file_name: &str, content: &[u8]) -> String {
408 let input = Input::stdin().with_name(Some(file_name));
409 let mut opened_input = input.open(content, None).unwrap();
411 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
412 }
414 fn syntax_is_same_for_inputkinds(&self, file_name: &str, content: &str) -> bool {
415 let as_file = self.syntax_for_real_file_with_content_os(file_name.as_ref(), content);
416 let as_reader = self.syntax_for_file_with_content_os(file_name.as_ref(), content);
417 let consistent = as_file == as_reader;
418 // TODO: Compare StdIn somehow?
420 if !consistent {
421 eprintln!(
422 "Inconsistent syntax detection:\nFor File: {}\nFor Reader: {}",
423 as_file, as_reader
424 )
425 }
427 consistent
428 }
429 }
431 #[test]
432 fn syntax_detection_basic() {
433 let test = SyntaxDetectionTest::new();
435 assert_eq!(test.syntax_for_file("test.rs"), "Rust");
436 assert_eq!(test.syntax_for_file("test.cpp"), "C++");
437 assert_eq!(test.syntax_for_file("test.build"), "NAnt Build File");
438 assert_eq!(
439 test.syntax_for_file("PKGBUILD"),
440 "Bourne Again Shell (bash)"
441 );
442 assert_eq!(test.syntax_for_file(".bashrc"), "Bourne Again Shell (bash)");
443 assert_eq!(test.syntax_for_file("Makefile"), "Makefile");
444 }
446 #[cfg(unix)]
447 #[test]
448 fn syntax_detection_invalid_utf8() {
449 use std::os::unix::ffi::OsStrExt;
451 let test = SyntaxDetectionTest::new();
453 assert_eq!(
454 test.syntax_for_file_os(OsStr::from_bytes(b"invalid_\xFEutf8_filename.rs")),
455 "Rust"
456 );
457 }
459 #[test]
460 fn syntax_detection_same_for_inputkinds() {
461 let mut test = SyntaxDetectionTest::new();
463 test.syntax_mapping
464 .insert("*.myext", MappingTarget::MapTo("C"))
465 .ok();
466 test.syntax_mapping
467 .insert("MY_FILE", MappingTarget::MapTo("Markdown"))
468 .ok();
470 assert!(test.syntax_is_same_for_inputkinds("Test.md", ""));
471 assert!(test.syntax_is_same_for_inputkinds("Test.txt", "#!/bin/bash"));
472 assert!(test.syntax_is_same_for_inputkinds(".bashrc", ""));
473 assert!(test.syntax_is_same_for_inputkinds("test.h", ""));
474 assert!(test.syntax_is_same_for_inputkinds("test.js", "#!/bin/bash"));
475 assert!(test.syntax_is_same_for_inputkinds("test.myext", ""));
476 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", ""));
477 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", "<?php"));
478 }
480 #[test]
481 fn syntax_detection_well_defined_mapping_for_duplicate_extensions() {
482 let test = SyntaxDetectionTest::new();
484 assert_eq!(test.syntax_for_file("test.h"), "C++");
485 assert_eq!(test.syntax_for_file("test.sass"), "Sass");
486 assert_eq!(test.syntax_for_file("test.js"), "JavaScript (Babel)");
487 assert_eq!(test.syntax_for_file("test.fs"), "F#");
488 assert_eq!(test.syntax_for_file("test.v"), "Verilog");
489 }
491 #[test]
492 fn syntax_detection_first_line() {
493 let test = SyntaxDetectionTest::new();
495 assert_eq!(
496 test.syntax_for_file_with_content("my_script", "#!/bin/bash"),
497 "Bourne Again Shell (bash)"
498 );
499 assert_eq!(
500 test.syntax_for_file_with_content("build", "#!/bin/bash"),
501 "Bourne Again Shell (bash)"
502 );
503 assert_eq!(
504 test.syntax_for_file_with_content("my_script", "<?php"),
505 "PHP"
506 );
507 }
509 #[test]
510 fn syntax_detection_with_custom_mapping() {
511 let mut test = SyntaxDetectionTest::new();
513 assert_eq!(test.syntax_for_file("test.h"), "C++");
514 test.syntax_mapping
515 .insert("*.h", MappingTarget::MapTo("C"))
516 .ok();
517 assert_eq!(test.syntax_for_file("test.h"), "C");
518 }
520 #[test]
521 fn syntax_detection_is_case_sensitive() {
522 let mut test = SyntaxDetectionTest::new();
524 assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
525 test.syntax_mapping
526 .insert("*.MD", MappingTarget::MapTo("Markdown"))
527 .ok();
528 assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
529 }
531 #[test]
532 fn syntax_detection_stdin_filename() {
533 let test = SyntaxDetectionTest::new();
535 // from file extension
536 assert_eq!(test.syntax_for_stdin_with_content("test.cpp", b"a"), "C++");
537 // from first line (fallback)
538 assert_eq!(
539 test.syntax_for_stdin_with_content("my_script", b"#!/bin/bash"),
540 "Bourne Again Shell (bash)"
541 );
542 }
544 #[cfg(unix)]
545 #[test]
546 fn syntax_detection_for_symlinked_file() {
547 use std::os::unix::fs::symlink;
549 let test = SyntaxDetectionTest::new();
550 let file_path = test.temp_dir.path().join("my_ssh_config_filename");
551 {
552 File::create(&file_path).unwrap();
553 }
554 let file_path_symlink = test.temp_dir.path().join(".ssh").join("config");
556 std::fs::create_dir(test.temp_dir.path().join(".ssh"))
557 .expect("creation of directory succeeds");
558 symlink(&file_path, &file_path_symlink).expect("creation of symbolic link succeeds");
560 let input = Input::ordinary_file(&file_path_symlink);
561 let dummy_stdin: &[u8] = &[];
562 let mut opened_input = input.open(dummy_stdin, None).unwrap();
564 assert_eq!(
565 test.get_syntax_name(None, &mut opened_input, &test.syntax_mapping),
566 "SSH Config"
567 );
568 }