3use std::path::{Path, PathBuf};
7use syntect::dumps::{from_binary, from_reader};
8use syntect::highlighting::{Theme, ThemeSet};
9use syntect::parsing::{SyntaxReference, SyntaxSet};
13use crate::bat_warning;
15use crate::input::{InputReader, OpenedInput, OpenedInputKind};
16use crate::syntax_mapping::{MappingTarget, SyntaxMapping};
19pub struct HighlightingAssets {
20 syntax_set_cell: LazyCell<SyntaxSet>,
21 serialized_syntax_set: Option<SerializedSyntaxSet>,
23 fallback_theme: Option<&'static str>,
27pub struct SyntaxReferenceInSet<'a> {
28 pub syntax: &'a SyntaxReference,
29 pub syntax_set: &'a SyntaxSet,
32const IGNORED_SUFFIXES: [&str; 10] = [
38 // Debian and derivatives apt/dpkg backups
41 // Red Hat and derivatives rpm backups
45 // Build system input/template files
49impl HighlightingAssets {
51 syntax_set: Option<SyntaxSet>,
52 serialized_syntax_set: Option<SerializedSyntaxSet>,
55 assert!(syntax_set.is_some() || serialized_syntax_set.is_some());
57 let syntax_set_cell = LazyCell::new();
58 if let Some(syntax_set) = syntax_set {
59 syntax_set_cell.fill(syntax_set).expect("can never fail");
64 serialized_syntax_set,
70 pub fn default_theme() -> &'static str {
74 #[cfg(feature = "build-assets")]
75 pub fn from_files(source_dir: &Path, include_integrated_assets: bool) -> Result<Self> {
76 let mut theme_set = if include_integrated_assets {
77 get_integrated_themeset()
82 let theme_dir = source_dir.join("themes");
83 if theme_dir.exists() {
84 let res = theme_set.add_from_folder(&theme_dir);
85 if let Err(err) = res {
87 "Failed to load one or more themes from '{}' (reason: '{}')",
88 theme_dir.to_string_lossy(),
94 "No themes were found in '{}', using the default set",
95 theme_dir.to_string_lossy()
99 let mut syntax_set_builder = if !include_integrated_assets {
100 let mut builder = syntect::parsing::SyntaxSetBuilder::new();
101 builder.add_plain_text_syntax();
104 from_binary::<SyntaxSet>(get_serialized_integrated_syntaxset()).into_builder()
107 let syntax_dir = source_dir.join("syntaxes");
108 if syntax_dir.exists() {
109 syntax_set_builder.add_from_folder(syntax_dir, true)?;
112 "No syntaxes were found in '{}', using the default set.",
113 syntax_dir.to_string_lossy()
117 if std::env::var("BAT_PRINT_SYNTAX_DEPENDENCIES").is_ok() {
118 // To trigger this code, run:
119 // BAT_PRINT_SYNTAX_DEPENDENCIES=1 cargo run -- cache --build --source assets --blank --target /tmp
120 crate::syntax_dependencies::print_syntax_dependencies(&syntax_set_builder);
123 let syntax_set = syntax_set_builder.build();
124 let missing_contexts = syntax_set.find_unlinked_contexts();
125 if !missing_contexts.is_empty() {
126 println!("Some referenced contexts could not be found!");
127 for context in missing_contexts {
128 println!("- {}", context);
132 Ok(HighlightingAssets::new(Some(syntax_set), None, theme_set))
135 pub fn from_cache(cache_path: &Path) -> Result<Self> {
136 Ok(HighlightingAssets::new(
138 Some(SerializedSyntaxSet::FromFile(
139 cache_path.join("syntaxes.bin"),
141 asset_from_cache(&cache_path.join("themes.bin"), "theme set")?,
145 pub fn from_binary() -> Self {
146 HighlightingAssets::new(
148 Some(SerializedSyntaxSet::FromBinary(
149 get_serialized_integrated_syntaxset(),
151 get_integrated_themeset(),
155 #[cfg(feature = "build-assets")]
156 pub fn save_to_cache(&self, target_dir: &Path, current_version: &str) -> Result<()> {
157 let _ = fs::create_dir_all(target_dir);
159 self.get_theme_set(),
160 &target_dir.join("themes.bin"),
164 self.get_syntax_set()?,
165 &target_dir.join("syntaxes.bin"),
170 "Writing metadata to folder {} ... ",
171 target_dir.to_string_lossy()
173 crate::assets_metadata::AssetsMetadata::new(current_version).save_to_folder(target_dir)?;
179 pub fn set_fallback_theme(&mut self, theme: &'static str) {
180 self.fallback_theme = Some(theme);
183 pub(crate) fn get_syntax_set(&self) -> Result<&SyntaxSet> {
184 if !self.syntax_set_cell.filled() {
185 self.syntax_set_cell.fill(
186 self.serialized_syntax_set
188 .expect("a dev forgot to setup serialized_syntax_set, please report to https://github.com/sharkdp/bat/issues")
193 // It is safe to .unwrap() because we just made sure it was .filled()
194 Ok(self.syntax_set_cell.borrow().unwrap())
197 /// Use [Self::get_syntaxes] instead
199 pub fn syntaxes(&self) -> &[SyntaxReference] {
200 self.get_syntax_set()
201 .expect(".syntaxes() is deprecated, use .get_syntaxes() instead")
205 pub fn get_syntaxes(&self) -> Result<&[SyntaxReference]> {
206 Ok(self.get_syntax_set()?.syntaxes())
209 fn get_theme_set(&self) -> &ThemeSet {
213 pub fn themes(&self) -> impl Iterator<Item = &str> {
214 self.get_theme_set().themes.keys().map(|s| s.as_ref())
217 /// Use [Self::get_syntax_for_file_name] instead
219 pub fn syntax_for_file_name(
221 file_name: impl AsRef<Path>,
222 mapping: &SyntaxMapping,
223 ) -> Option<&SyntaxReference> {
224 self.get_syntax_for_file_name(file_name, mapping)
226 ".syntax_for_file_name() is deprecated, use .get_syntax_for_file_name() instead",
228 .map(|syntax_in_set| syntax_in_set.syntax)
231 pub fn get_syntax_for_file_name(
233 file_name: impl AsRef<Path>,
234 mapping: &SyntaxMapping,
235 ) -> Result<Option<SyntaxReferenceInSet>> {
236 let file_name = file_name.as_ref();
237 Ok(match mapping.get_syntax_for(file_name) {
238 Some(MappingTarget::MapToUnknown) => None,
239 Some(MappingTarget::MapTo(syntax_name)) => {
240 let syntax_set = self.get_syntax_set()?;
242 .find_syntax_by_name(syntax_name)
243 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
245 None => self.get_extension_syntax(file_name.as_os_str())?,
249 pub(crate) fn get_theme(&self, theme: &str) -> &Theme {
250 match self.get_theme_set().themes.get(theme) {
251 Some(theme) => theme,
253 if theme == "ansi-light" || theme == "ansi-dark" {
254 bat_warning!("Theme '{}' is deprecated, using 'ansi' instead.", theme);
255 return self.get_theme("ansi");
257 if !theme.is_empty() {
258 bat_warning!("Unknown theme '{}', using default.", theme)
260 &self.get_theme_set().themes
261 [self.fallback_theme.unwrap_or_else(|| Self::default_theme())]
266 pub(crate) fn get_syntax(
268 language: Option<&str>,
269 input: &mut OpenedInput,
270 mapping: &SyntaxMapping,
271 ) -> Result<SyntaxReferenceInSet> {
272 if let Some(language) = language {
273 let syntax_set = self.get_syntax_set()?;
275 .find_syntax_by_token(language)
276 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
277 .ok_or_else(|| ErrorKind::UnknownSyntax(language.to_owned()).into())
279 let line_syntax = self.get_first_line_syntax(&mut input.reader)?;
281 // Get the path of the file:
282 // If this was set by the metadata, that will take priority.
283 // If it wasn't, it will use the real file path (if available).
289 .or_else(|| match input.kind {
290 OpenedInputKind::OrdinaryFile(ref path) => Some(path),
294 if let Some(path_str) = path_str {
295 // If a path was provided, we try and detect the syntax based on extension mappings.
296 let path = Path::new(path_str);
297 let absolute_path = PathAbs::new(path)
299 .map(|p| p.as_path().to_path_buf())
300 .unwrap_or_else(|| path.to_owned());
302 match mapping.get_syntax_for(absolute_path) {
303 Some(MappingTarget::MapToUnknown) => line_syntax.ok_or_else(|| {
304 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
307 Some(MappingTarget::MapTo(syntax_name)) => {
308 let syntax_set = self.get_syntax_set()?;
310 .find_syntax_by_name(syntax_name)
311 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set })
312 .ok_or_else(|| ErrorKind::UnknownSyntax(syntax_name.to_owned()).into())
316 let file_name = path.file_name().unwrap_or_default();
317 self.get_extension_syntax(file_name)?
320 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
325 // If a path wasn't provided, we fall back to the detect first-line syntax.
326 line_syntax.ok_or_else(|| ErrorKind::UndetectedSyntax("[unknown]".into()).into())
331 fn get_extension_syntax(&self, file_name: &OsStr) -> Result<Option<SyntaxReferenceInSet>> {
332 let mut syntax = self.find_syntax_by_file_name(file_name)?;
333 if syntax.is_none() {
334 syntax = self.find_syntax_by_file_name_extension(file_name)?;
336 if syntax.is_none() {
337 syntax = self.get_extension_syntax_with_stripped_suffix(file_name)?;
342 fn find_syntax_by_file_name(&self, file_name: &OsStr) -> Result<Option<SyntaxReferenceInSet>> {
343 let syntax_set = self.get_syntax_set()?;
345 .find_syntax_by_extension(file_name.to_str().unwrap_or_default())
346 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
349 fn find_syntax_by_file_name_extension(
352 ) -> Result<Option<SyntaxReferenceInSet>> {
353 let file_path = Path::new(file_name);
354 let syntax_set = self.get_syntax_set()?;
356 .find_syntax_by_extension(
359 .and_then(|x| x.to_str())
360 .unwrap_or_default(),
362 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
365 /// If we find an ignored suffix on the file name, e.g. '~', we strip it and
366 /// then try again to find a syntax without it. Note that we do this recursively.
367 fn get_extension_syntax_with_stripped_suffix(
370 ) -> Result<Option<SyntaxReferenceInSet>> {
371 let file_path = Path::new(file_name);
372 let mut syntax = None;
373 if let Some(file_str) = file_path.to_str() {
374 for suffix in IGNORED_SUFFIXES.iter() {
375 if let Some(stripped_filename) = file_str.strip_suffix(suffix) {
376 syntax = self.get_extension_syntax(OsStr::new(stripped_filename))?;
384 fn get_first_line_syntax(
386 reader: &mut InputReader,
387 ) -> Result<Option<SyntaxReferenceInSet>> {
388 let syntax_set = self.get_syntax_set()?;
389 Ok(String::from_utf8(reader.first_line.clone())
391 .and_then(|l| syntax_set.find_syntax_by_first_line(&l))
392 .map(|syntax| SyntaxReferenceInSet { syntax, syntax_set }))
396/// A SyntaxSet in serialized form, i.e. bincoded and flate2 compressed.
397/// We keep it in this format since we want to load it lazily.
399enum SerializedSyntaxSet {
400 /// The data comes from a user-generated cache file.
403 /// The data to use is embedded into the bat binary.
404 FromBinary(&'static [u8]),
407impl SerializedSyntaxSet {
408 fn deserialize(&self) -> Result<SyntaxSet> {
410 SerializedSyntaxSet::FromBinary(data) => Ok(from_binary(data)),
411 SerializedSyntaxSet::FromFile(ref path) => asset_from_cache(&path, "syntax set"),
416fn get_serialized_integrated_syntaxset() -> &'static [u8] {
417 include_bytes!("../assets/syntaxes.bin")
420fn get_integrated_themeset() -> ThemeSet {
421 from_binary(include_bytes!("../assets/themes.bin"))
424#[cfg(feature = "build-assets")]
425fn asset_to_cache<T: serde::Serialize>(asset: &T, path: &Path, description: &str) -> Result<()> {
426 print!("Writing {} to {} ... ", description, path.to_string_lossy());
427 syntect::dumps::dump_to_file(asset, &path).chain_err(|| {
429 "Could not save {} to {}",
431 path.to_string_lossy()
438fn asset_from_cache<T: serde::de::DeserializeOwned>(path: &Path, description: &str) -> Result<T> {
439 let contents = fs::read(path).chain_err(|| {
441 "Could not load cached {} '{}'",
443 path.to_string_lossy()
446 from_reader(&contents[..]).chain_err(|| format!("Could not parse cached {}", description))
456 use std::io::{BufReader, Write};
457 use tempfile::TempDir;
459 use crate::input::Input;
461 struct SyntaxDetectionTest<'a> {
462 assets: HighlightingAssets,
463 pub syntax_mapping: SyntaxMapping<'a>,
464 pub temp_dir: TempDir,
467 impl<'a> SyntaxDetectionTest<'a> {
469 SyntaxDetectionTest {
470 assets: HighlightingAssets::from_binary(),
471 syntax_mapping: SyntaxMapping::builtin(),
472 temp_dir: TempDir::new().expect("creation of temporary directory"),
478 language: Option<&str>,
479 input: &mut OpenedInput,
480 mapping: &SyntaxMapping,
483 .get_syntax(language, input, mapping)
484 .map(|syntax_in_set| syntax_in_set.syntax.name.clone())
485 .unwrap_or_else(|_| "!no syntax!".to_owned())
488 fn syntax_for_real_file_with_content_os(
493 let file_path = self.temp_dir.path().join(file_name);
495 let mut temp_file = File::create(&file_path).unwrap();
496 writeln!(temp_file, "{}", first_line).unwrap();
499 let input = Input::ordinary_file(&file_path);
500 let dummy_stdin: &[u8] = &[];
501 let mut opened_input = input.open(dummy_stdin, None).unwrap();
503 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
506 fn syntax_for_file_with_content_os(&self, file_name: &OsStr, first_line: &str) -> String {
507 let file_path = self.temp_dir.path().join(file_name);
508 let input = Input::from_reader(Box::new(BufReader::new(first_line.as_bytes())))
509 .with_name(Some(&file_path));
510 let dummy_stdin: &[u8] = &[];
511 let mut opened_input = input.open(dummy_stdin, None).unwrap();
513 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
517 fn syntax_for_file_os(&self, file_name: &OsStr) -> String {
518 self.syntax_for_file_with_content_os(file_name, "")
521 fn syntax_for_file_with_content(&self, file_name: &str, first_line: &str) -> String {
522 self.syntax_for_file_with_content_os(OsStr::new(file_name), first_line)
525 fn syntax_for_file(&self, file_name: &str) -> String {
526 self.syntax_for_file_with_content(file_name, "")
529 fn syntax_for_stdin_with_content(&self, file_name: &str, content: &[u8]) -> String {
530 let input = Input::stdin().with_name(Some(file_name));
531 let mut opened_input = input.open(content, None).unwrap();
533 self.get_syntax_name(None, &mut opened_input, &self.syntax_mapping)
536 fn syntax_is_same_for_inputkinds(&self, file_name: &str, content: &str) -> bool {
537 let as_file = self.syntax_for_real_file_with_content_os(file_name.as_ref(), content);
538 let as_reader = self.syntax_for_file_with_content_os(file_name.as_ref(), content);
539 let consistent = as_file == as_reader;
540 // TODO: Compare StdIn somehow?
544 "Inconsistent syntax detection:\nFor File: {}\nFor Reader: {}",
554 fn syntax_detection_basic() {
555 let test = SyntaxDetectionTest::new();
557 assert_eq!(test.syntax_for_file("test.rs"), "Rust");
558 assert_eq!(test.syntax_for_file("test.cpp"), "C++");
559 assert_eq!(test.syntax_for_file("test.build"), "NAnt Build File");
561 test.syntax_for_file("PKGBUILD"),
562 "Bourne Again Shell (bash)"
564 assert_eq!(test.syntax_for_file(".bashrc"), "Bourne Again Shell (bash)");
565 assert_eq!(test.syntax_for_file("Makefile"), "Makefile");
570 fn syntax_detection_invalid_utf8() {
571 use std::os::unix::ffi::OsStrExt;
573 let test = SyntaxDetectionTest::new();
576 test.syntax_for_file_os(OsStr::from_bytes(b"invalid_\xFEutf8_filename.rs")),
582 fn syntax_detection_same_for_inputkinds() {
583 let mut test = SyntaxDetectionTest::new();
586 .insert("*.myext", MappingTarget::MapTo("C"))
589 .insert("MY_FILE", MappingTarget::MapTo("Markdown"))
592 assert!(test.syntax_is_same_for_inputkinds("Test.md", ""));
593 assert!(test.syntax_is_same_for_inputkinds("Test.txt", "#!/bin/bash"));
594 assert!(test.syntax_is_same_for_inputkinds(".bashrc", ""));
595 assert!(test.syntax_is_same_for_inputkinds("test.h", ""));
596 assert!(test.syntax_is_same_for_inputkinds("test.js", "#!/bin/bash"));
597 assert!(test.syntax_is_same_for_inputkinds("test.myext", ""));
598 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", ""));
599 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", "<?php"));
603 fn syntax_detection_well_defined_mapping_for_duplicate_extensions() {
604 let test = SyntaxDetectionTest::new();
606 assert_eq!(test.syntax_for_file("test.h"), "C++");
607 assert_eq!(test.syntax_for_file("test.sass"), "Sass");
608 assert_eq!(test.syntax_for_file("test.js"), "JavaScript (Babel)");
609 assert_eq!(test.syntax_for_file("test.fs"), "F#");
610 assert_eq!(test.syntax_for_file("test.v"), "Verilog");
614 fn syntax_detection_first_line() {
615 let test = SyntaxDetectionTest::new();
618 test.syntax_for_file_with_content("my_script", "#!/bin/bash"),
619 "Bourne Again Shell (bash)"
622 test.syntax_for_file_with_content("build", "#!/bin/bash"),
623 "Bourne Again Shell (bash)"
626 test.syntax_for_file_with_content("my_script", "<?php"),
632 fn syntax_detection_with_custom_mapping() {
633 let mut test = SyntaxDetectionTest::new();
635 assert_eq!(test.syntax_for_file("test.h"), "C++");
637 .insert("*.h", MappingTarget::MapTo("C"))
639 assert_eq!(test.syntax_for_file("test.h"), "C");
643 fn syntax_detection_is_case_sensitive() {
644 let mut test = SyntaxDetectionTest::new();
646 assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
648 .insert("*.MD", MappingTarget::MapTo("Markdown"))
650 assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
654 fn syntax_detection_stdin_filename() {
655 let test = SyntaxDetectionTest::new();
657 // from file extension
658 assert_eq!(test.syntax_for_stdin_with_content("test.cpp", b"a"), "C++");
659 // from first line (fallback)
661 test.syntax_for_stdin_with_content("my_script", b"#!/bin/bash"),
662 "Bourne Again Shell (bash)"
668 fn syntax_detection_for_symlinked_file() {
669 use std::os::unix::fs::symlink;
671 let test = SyntaxDetectionTest::new();
672 let file_path = test.temp_dir.path().join("my_ssh_config_filename");
674 File::create(&file_path).unwrap();
676 let file_path_symlink = test.temp_dir.path().join(".ssh").join("config");
678 std::fs::create_dir(test.temp_dir.path().join(".ssh"))
679 .expect("creation of directory succeeds");
680 symlink(&file_path, &file_path_symlink).expect("creation of symbolic link succeeds");
682 let input = Input::ordinary_file(&file_path_symlink);
683 let dummy_stdin: &[u8] = &[];
684 let mut opened_input = input.open(dummy_stdin, None).unwrap();
687 test.get_syntax_name(None, &mut opened_input, &test.syntax_mapping),
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list