git.iko.soy/bat
on deec290 help sign in
bat/src/assets.rs
537 lines 19 kB history raw
1use std::collections::BTreeMap;
2use std::ffi::OsStr;
3use std::fs::{self, File};
4use std::io::BufReader;
5use std::path::Path;
7use syntect::dumps::{dump_to_file, from_binary, from_reader};
8use syntect::highlighting::{Theme, ThemeSet};
9use syntect::parsing::{SyntaxReference, SyntaxSet, SyntaxSetBuilder};
11use path_abs::PathAbs;
13use crate::assets_metadata::AssetsMetadata;
14use crate::bat_warning;
15use crate::error::*;
16use crate::input::{InputReader, OpenedInput, OpenedInputKind};
17use crate::syntax_mapping::{MappingTarget, SyntaxMapping};
19#[derive(Debug)]
20pub struct HighlightingAssets {
21 pub(crate) syntax_set: SyntaxSet,
22 pub(crate) theme_set: ThemeSet,
23 fallback_theme: Option<&'static str>,
26impl HighlightingAssets {
27 pub fn default_theme() -> &'static str {
28 "Monokai Extended"
29 }
31 pub fn from_files(source_dir: &Path, include_integrated_assets: bool) -> Result<Self> {
32 let mut theme_set = if include_integrated_assets {
33 Self::get_integrated_themeset()
34 } else {
35 ThemeSet {
36 themes: BTreeMap::new(),
37 }
38 };
40 let theme_dir = source_dir.join("themes");
41 if theme_dir.exists() {
42 let res = theme_set.add_from_folder(&theme_dir);
43 if let Err(err) = res {
44 println!(
45 "Failed to load one or more themes from '{}' (reason: '{}')",
46 theme_dir.to_string_lossy(),
47 err,
48 );
49 }
50 } else {
51 println!(
52 "No themes were found in '{}', using the default set",
53 theme_dir.to_string_lossy()
54 );
55 }
57 let mut syntax_set_builder = if !include_integrated_assets {
58 let mut builder = SyntaxSetBuilder::new();
59 builder.add_plain_text_syntax();
60 builder
61 } else {
62 Self::get_integrated_syntaxset().into_builder()
63 };
65 let syntax_dir = source_dir.join("syntaxes");
66 if syntax_dir.exists() {
67 syntax_set_builder.add_from_folder(syntax_dir, true)?;
68 } else {
69 println!(
70 "No syntaxes were found in '{}', using the default set.",
71 syntax_dir.to_string_lossy()
72 );
73 }
75 Ok(HighlightingAssets {
76 syntax_set: syntax_set_builder.build(),
77 theme_set,
78 fallback_theme: None,
79 })
80 }
82 pub fn from_cache(cache_path: &Path) -> Result<Self> {
83 let syntax_set_path = cache_path.join("syntaxes.bin");
84 let theme_set_path = cache_path.join("themes.bin");
86 let syntax_set_file = File::open(&syntax_set_path).chain_err(|| {
87 format!(
88 "Could not load cached syntax set '{}'",
89 syntax_set_path.to_string_lossy()
90 )
91 })?;
92 let syntax_set: SyntaxSet = from_reader(BufReader::new(syntax_set_file))
93 .chain_err(|| "Could not parse cached syntax set")?;
95 let theme_set_file = File::open(&theme_set_path).chain_err(|| {
96 format!(
97 "Could not load cached theme set '{}'",
98 theme_set_path.to_string_lossy()
99 )
100 })?;
101 let theme_set: ThemeSet = from_reader(BufReader::new(theme_set_file))
102 .chain_err(|| "Could not parse cached theme set")?;
104 Ok(HighlightingAssets {
105 syntax_set,
106 theme_set,
107 fallback_theme: None,
108 })
109 }
111 fn get_integrated_syntaxset() -> SyntaxSet {
112 from_binary(include_bytes!("../assets/syntaxes.bin"))
113 }
115 fn get_integrated_themeset() -> ThemeSet {
116 from_binary(include_bytes!("../assets/themes.bin"))
117 }
119 pub fn from_binary() -> Self {
120 let syntax_set = Self::get_integrated_syntaxset();
121 let theme_set = Self::get_integrated_themeset();
123 HighlightingAssets {
124 syntax_set,
125 theme_set,
126 fallback_theme: None,
127 }
128 }
130 pub fn save_to_cache(&self, target_dir: &Path, current_version: &str) -> Result<()> {
131 let _ = fs::create_dir_all(target_dir);
132 let theme_set_path = target_dir.join("themes.bin");
133 let syntax_set_path = target_dir.join("syntaxes.bin");
135 print!(
136 "Writing theme set to {} ... ",
137 theme_set_path.to_string_lossy()
138 );
139 dump_to_file(&self.theme_set, &theme_set_path).chain_err(|| {
140 format!(
141 "Could not save theme set to {}",
142 theme_set_path.to_string_lossy()
143 )
144 })?;
145 println!("okay");
147 print!(
148 "Writing syntax set to {} ... ",
149 syntax_set_path.to_string_lossy()
150 );
151 dump_to_file(&self.syntax_set, &syntax_set_path).chain_err(|| {
152 format!(
153 "Could not save syntax set to {}",
154 syntax_set_path.to_string_lossy()
155 )
156 })?;
157 println!("okay");
159 print!(
160 "Writing metadata to folder {} ... ",
161 target_dir.to_string_lossy()
162 );
163 AssetsMetadata::new(current_version).save_to_folder(target_dir)?;
164 println!("okay");
166 Ok(())
167 }
169 pub fn set_fallback_theme(&mut self, theme: &'static str) {
170 self.fallback_theme = Some(theme);
171 }
173 pub fn syntaxes(&self) -> &[SyntaxReference] {
174 self.syntax_set.syntaxes()
175 }
177 pub fn themes(&self) -> impl Iterator<Item = &str> {
178 self.theme_set.themes.keys().map(|s| s.as_ref())
179 }
181 pub fn syntax_for_file_name(
182 &self,
183 file_name: impl AsRef<Path>,
184 mapping: &SyntaxMapping,
185 ) -> Option<&SyntaxReference> {
186 let file_name = file_name.as_ref();
187 match mapping.get_syntax_for(file_name) {
188 Some(MappingTarget::MapToUnknown) => None,
189 Some(MappingTarget::MapTo(syntax_name)) => {
190 self.syntax_set.find_syntax_by_name(syntax_name)
191 }
192 None => self.get_extension_syntax(file_name.as_os_str()),
193 }
194 }
196 pub(crate) fn get_theme(&self, theme: &str) -> &Theme {
197 match self.theme_set.themes.get(theme) {
198 Some(theme) => theme,
199 None => {
200 if theme == "ansi-light" || theme == "ansi-dark" {
201 bat_warning!("Theme '{}' is deprecated, using 'ansi' instead.", theme);
202 return self.get_theme("ansi");
203 }
204 if theme != "" {
205 bat_warning!("Unknown theme '{}', using default.", theme)
206 }
207 &self.theme_set.themes[self.fallback_theme.unwrap_or_else(|| Self::default_theme())]
208 }
209 }
210 }
212 pub(crate) fn get_syntax(
213 &self,
214 language: Option<&str>,
215 input: &mut OpenedInput,
216 mapping: &SyntaxMapping,
217 ) -> Result<&SyntaxReference> {
218 if let Some(language) = language {
219 self.syntax_set
220 .find_syntax_by_token(language)
221 .ok_or_else(|| ErrorKind::UnknownSyntax(language.to_owned()).into())
222 } else {
223 let line_syntax = self.get_first_line_syntax(&mut input.reader);
225 // Get the path of the file:
226 // If this was set by the metadata, that will take priority.
227 // If it wasn't, it will use the real file path (if available).
228 let path_str =
229 input
230 .metadata
231 .user_provided_name
232 .as_ref()
233 .or_else(|| match input.kind {
234 OpenedInputKind::OrdinaryFile(ref path) => Some(path),
235 _ => None,
236 });
238 if let Some(path_str) = path_str {
239 // If a path was provided, we try and detect the syntax based on extension mappings.
240 let path = Path::new(path_str);
241 let absolute_path = PathAbs::new(path)
242 .ok()
243 .map(|p| p.as_path().to_path_buf())
244 .unwrap_or_else(|| path.to_owned());
246 match mapping.get_syntax_for(absolute_path) {
247 Some(MappingTarget::MapToUnknown) => line_syntax.ok_or_else(|| {
248 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
249 }),
251 Some(MappingTarget::MapTo(syntax_name)) => self
252 .syntax_set
253 .find_syntax_by_name(syntax_name)
254 .ok_or_else(|| ErrorKind::UnknownSyntax(syntax_name.to_owned()).into()),
256 None => {
257 let file_name = path.file_name().unwrap_or_default();
258 self.get_extension_syntax(file_name)
259 .or(line_syntax)
260 .ok_or_else(|| {
261 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
262 })
263 }
264 }
265 } else {
266 // If a path wasn't provided, we fall back to the detect first-line syntax.
267 line_syntax.ok_or_else(|| ErrorKind::UndetectedSyntax("[unknown]".into()).into())
268 }
269 }
270 }
272 fn get_extension_syntax(&self, file_name: &OsStr) -> Option<&SyntaxReference> {
273 self.syntax_set
274 .find_syntax_by_extension(file_name.to_str().unwrap_or_default())
275 .or_else(|| {
276 self.syntax_set.find_syntax_by_extension(
277 Path::new(file_name)
278 .extension()
279 .and_then(|x| x.to_str())
280 .unwrap_or_default(),
281 )
282 })
283 }
285 fn get_first_line_syntax(&self, reader: &mut InputReader) -> Option<&SyntaxReference> {
286 String::from_utf8(reader.first_line.clone())
287 .ok()
288 .and_then(|l| self.syntax_set.find_syntax_by_first_line(&l))
289 }
292#[cfg(test)]
293mod tests {
294 use super::*;
296 use std::ffi::OsStr;
298 use std::fs::File;
299 use std::io::Write;
300 use tempdir::TempDir;
302 use crate::input::Input;
304 struct SyntaxDetectionTest<'a> {
305 assets: HighlightingAssets,
306 pub syntax_mapping: SyntaxMapping<'a>,
307 pub temp_dir: TempDir,
308 }
310 impl<'a> SyntaxDetectionTest<'a> {
311 fn new() -> Self {
312 SyntaxDetectionTest {
313 assets: HighlightingAssets::from_binary(),
314 syntax_mapping: SyntaxMapping::builtin(),
315 temp_dir: TempDir::new("bat_syntax_detection_tests")
316 .expect("creation of temporary directory"),
317 }
318 }
320 fn syntax_for_real_file_with_content_os(
321 &self,
322 file_name: &OsStr,
323 first_line: &str,
324 ) -> String {
325 let file_path = self.temp_dir.path().join(file_name);
326 {
327 let mut temp_file = File::create(&file_path).unwrap();
328 writeln!(temp_file, "{}", first_line).unwrap();
329 }
331 let input = Input::ordinary_file(file_path.as_os_str());
332 let dummy_stdin: &[u8] = &[];
333 let mut opened_input = input.open(dummy_stdin).unwrap();
335 self.assets
336 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
337 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
338 .name
339 .clone()
340 }
342 fn syntax_for_file_with_content_os(&self, file_name: &OsStr, first_line: &str) -> String {
343 let file_path = self.temp_dir.path().join(file_name);
344 let input = Input::from_reader(Box::new(BufReader::new(first_line.as_bytes())))
345 .with_name(Some(file_path.as_os_str()));
346 let dummy_stdin: &[u8] = &[];
347 let mut opened_input = input.open(dummy_stdin).unwrap();
349 self.assets
350 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
351 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
352 .name
353 .clone()
354 }
356 #[cfg(unix)]
357 fn syntax_for_file_os(&self, file_name: &OsStr) -> String {
358 self.syntax_for_file_with_content_os(file_name, "")
359 }
361 fn syntax_for_file_with_content(&self, file_name: &str, first_line: &str) -> String {
362 self.syntax_for_file_with_content_os(OsStr::new(file_name), first_line)
363 }
365 fn syntax_for_file(&self, file_name: &str) -> String {
366 self.syntax_for_file_with_content(file_name, "")
367 }
369 fn syntax_for_stdin_with_content(&self, file_name: &str, content: &[u8]) -> String {
370 let input = Input::stdin().with_name(Some(OsStr::new(file_name)));
371 let mut opened_input = input.open(content).unwrap();
373 self.assets
374 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
375 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
376 .name
377 .clone()
378 }
380 fn syntax_is_same_for_inputkinds(&self, file_name: &str, content: &str) -> bool {
381 let as_file = self.syntax_for_real_file_with_content_os(file_name.as_ref(), content);
382 let as_reader = self.syntax_for_file_with_content_os(file_name.as_ref(), content);
383 let consistent = as_file == as_reader;
384 // TODO: Compare StdIn somehow?
386 if !consistent {
387 eprintln!(
388 "Inconsistent syntax detection:\nFor File: {}\nFor Reader: {}",
389 as_file, as_reader
390 )
391 }
393 consistent
394 }
395 }
397 #[test]
398 fn syntax_detection_basic() {
399 let test = SyntaxDetectionTest::new();
401 assert_eq!(test.syntax_for_file("test.rs"), "Rust");
402 assert_eq!(test.syntax_for_file("test.cpp"), "C++");
403 assert_eq!(test.syntax_for_file("test.build"), "NAnt Build File");
404 assert_eq!(
405 test.syntax_for_file("PKGBUILD"),
406 "Bourne Again Shell (bash)"
407 );
408 assert_eq!(test.syntax_for_file(".bashrc"), "Bourne Again Shell (bash)");
409 assert_eq!(test.syntax_for_file("Makefile"), "Makefile");
410 }
412 #[cfg(unix)]
413 #[test]
414 fn syntax_detection_invalid_utf8() {
415 use std::os::unix::ffi::OsStrExt;
417 let test = SyntaxDetectionTest::new();
419 assert_eq!(
420 test.syntax_for_file_os(OsStr::from_bytes(b"invalid_\xFEutf8_filename.rs")),
421 "Rust"
422 );
423 }
425 #[test]
426 fn syntax_detection_same_for_inputkinds() {
427 let mut test = SyntaxDetectionTest::new();
429 test.syntax_mapping
430 .insert("*.myext", MappingTarget::MapTo("C"))
431 .ok();
432 test.syntax_mapping
433 .insert("MY_FILE", MappingTarget::MapTo("Markdown"))
434 .ok();
436 assert!(test.syntax_is_same_for_inputkinds("Test.md", ""));
437 assert!(test.syntax_is_same_for_inputkinds("Test.txt", "#!/bin/bash"));
438 assert!(test.syntax_is_same_for_inputkinds(".bashrc", ""));
439 assert!(test.syntax_is_same_for_inputkinds("test.h", ""));
440 assert!(test.syntax_is_same_for_inputkinds("test.js", "#!/bin/bash"));
441 assert!(test.syntax_is_same_for_inputkinds("test.myext", ""));
442 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", ""));
443 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", "<?php"));
444 }
446 #[test]
447 fn syntax_detection_well_defined_mapping_for_duplicate_extensions() {
448 let test = SyntaxDetectionTest::new();
450 assert_eq!(test.syntax_for_file("test.h"), "C++");
451 assert_eq!(test.syntax_for_file("test.sass"), "Sass");
452 assert_eq!(test.syntax_for_file("test.js"), "JavaScript (Babel)");
453 assert_eq!(test.syntax_for_file("test.fs"), "F#");
454 }
456 #[test]
457 fn syntax_detection_first_line() {
458 let test = SyntaxDetectionTest::new();
460 assert_eq!(
461 test.syntax_for_file_with_content("my_script", "#!/bin/bash"),
462 "Bourne Again Shell (bash)"
463 );
464 assert_eq!(
465 test.syntax_for_file_with_content("build", "#!/bin/bash"),
466 "Bourne Again Shell (bash)"
467 );
468 assert_eq!(
469 test.syntax_for_file_with_content("my_script", "<?php"),
470 "PHP"
471 );
472 }
474 #[test]
475 fn syntax_detection_with_custom_mapping() {
476 let mut test = SyntaxDetectionTest::new();
478 assert_eq!(test.syntax_for_file("test.h"), "C++");
479 test.syntax_mapping
480 .insert("*.h", MappingTarget::MapTo("C"))
481 .ok();
482 assert_eq!(test.syntax_for_file("test.h"), "C");
483 }
485 #[test]
486 fn syntax_detection_is_case_sensitive() {
487 let mut test = SyntaxDetectionTest::new();
489 assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
490 test.syntax_mapping
491 .insert("*.MD", MappingTarget::MapTo("Markdown"))
492 .ok();
493 assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
494 }
496 #[test]
497 fn syntax_detection_stdin_filename() {
498 let test = SyntaxDetectionTest::new();
500 // from file extension
501 assert_eq!(test.syntax_for_stdin_with_content("test.cpp", b"a"), "C++");
502 // from first line (fallback)
503 assert_eq!(
504 test.syntax_for_stdin_with_content("my_script", b"#!/bin/bash"),
505 "Bourne Again Shell (bash)"
506 );
507 }
509 #[cfg(unix)]
510 #[test]
511 fn syntax_detection_for_symlinked_file() {
512 use std::os::unix::fs::symlink;
514 let test = SyntaxDetectionTest::new();
515 let file_path = test.temp_dir.path().join("my_ssh_config_filename");
516 {
517 File::create(&file_path).unwrap();
518 }
519 let file_path_symlink = test.temp_dir.path().join(".ssh").join("config");
521 std::fs::create_dir(test.temp_dir.path().join(".ssh"))
522 .expect("creation of directory succeeds");
523 symlink(&file_path, &file_path_symlink).expect("creation of symbolic link succeeds");
525 let input = Input::ordinary_file(file_path_symlink.as_os_str());
526 let dummy_stdin: &[u8] = &[];
527 let mut opened_input = input.open(dummy_stdin).unwrap();
529 assert_eq!(
530 test.assets
531 .get_syntax(None, &mut opened_input, &test.syntax_mapping)
532 .unwrap_or_else(|_| test.assets.syntax_set.find_syntax_plain_text())
533 .name,
534 "SSH Config"
535 );
536 }