git.iko.soy/bat
on 3d07dec help sign in
bat/src/assets.rs
539 lines 19 kB history raw
1use std::collections::BTreeMap;
2use std::ffi::OsStr;
3use std::fs::{self, File};
4use std::io::BufReader;
5use std::path::Path;
7use syntect::dumps::{dump_to_file, from_binary, from_reader};
8use syntect::highlighting::{Theme, ThemeSet};
9use syntect::parsing::{SyntaxReference, SyntaxSet, SyntaxSetBuilder};
11use path_abs::PathAbs;
13use crate::assets_metadata::AssetsMetadata;
14use crate::error::*;
15use crate::input::{InputReader, OpenedInput, OpenedInputKind};
16use crate::syntax_mapping::{MappingTarget, SyntaxMapping};
18#[derive(Debug)]
19pub struct HighlightingAssets {
20 pub(crate) syntax_set: SyntaxSet,
21 pub(crate) theme_set: ThemeSet,
22 fallback_theme: Option<&'static str>,
25impl HighlightingAssets {
26 pub fn default_theme() -> &'static str {
27 "Monokai Extended"
28 }
30 pub fn from_files(source_dir: &Path, include_integrated_assets: bool) -> Result<Self> {
31 let mut theme_set = if include_integrated_assets {
32 Self::get_integrated_themeset()
33 } else {
34 ThemeSet {
35 themes: BTreeMap::new(),
36 }
37 };
39 let theme_dir = source_dir.join("themes");
41 let res = theme_set.add_from_folder(&theme_dir);
42 if res.is_err() {
43 println!(
44 "No themes were found in '{}', using the default set",
45 theme_dir.to_string_lossy()
46 );
47 }
49 let mut syntax_set_builder = if !include_integrated_assets {
50 let mut builder = SyntaxSetBuilder::new();
51 builder.add_plain_text_syntax();
52 builder
53 } else {
54 Self::get_integrated_syntaxset().into_builder()
55 };
57 let syntax_dir = source_dir.join("syntaxes");
58 if syntax_dir.exists() {
59 syntax_set_builder.add_from_folder(syntax_dir, true)?;
60 } else {
61 println!(
62 "No syntaxes were found in '{}', using the default set.",
63 syntax_dir.to_string_lossy()
64 );
65 }
67 Ok(HighlightingAssets {
68 syntax_set: syntax_set_builder.build(),
69 theme_set,
70 fallback_theme: None,
71 })
72 }
74 pub fn from_cache(cache_path: &Path) -> Result<Self> {
75 let syntax_set_path = cache_path.join("syntaxes.bin");
76 let theme_set_path = cache_path.join("themes.bin");
78 let syntax_set_file = File::open(&syntax_set_path).chain_err(|| {
79 format!(
80 "Could not load cached syntax set '{}'",
81 syntax_set_path.to_string_lossy()
82 )
83 })?;
84 let syntax_set: SyntaxSet = from_reader(BufReader::new(syntax_set_file))
85 .chain_err(|| "Could not parse cached syntax set")?;
87 let theme_set_file = File::open(&theme_set_path).chain_err(|| {
88 format!(
89 "Could not load cached theme set '{}'",
90 theme_set_path.to_string_lossy()
91 )
92 })?;
93 let theme_set: ThemeSet = from_reader(BufReader::new(theme_set_file))
94 .chain_err(|| "Could not parse cached theme set")?;
96 Ok(HighlightingAssets {
97 syntax_set,
98 theme_set,
99 fallback_theme: None,
100 })
101 }
103 fn get_integrated_syntaxset() -> SyntaxSet {
104 from_binary(include_bytes!("../assets/syntaxes.bin"))
105 }
107 fn get_integrated_themeset() -> ThemeSet {
108 from_binary(include_bytes!("../assets/themes.bin"))
109 }
111 pub fn from_binary() -> Self {
112 let syntax_set = Self::get_integrated_syntaxset();
113 let theme_set = Self::get_integrated_themeset();
115 HighlightingAssets {
116 syntax_set,
117 theme_set,
118 fallback_theme: None,
119 }
120 }
122 pub fn save_to_cache(&self, target_dir: &Path, current_version: &str) -> Result<()> {
123 let _ = fs::create_dir_all(target_dir);
124 let theme_set_path = target_dir.join("themes.bin");
125 let syntax_set_path = target_dir.join("syntaxes.bin");
127 print!(
128 "Writing theme set to {} ... ",
129 theme_set_path.to_string_lossy()
130 );
131 dump_to_file(&self.theme_set, &theme_set_path).chain_err(|| {
132 format!(
133 "Could not save theme set to {}",
134 theme_set_path.to_string_lossy()
135 )
136 })?;
137 println!("okay");
139 print!(
140 "Writing syntax set to {} ... ",
141 syntax_set_path.to_string_lossy()
142 );
143 dump_to_file(&self.syntax_set, &syntax_set_path).chain_err(|| {
144 format!(
145 "Could not save syntax set to {}",
146 syntax_set_path.to_string_lossy()
147 )
148 })?;
149 println!("okay");
151 print!(
152 "Writing metadata to folder {} ... ",
153 target_dir.to_string_lossy()
154 );
155 AssetsMetadata::new(current_version).save_to_folder(target_dir)?;
156 println!("okay");
158 Ok(())
159 }
161 pub fn set_fallback_theme(&mut self, theme: &'static str) {
162 self.fallback_theme = Some(theme);
163 }
165 pub fn syntaxes(&self) -> &[SyntaxReference] {
166 self.syntax_set.syntaxes()
167 }
169 pub fn themes(&self) -> impl Iterator<Item = &str> {
170 self.theme_set.themes.keys().map(|s| s.as_ref())
171 }
173 pub fn syntax_for_file_name(
174 &self,
175 file_name: impl AsRef<Path>,
176 mapping: &SyntaxMapping,
177 ) -> Option<&SyntaxReference> {
178 let file_name = file_name.as_ref();
179 match mapping.get_syntax_for(file_name) {
180 Some(MappingTarget::MapToUnknown) => None,
181 Some(MappingTarget::MapTo(syntax_name)) => {
182 self.syntax_set.find_syntax_by_name(syntax_name)
183 }
184 None => self.get_extension_syntax(file_name.as_os_str()),
185 }
186 }
188 pub(crate) fn get_theme(&self, theme: &str) -> &Theme {
189 match self.theme_set.themes.get(theme) {
190 Some(theme) => theme,
191 None => {
192 if theme == "ansi-light" || theme == "ansi-dark" {
193 use ansi_term::Colour::Yellow;
194 eprintln!(
195 "{}: Theme '{}' is deprecated, using 'ansi' instead.",
196 Yellow.paint("[bat warning]"),
197 theme
198 );
199 return self.get_theme("ansi");
200 }
201 if theme != "" {
202 use ansi_term::Colour::Yellow;
203 eprintln!(
204 "{}: Unknown theme '{}', using default.",
205 Yellow.paint("[bat warning]"),
206 theme
207 );
208 }
209 &self.theme_set.themes[self.fallback_theme.unwrap_or_else(|| Self::default_theme())]
210 }
211 }
212 }
214 pub(crate) fn get_syntax(
215 &self,
216 language: Option<&str>,
217 input: &mut OpenedInput,
218 mapping: &SyntaxMapping,
219 ) -> Result<&SyntaxReference> {
220 if let Some(language) = language {
221 self.syntax_set
222 .find_syntax_by_token(language)
223 .ok_or_else(|| ErrorKind::UnknownSyntax(language.to_owned()).into())
224 } else {
225 let line_syntax = self.get_first_line_syntax(&mut input.reader);
227 // Get the path of the file:
228 // If this was set by the metadata, that will take priority.
229 // If it wasn't, it will use the real file path (if available).
230 let path_str =
231 input
232 .metadata
233 .user_provided_name
234 .as_ref()
235 .or_else(|| match input.kind {
236 OpenedInputKind::OrdinaryFile(ref path) => Some(path),
237 _ => None,
238 });
240 if let Some(path_str) = path_str {
241 // If a path was provided, we try and detect the syntax based on extension mappings.
242 let path = Path::new(path_str);
243 let absolute_path = PathAbs::new(path)
244 .ok()
245 .map(|p| p.as_path().to_path_buf())
246 .unwrap_or_else(|| path.to_owned());
248 match mapping.get_syntax_for(absolute_path) {
249 Some(MappingTarget::MapToUnknown) => line_syntax.ok_or_else(|| {
250 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
251 }),
253 Some(MappingTarget::MapTo(syntax_name)) => self
254 .syntax_set
255 .find_syntax_by_name(syntax_name)
256 .ok_or_else(|| ErrorKind::UnknownSyntax(syntax_name.to_owned()).into()),
258 None => {
259 let file_name = path.file_name().unwrap_or_default();
260 self.get_extension_syntax(file_name)
261 .or(line_syntax)
262 .ok_or_else(|| {
263 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
264 })
265 }
266 }
267 } else {
268 // If a path wasn't provided, we fall back to the detect first-line syntax.
269 line_syntax.ok_or_else(|| ErrorKind::UndetectedSyntax("[unknown]".into()).into())
270 }
271 }
272 }
274 fn get_extension_syntax(&self, file_name: &OsStr) -> Option<&SyntaxReference> {
275 self.syntax_set
276 .find_syntax_by_extension(file_name.to_str().unwrap_or_default())
277 .or_else(|| {
278 self.syntax_set.find_syntax_by_extension(
279 Path::new(file_name)
280 .extension()
281 .and_then(|x| x.to_str())
282 .unwrap_or_default(),
283 )
284 })
285 }
287 fn get_first_line_syntax(&self, reader: &mut InputReader) -> Option<&SyntaxReference> {
288 String::from_utf8(reader.first_line.clone())
289 .ok()
290 .and_then(|l| self.syntax_set.find_syntax_by_first_line(&l))
291 }
294#[cfg(test)]
295mod tests {
296 use super::*;
298 use std::ffi::OsStr;
300 use std::fs::File;
301 use std::io::Write;
302 use tempdir::TempDir;
304 use crate::input::Input;
306 struct SyntaxDetectionTest<'a> {
307 assets: HighlightingAssets,
308 pub syntax_mapping: SyntaxMapping<'a>,
309 pub temp_dir: TempDir,
310 }
312 impl<'a> SyntaxDetectionTest<'a> {
313 fn new() -> Self {
314 SyntaxDetectionTest {
315 assets: HighlightingAssets::from_binary(),
316 syntax_mapping: SyntaxMapping::builtin(),
317 temp_dir: TempDir::new("bat_syntax_detection_tests")
318 .expect("creation of temporary directory"),
319 }
320 }
322 fn syntax_for_real_file_with_content_os(
323 &self,
324 file_name: &OsStr,
325 first_line: &str,
326 ) -> String {
327 let file_path = self.temp_dir.path().join(file_name);
328 {
329 let mut temp_file = File::create(&file_path).unwrap();
330 writeln!(temp_file, "{}", first_line).unwrap();
331 }
333 let input = Input::ordinary_file(file_path.as_os_str());
334 let dummy_stdin: &[u8] = &[];
335 let mut opened_input = input.open(dummy_stdin).unwrap();
337 self.assets
338 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
339 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
340 .name
341 .clone()
342 }
344 fn syntax_for_file_with_content_os(&self, file_name: &OsStr, first_line: &str) -> String {
345 let file_path = self.temp_dir.path().join(file_name);
346 let input = Input::from_reader(Box::new(BufReader::new(first_line.as_bytes())))
347 .with_name(Some(file_path.as_os_str()));
348 let dummy_stdin: &[u8] = &[];
349 let mut opened_input = input.open(dummy_stdin).unwrap();
351 self.assets
352 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
353 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
354 .name
355 .clone()
356 }
358 #[cfg(unix)]
359 fn syntax_for_file_os(&self, file_name: &OsStr) -> String {
360 self.syntax_for_file_with_content_os(file_name, "")
361 }
363 fn syntax_for_file_with_content(&self, file_name: &str, first_line: &str) -> String {
364 self.syntax_for_file_with_content_os(OsStr::new(file_name), first_line)
365 }
367 fn syntax_for_file(&self, file_name: &str) -> String {
368 self.syntax_for_file_with_content(file_name, "")
369 }
371 fn syntax_for_stdin_with_content(&self, file_name: &str, content: &[u8]) -> String {
372 let input = Input::stdin().with_name(Some(OsStr::new(file_name)));
373 let mut opened_input = input.open(content).unwrap();
375 self.assets
376 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
377 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
378 .name
379 .clone()
380 }
382 fn syntax_is_same_for_inputkinds(&self, file_name: &str, content: &str) -> bool {
383 let as_file = self.syntax_for_real_file_with_content_os(file_name.as_ref(), content);
384 let as_reader = self.syntax_for_file_with_content_os(file_name.as_ref(), content);
385 let consistent = as_file == as_reader;
386 // TODO: Compare StdIn somehow?
388 if !consistent {
389 eprintln!(
390 "Inconsistent syntax detection:\nFor File: {}\nFor Reader: {}",
391 as_file, as_reader
392 )
393 }
395 consistent
396 }
397 }
399 #[test]
400 fn syntax_detection_basic() {
401 let test = SyntaxDetectionTest::new();
403 assert_eq!(test.syntax_for_file("test.rs"), "Rust");
404 assert_eq!(test.syntax_for_file("test.cpp"), "C++");
405 assert_eq!(test.syntax_for_file("test.build"), "NAnt Build File");
406 assert_eq!(
407 test.syntax_for_file("PKGBUILD"),
408 "Bourne Again Shell (bash)"
409 );
410 assert_eq!(test.syntax_for_file(".bashrc"), "Bourne Again Shell (bash)");
411 assert_eq!(test.syntax_for_file("Makefile"), "Makefile");
412 }
414 #[cfg(unix)]
415 #[test]
416 fn syntax_detection_invalid_utf8() {
417 use std::os::unix::ffi::OsStrExt;
419 let test = SyntaxDetectionTest::new();
421 assert_eq!(
422 test.syntax_for_file_os(OsStr::from_bytes(b"invalid_\xFEutf8_filename.rs")),
423 "Rust"
424 );
425 }
427 #[test]
428 fn syntax_detection_same_for_inputkinds() {
429 let mut test = SyntaxDetectionTest::new();
431 test.syntax_mapping
432 .insert("*.myext", MappingTarget::MapTo("C"))
433 .ok();
434 test.syntax_mapping
435 .insert("MY_FILE", MappingTarget::MapTo("Markdown"))
436 .ok();
438 assert!(test.syntax_is_same_for_inputkinds("Test.md", ""));
439 assert!(test.syntax_is_same_for_inputkinds("Test.txt", "#!/bin/bash"));
440 assert!(test.syntax_is_same_for_inputkinds(".bashrc", ""));
441 assert!(test.syntax_is_same_for_inputkinds("test.h", ""));
442 assert!(test.syntax_is_same_for_inputkinds("test.js", "#!/bin/bash"));
443 assert!(test.syntax_is_same_for_inputkinds("test.myext", ""));
444 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", ""));
445 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", "<?php"));
446 }
448 #[test]
449 fn syntax_detection_well_defined_mapping_for_duplicate_extensions() {
450 let test = SyntaxDetectionTest::new();
452 assert_eq!(test.syntax_for_file("test.h"), "C++");
453 assert_eq!(test.syntax_for_file("test.sass"), "Sass");
454 assert_eq!(test.syntax_for_file("test.js"), "JavaScript (Babel)");
455 assert_eq!(test.syntax_for_file("test.fs"), "F#");
456 }
458 #[test]
459 fn syntax_detection_first_line() {
460 let test = SyntaxDetectionTest::new();
462 assert_eq!(
463 test.syntax_for_file_with_content("my_script", "#!/bin/bash"),
464 "Bourne Again Shell (bash)"
465 );
466 assert_eq!(
467 test.syntax_for_file_with_content("build", "#!/bin/bash"),
468 "Bourne Again Shell (bash)"
469 );
470 assert_eq!(
471 test.syntax_for_file_with_content("my_script", "<?php"),
472 "PHP"
473 );
474 }
476 #[test]
477 fn syntax_detection_with_custom_mapping() {
478 let mut test = SyntaxDetectionTest::new();
480 assert_eq!(test.syntax_for_file("test.h"), "C++");
481 test.syntax_mapping
482 .insert("*.h", MappingTarget::MapTo("C"))
483 .ok();
484 assert_eq!(test.syntax_for_file("test.h"), "C");
485 }
487 #[test]
488 fn syntax_detection_is_case_sensitive() {
489 let mut test = SyntaxDetectionTest::new();
491 assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
492 test.syntax_mapping
493 .insert("*.MD", MappingTarget::MapTo("Markdown"))
494 .ok();
495 assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
496 }
498 #[test]
499 fn syntax_detection_stdin_filename() {
500 let test = SyntaxDetectionTest::new();
502 // from file extension
503 assert_eq!(test.syntax_for_stdin_with_content("test.cpp", b"a"), "C++");
504 // from first line (fallback)
505 assert_eq!(
506 test.syntax_for_stdin_with_content("my_script", b"#!/bin/bash"),
507 "Bourne Again Shell (bash)"
508 );
509 }
511 #[cfg(unix)]
512 #[test]
513 fn syntax_detection_for_symlinked_file() {
514 use std::os::unix::fs::symlink;
516 let test = SyntaxDetectionTest::new();
517 let file_path = test.temp_dir.path().join("my_ssh_config_filename");
518 {
519 File::create(&file_path).unwrap();
520 }
521 let file_path_symlink = test.temp_dir.path().join(".ssh").join("config");
523 std::fs::create_dir(test.temp_dir.path().join(".ssh"))
524 .expect("creation of directory succeeds");
525 symlink(&file_path, &file_path_symlink).expect("creation of symbolic link succeeds");
527 let input = Input::ordinary_file(file_path_symlink.as_os_str());
528 let dummy_stdin: &[u8] = &[];
529 let mut opened_input = input.open(dummy_stdin).unwrap();
531 assert_eq!(
532 test.assets
533 .get_syntax(None, &mut opened_input, &test.syntax_mapping)
534 .unwrap_or_else(|_| test.assets.syntax_set.find_syntax_plain_text())
535 .name,
536 "SSH Config"
537 );
538 }