git.iko.soy/bat
on b623adc help sign in
bat/src/assets.rs
530 lines 18 kB history raw
1use std::collections::BTreeMap;
2use std::ffi::OsStr;
3use std::fs::{self, File};
4use std::io::BufReader;
5use std::path::Path;
7use syntect::dumps::{dump_to_file, from_binary, from_reader};
8use syntect::highlighting::{Theme, ThemeSet};
9use syntect::parsing::{SyntaxReference, SyntaxSet, SyntaxSetBuilder};
11use path_abs::PathAbs;
13use crate::assets_metadata::AssetsMetadata;
14use crate::error::*;
15use crate::input::{InputReader, OpenedInput, OpenedInputKind};
16use crate::syntax_mapping::{MappingTarget, SyntaxMapping};
18#[derive(Debug)]
19pub struct HighlightingAssets {
20 pub(crate) syntax_set: SyntaxSet,
21 pub(crate) theme_set: ThemeSet,
22 fallback_theme: Option<&'static str>,
25impl HighlightingAssets {
26 pub fn default_theme() -> &'static str {
27 "Monokai Extended"
28 }
30 pub fn from_files(source_dir: &Path, include_integrated_assets: bool) -> Result<Self> {
31 let mut theme_set = if include_integrated_assets {
32 Self::get_integrated_themeset()
33 } else {
34 ThemeSet {
35 themes: BTreeMap::new(),
36 }
37 };
39 let theme_dir = source_dir.join("themes");
41 let res = theme_set.add_from_folder(&theme_dir);
42 if res.is_err() {
43 println!(
44 "No themes were found in '{}', using the default set",
45 theme_dir.to_string_lossy()
46 );
47 }
49 let mut syntax_set_builder = if !include_integrated_assets {
50 let mut builder = SyntaxSetBuilder::new();
51 builder.add_plain_text_syntax();
52 builder
53 } else {
54 Self::get_integrated_syntaxset().into_builder()
55 };
57 let syntax_dir = source_dir.join("syntaxes");
58 if syntax_dir.exists() {
59 syntax_set_builder.add_from_folder(syntax_dir, true)?;
60 } else {
61 println!(
62 "No syntaxes were found in '{}', using the default set.",
63 syntax_dir.to_string_lossy()
64 );
65 }
67 Ok(HighlightingAssets {
68 syntax_set: syntax_set_builder.build(),
69 theme_set,
70 fallback_theme: None,
71 })
72 }
74 pub fn from_cache(cache_path: &Path) -> Result<Self> {
75 let syntax_set_path = cache_path.join("syntaxes.bin");
76 let theme_set_path = cache_path.join("themes.bin");
78 let syntax_set_file = File::open(&syntax_set_path).chain_err(|| {
79 format!(
80 "Could not load cached syntax set '{}'",
81 syntax_set_path.to_string_lossy()
82 )
83 })?;
84 let syntax_set: SyntaxSet = from_reader(BufReader::new(syntax_set_file))
85 .chain_err(|| "Could not parse cached syntax set")?;
87 let theme_set_file = File::open(&theme_set_path).chain_err(|| {
88 format!(
89 "Could not load cached theme set '{}'",
90 theme_set_path.to_string_lossy()
91 )
92 })?;
93 let theme_set: ThemeSet = from_reader(BufReader::new(theme_set_file))
94 .chain_err(|| "Could not parse cached theme set")?;
96 Ok(HighlightingAssets {
97 syntax_set,
98 theme_set,
99 fallback_theme: None,
100 })
101 }
103 fn get_integrated_syntaxset() -> SyntaxSet {
104 from_binary(include_bytes!("../assets/syntaxes.bin"))
105 }
107 fn get_integrated_themeset() -> ThemeSet {
108 from_binary(include_bytes!("../assets/themes.bin"))
109 }
111 pub fn from_binary() -> Self {
112 let syntax_set = Self::get_integrated_syntaxset();
113 let theme_set = Self::get_integrated_themeset();
115 HighlightingAssets {
116 syntax_set,
117 theme_set,
118 fallback_theme: None,
119 }
120 }
122 pub fn save_to_cache(&self, target_dir: &Path, current_version: &str) -> Result<()> {
123 let _ = fs::create_dir_all(target_dir);
124 let theme_set_path = target_dir.join("themes.bin");
125 let syntax_set_path = target_dir.join("syntaxes.bin");
127 print!(
128 "Writing theme set to {} ... ",
129 theme_set_path.to_string_lossy()
130 );
131 dump_to_file(&self.theme_set, &theme_set_path).chain_err(|| {
132 format!(
133 "Could not save theme set to {}",
134 theme_set_path.to_string_lossy()
135 )
136 })?;
137 println!("okay");
139 print!(
140 "Writing syntax set to {} ... ",
141 syntax_set_path.to_string_lossy()
142 );
143 dump_to_file(&self.syntax_set, &syntax_set_path).chain_err(|| {
144 format!(
145 "Could not save syntax set to {}",
146 syntax_set_path.to_string_lossy()
147 )
148 })?;
149 println!("okay");
151 print!(
152 "Writing metadata to folder {} ... ",
153 target_dir.to_string_lossy()
154 );
155 AssetsMetadata::new(current_version).save_to_folder(target_dir)?;
156 println!("okay");
158 Ok(())
159 }
161 pub fn set_fallback_theme(&mut self, theme: &'static str) {
162 self.fallback_theme = Some(theme);
163 }
165 pub fn syntaxes(&self) -> &[SyntaxReference] {
166 self.syntax_set.syntaxes()
167 }
169 pub fn themes(&self) -> impl Iterator<Item = &str> {
170 self.theme_set.themes.keys().map(|s| s.as_ref())
171 }
173 pub fn syntax_for_file_name(
174 &self,
175 file_name: impl AsRef<Path>,
176 mapping: &SyntaxMapping,
177 ) -> Option<&SyntaxReference> {
178 let file_name = file_name.as_ref();
179 match mapping.get_syntax_for(file_name) {
180 Some(MappingTarget::MapToUnknown) => None,
181 Some(MappingTarget::MapTo(syntax_name)) => {
182 self.syntax_set.find_syntax_by_name(syntax_name)
183 }
184 None => self.get_extension_syntax(file_name.as_os_str()),
185 }
186 }
188 pub(crate) fn get_theme(&self, theme: &str) -> &Theme {
189 match self.theme_set.themes.get(theme) {
190 Some(theme) => theme,
191 None => {
192 if theme != "" {
193 use ansi_term::Colour::Yellow;
194 eprintln!(
195 "{}: Unknown theme '{}', using default.",
196 Yellow.paint("[bat warning]"),
197 theme
198 );
199 }
200 &self.theme_set.themes[self.fallback_theme.unwrap_or_else(|| Self::default_theme())]
201 }
202 }
203 }
205 pub(crate) fn get_syntax(
206 &self,
207 language: Option<&str>,
208 input: &mut OpenedInput,
209 mapping: &SyntaxMapping,
210 ) -> Result<&SyntaxReference> {
211 if let Some(language) = language {
212 self.syntax_set
213 .find_syntax_by_token(language)
214 .ok_or_else(|| ErrorKind::UnknownSyntax(language.to_owned()).into())
215 } else {
216 let line_syntax = self.get_first_line_syntax(&mut input.reader);
218 // Get the path of the file:
219 // If this was set by the metadata, that will take priority.
220 // If it wasn't, it will use the real file path (if available).
221 let path_str =
222 input
223 .metadata
224 .user_provided_name
225 .as_ref()
226 .or_else(|| match input.kind {
227 OpenedInputKind::OrdinaryFile(ref path) => Some(path),
228 _ => None,
229 });
231 if let Some(path_str) = path_str {
232 // If a path was provided, we try and detect the syntax based on extension mappings.
233 let path = Path::new(path_str);
234 let absolute_path = PathAbs::new(path)
235 .ok()
236 .map(|p| p.as_path().to_path_buf())
237 .unwrap_or_else(|| path.to_owned());
239 match mapping.get_syntax_for(absolute_path) {
240 Some(MappingTarget::MapToUnknown) => line_syntax.ok_or_else(|| {
241 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
242 }),
244 Some(MappingTarget::MapTo(syntax_name)) => self
245 .syntax_set
246 .find_syntax_by_name(syntax_name)
247 .ok_or_else(|| ErrorKind::UnknownSyntax(syntax_name.to_owned()).into()),
249 None => {
250 let file_name = path.file_name().unwrap_or_default();
251 self.get_extension_syntax(file_name)
252 .or(line_syntax)
253 .ok_or_else(|| {
254 ErrorKind::UndetectedSyntax(path.to_string_lossy().into()).into()
255 })
256 }
257 }
258 } else {
259 // If a path wasn't provided, we fall back to the detect first-line syntax.
260 line_syntax.ok_or_else(|| ErrorKind::UndetectedSyntax("[unknown]".into()).into())
261 }
262 }
263 }
265 fn get_extension_syntax(&self, file_name: &OsStr) -> Option<&SyntaxReference> {
266 self.syntax_set
267 .find_syntax_by_extension(file_name.to_str().unwrap_or_default())
268 .or_else(|| {
269 self.syntax_set.find_syntax_by_extension(
270 Path::new(file_name)
271 .extension()
272 .and_then(|x| x.to_str())
273 .unwrap_or_default(),
274 )
275 })
276 }
278 fn get_first_line_syntax(&self, reader: &mut InputReader) -> Option<&SyntaxReference> {
279 String::from_utf8(reader.first_line.clone())
280 .ok()
281 .and_then(|l| self.syntax_set.find_syntax_by_first_line(&l))
282 }
285#[cfg(test)]
286mod tests {
287 use super::*;
289 use std::ffi::OsStr;
291 use std::fs::File;
292 use std::io::Write;
293 use tempdir::TempDir;
295 use crate::input::Input;
297 struct SyntaxDetectionTest<'a> {
298 assets: HighlightingAssets,
299 pub syntax_mapping: SyntaxMapping<'a>,
300 pub temp_dir: TempDir,
301 }
303 impl<'a> SyntaxDetectionTest<'a> {
304 fn new() -> Self {
305 SyntaxDetectionTest {
306 assets: HighlightingAssets::from_binary(),
307 syntax_mapping: SyntaxMapping::builtin(),
308 temp_dir: TempDir::new("bat_syntax_detection_tests")
309 .expect("creation of temporary directory"),
310 }
311 }
313 fn syntax_for_real_file_with_content_os(
314 &self,
315 file_name: &OsStr,
316 first_line: &str,
317 ) -> String {
318 let file_path = self.temp_dir.path().join(file_name);
319 {
320 let mut temp_file = File::create(&file_path).unwrap();
321 writeln!(temp_file, "{}", first_line).unwrap();
322 }
324 let input = Input::ordinary_file(file_path.as_os_str());
325 let dummy_stdin: &[u8] = &[];
326 let mut opened_input = input.open(dummy_stdin).unwrap();
328 self.assets
329 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
330 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
331 .name
332 .clone()
333 }
335 fn syntax_for_file_with_content_os(&self, file_name: &OsStr, first_line: &str) -> String {
336 let file_path = self.temp_dir.path().join(file_name);
337 let input = Input::from_reader(Box::new(BufReader::new(first_line.as_bytes())))
338 .with_name(Some(file_path.as_os_str()));
339 let dummy_stdin: &[u8] = &[];
340 let mut opened_input = input.open(dummy_stdin).unwrap();
342 self.assets
343 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
344 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
345 .name
346 .clone()
347 }
349 #[cfg(unix)]
350 fn syntax_for_file_os(&self, file_name: &OsStr) -> String {
351 self.syntax_for_file_with_content_os(file_name, "")
352 }
354 fn syntax_for_file_with_content(&self, file_name: &str, first_line: &str) -> String {
355 self.syntax_for_file_with_content_os(OsStr::new(file_name), first_line)
356 }
358 fn syntax_for_file(&self, file_name: &str) -> String {
359 self.syntax_for_file_with_content(file_name, "")
360 }
362 fn syntax_for_stdin_with_content(&self, file_name: &str, content: &[u8]) -> String {
363 let input = Input::stdin().with_name(Some(OsStr::new(file_name)));
364 let mut opened_input = input.open(content).unwrap();
366 self.assets
367 .get_syntax(None, &mut opened_input, &self.syntax_mapping)
368 .unwrap_or_else(|_| self.assets.syntax_set.find_syntax_plain_text())
369 .name
370 .clone()
371 }
373 fn syntax_is_same_for_inputkinds(&self, file_name: &str, content: &str) -> bool {
374 let as_file = self.syntax_for_real_file_with_content_os(file_name.as_ref(), content);
375 let as_reader = self.syntax_for_file_with_content_os(file_name.as_ref(), content);
376 let consistent = as_file == as_reader;
377 // TODO: Compare StdIn somehow?
379 if !consistent {
380 eprintln!(
381 "Inconsistent syntax detection:\nFor File: {}\nFor Reader: {}",
382 as_file, as_reader
383 )
384 }
386 consistent
387 }
388 }
390 #[test]
391 fn syntax_detection_basic() {
392 let test = SyntaxDetectionTest::new();
394 assert_eq!(test.syntax_for_file("test.rs"), "Rust");
395 assert_eq!(test.syntax_for_file("test.cpp"), "C++");
396 assert_eq!(test.syntax_for_file("test.build"), "NAnt Build File");
397 assert_eq!(
398 test.syntax_for_file("PKGBUILD"),
399 "Bourne Again Shell (bash)"
400 );
401 assert_eq!(test.syntax_for_file(".bashrc"), "Bourne Again Shell (bash)");
402 assert_eq!(test.syntax_for_file("Makefile"), "Makefile");
403 }
405 #[cfg(unix)]
406 #[test]
407 fn syntax_detection_invalid_utf8() {
408 use std::os::unix::ffi::OsStrExt;
410 let test = SyntaxDetectionTest::new();
412 assert_eq!(
413 test.syntax_for_file_os(OsStr::from_bytes(b"invalid_\xFEutf8_filename.rs")),
414 "Rust"
415 );
416 }
418 #[test]
419 fn syntax_detection_same_for_inputkinds() {
420 let mut test = SyntaxDetectionTest::new();
422 test.syntax_mapping
423 .insert("*.myext", MappingTarget::MapTo("C"))
424 .ok();
425 test.syntax_mapping
426 .insert("MY_FILE", MappingTarget::MapTo("Markdown"))
427 .ok();
429 assert!(test.syntax_is_same_for_inputkinds("Test.md", ""));
430 assert!(test.syntax_is_same_for_inputkinds("Test.txt", "#!/bin/bash"));
431 assert!(test.syntax_is_same_for_inputkinds(".bashrc", ""));
432 assert!(test.syntax_is_same_for_inputkinds("test.h", ""));
433 assert!(test.syntax_is_same_for_inputkinds("test.js", "#!/bin/bash"));
434 assert!(test.syntax_is_same_for_inputkinds("test.myext", ""));
435 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", ""));
436 assert!(test.syntax_is_same_for_inputkinds("MY_FILE", "<?php"));
437 }
439 #[test]
440 fn syntax_detection_well_defined_mapping_for_duplicate_extensions() {
441 let test = SyntaxDetectionTest::new();
443 assert_eq!(test.syntax_for_file("test.h"), "C++");
444 assert_eq!(test.syntax_for_file("test.sass"), "Sass");
445 assert_eq!(test.syntax_for_file("test.js"), "JavaScript (Babel)");
446 assert_eq!(test.syntax_for_file("test.fs"), "F#");
447 }
449 #[test]
450 fn syntax_detection_first_line() {
451 let test = SyntaxDetectionTest::new();
453 assert_eq!(
454 test.syntax_for_file_with_content("my_script", "#!/bin/bash"),
455 "Bourne Again Shell (bash)"
456 );
457 assert_eq!(
458 test.syntax_for_file_with_content("build", "#!/bin/bash"),
459 "Bourne Again Shell (bash)"
460 );
461 assert_eq!(
462 test.syntax_for_file_with_content("my_script", "<?php"),
463 "PHP"
464 );
465 }
467 #[test]
468 fn syntax_detection_with_custom_mapping() {
469 let mut test = SyntaxDetectionTest::new();
471 assert_eq!(test.syntax_for_file("test.h"), "C++");
472 test.syntax_mapping
473 .insert("*.h", MappingTarget::MapTo("C"))
474 .ok();
475 assert_eq!(test.syntax_for_file("test.h"), "C");
476 }
478 #[test]
479 fn syntax_detection_is_case_sensitive() {
480 let mut test = SyntaxDetectionTest::new();
482 assert_ne!(test.syntax_for_file("README.MD"), "Markdown");
483 test.syntax_mapping
484 .insert("*.MD", MappingTarget::MapTo("Markdown"))
485 .ok();
486 assert_eq!(test.syntax_for_file("README.MD"), "Markdown");
487 }
489 #[test]
490 fn syntax_detection_stdin_filename() {
491 let test = SyntaxDetectionTest::new();
493 // from file extension
494 assert_eq!(test.syntax_for_stdin_with_content("test.cpp", b"a"), "C++");
495 // from first line (fallback)
496 assert_eq!(
497 test.syntax_for_stdin_with_content("my_script", b"#!/bin/bash"),
498 "Bourne Again Shell (bash)"
499 );
500 }
502 #[cfg(unix)]
503 #[test]
504 fn syntax_detection_for_symlinked_file() {
505 use std::os::unix::fs::symlink;
507 let test = SyntaxDetectionTest::new();
508 let file_path = test.temp_dir.path().join("my_ssh_config_filename");
509 {
510 File::create(&file_path).unwrap();
511 }
512 let file_path_symlink = test.temp_dir.path().join(".ssh").join("config");
514 std::fs::create_dir(test.temp_dir.path().join(".ssh"))
515 .expect("creation of directory succeeds");
516 symlink(&file_path, &file_path_symlink).expect("creation of symbolic link succeeds");
518 let input = Input::ordinary_file(file_path_symlink.as_os_str());
519 let dummy_stdin: &[u8] = &[];
520 let mut opened_input = input.open(dummy_stdin).unwrap();
522 assert_eq!(
523 test.assets
524 .get_syntax(None, &mut opened_input, &test.syntax_mapping)
525 .unwrap_or_else(|_| test.assets.syntax_set.find_syntax_plain_text())
526 .name,
527 "SSH Config"
528 );
529 }