git.iko.soy/bat
on master help sign in
Add support for UTF-16LE and UTF-16BE
commit25d96da4a216a5fc0766c28d077e7de57fb735a5 authorsharkdp <davidpeter@web.de> dateSun 7 Oct 2018, 16:44 (8 years ago) parente97095b
5 files changed+137−13
compare with master
unifiedsplit
Cargo.lock+65 Cargo.toml+1 src/inputfile.rs+47−2 src/main.rs+1 src/printer.rs+23−11
Cargo.lock+65
@@ -52,6 +52,7 @@ dependencies = [
5252 "console 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
5353 "content_inspector 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
5454 "directories 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
55+ "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
5556 "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
5657 "git2 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)",
5758 "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -166,6 +167,63 @@ dependencies = [
166167 "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
167168]
168169
170+[[package]]
171+name = "encoding"
172+version = "0.2.33"
173+source = "registry+https://github.com/rust-lang/crates.io-index"
174+dependencies = [
175+ "encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)",
176+ "encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)",
177+ "encoding-index-simpchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)",
178+ "encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)",
179+ "encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)",
180+]
181+
182+[[package]]
183+name = "encoding-index-japanese"
184+version = "1.20141219.5"
185+source = "registry+https://github.com/rust-lang/crates.io-index"
186+dependencies = [
187+ "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
188+]
189+
190+[[package]]
191+name = "encoding-index-korean"
192+version = "1.20141219.5"
193+source = "registry+https://github.com/rust-lang/crates.io-index"
194+dependencies = [
195+ "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
196+]
197+
198+[[package]]
199+name = "encoding-index-simpchinese"
200+version = "1.20141219.5"
201+source = "registry+https://github.com/rust-lang/crates.io-index"
202+dependencies = [
203+ "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
204+]
205+
206+[[package]]
207+name = "encoding-index-singlebyte"
208+version = "1.20141219.5"
209+source = "registry+https://github.com/rust-lang/crates.io-index"
210+dependencies = [
211+ "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
212+]
213+
214+[[package]]
215+name = "encoding-index-tradchinese"
216+version = "1.20141219.5"
217+source = "registry+https://github.com/rust-lang/crates.io-index"
218+dependencies = [
219+ "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
220+]
221+
222+[[package]]
223+name = "encoding_index_tests"
224+version = "0.1.4"
225+source = "registry+https://github.com/rust-lang/crates.io-index"
226+
169227[[package]]
170228name = "error-chain"
171229version = "0.12.0"
@@ -841,6 +899,13 @@ dependencies = [
841899"checksum console 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd48adf136733979b49e15bc3b4c43cc0d3c85ece7bd08e6daa414c6fcb13e6"
842900"checksum content_inspector 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d902e17eb0038a23c88baa0d78c75fac7968132e73f4fdb9ea77b03d2641b669"
843901"checksum directories 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b106a38a9bf6c763c6c2e2c3332ab7635da453a68a6babca776386b3b287d338"
902+"checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
903+"checksum encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
904+"checksum encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
905+"checksum encoding-index-simpchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
906+"checksum encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
907+"checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
908+"checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
844909"checksum error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07e791d3be96241c77c43846b665ef1384606da2cd2a48730abe606a12906e02"
845910"checksum flate2 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "37847f133aae7acf82bb9577ccd8bda241df836787642654286e79679826a54b"
846911"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
Cargo.toml+1
@@ -23,6 +23,7 @@ directories = "1.0"
2323lazy_static = "1.0"
2424wild = "2.0"
2525content_inspector = "0.2.2"
26+encoding = "0.2"
2627
2728[dependencies.git2]
2829version = "0.7"
src/inputfile.rs+47−2
@@ -1,6 +1,8 @@
11use std::fs::File;
22use std::io::{self, BufRead, BufReader};
33
4+use content_inspector::{self, ContentType};
5+
46use errors::*;
57
68const THEME_PREVIEW_FILE: &[u8] = include_bytes!("../assets/theme_preview.rs");
@@ -8,6 +10,7 @@ const THEME_PREVIEW_FILE: &[u8] = include_bytes!("../assets/theme_preview.rs");
810pub struct InputFileReader<'a> {
911 inner: Box<dyn BufRead + 'a>,
1012 pub first_line: Vec<u8>,
13+ pub content_type: ContentType,
1114}
1215
1316impl<'a> InputFileReader<'a> {
@@ -15,18 +18,31 @@ impl<'a> InputFileReader<'a> {
1518 let mut first_line = vec![];
1619 reader.read_until(b'\n', &mut first_line).ok();
1720
21+ let content_type = content_inspector::inspect(&first_line[..]);
22+
23+ if content_type == ContentType::UTF_16LE {
24+ reader.read_until(0x00, &mut first_line).ok();
25+ }
26+
1827 InputFileReader {
1928 inner: Box::new(reader),
2029 first_line,
30+ content_type,
2131 }
2232 }
2333
2434 pub fn read_line(&mut self, buf: &mut Vec<u8>) -> io::Result<bool> {
2535 if self.first_line.is_empty() {
26− self.inner.read_until(b'\n', buf).map(|size| size > 0)
36+ let res = self.inner.read_until(b'\n', buf).map(|size| size > 0)?;
37+
38+ if self.content_type == ContentType::UTF_16LE {
39+ self.inner.read_until(0x00, buf).ok();
40+ }
41+
42+ Ok(res)
2743 } else {
2844 buf.append(&mut self.first_line);
29− return Ok(true);
45+ Ok(true)
3046 }
3147 }
3248}
@@ -84,3 +100,32 @@ fn basic() {
84100 assert_eq!(false, res.unwrap());
85101 assert!(buffer.is_empty());
86102}
103+
104+#[test]
105+fn utf16le() {
106+ let content = b"\xFF\xFE\x73\x00\x0A\x00\x64\x00";
107+ let mut reader = InputFileReader::new(&content[..]);
108+
109+ assert_eq!(b"\xFF\xFE\x73\x00\x0A\x00", &reader.first_line[..]);
110+
111+ let mut buffer = vec![];
112+
113+ let res = reader.read_line(&mut buffer);
114+ assert!(res.is_ok());
115+ assert_eq!(true, res.unwrap());
116+ assert_eq!(b"\xFF\xFE\x73\x00\x0A\x00", &buffer[..]);
117+
118+ buffer.clear();
119+
120+ let res = reader.read_line(&mut buffer);
121+ assert!(res.is_ok());
122+ assert_eq!(true, res.unwrap());
123+ assert_eq!(b"\x64\x00", &buffer[..]);
124+
125+ buffer.clear();
126+
127+ let res = reader.read_line(&mut buffer);
128+ assert!(res.is_ok());
129+ assert_eq!(false, res.unwrap());
130+ assert!(buffer.is_empty());
131+}
src/main.rs+1
@@ -15,6 +15,7 @@ extern crate atty;
1515extern crate console;
1616extern crate content_inspector;
1717extern crate directories;
18+extern crate encoding;
1819extern crate git2;
1920extern crate syntect;
2021extern crate wild;
src/printer.rs+23−11
@@ -9,7 +9,10 @@ use console::AnsiCodeIterator;
99use syntect::easy::HighlightLines;
1010use syntect::highlighting::Theme;
1111
12−use content_inspector::{self, ContentType};
12+use content_inspector::ContentType;
13+
14+use encoding::all::{UTF_16BE, UTF_16LE};
15+use encoding::{DecoderTrap, Encoding};
1316
1417use app::Config;
1518use assets::HighlightingAssets;
@@ -121,12 +124,9 @@ impl<'a> InteractivePrinter<'a> {
121124 panel_width = 0;
122125 }
123126
124− // Determine file content type
125− let content_type = content_inspector::inspect(&reader.first_line[..]);
126−
127127 let mut line_changes = None;
128128
129− let highlighter = if content_type.is_binary() {
129+ let highlighter = if reader.content_type.is_binary() {
130130 None
131131 } else {
132132 // Get the Git modifications
@@ -149,8 +149,8 @@ impl<'a> InteractivePrinter<'a> {
149149 colors,
150150 config,
151151 decorations,
152+ content_type: reader.content_type,
152153 ansi_prefix_sgr: String::new(),
153− content_type,
154154 line_changes,
155155 highlighter,
156156 }
@@ -207,10 +207,11 @@ impl<'a> Printer for InteractivePrinter<'a> {
207207 _ => ("", "STDIN"),
208208 };
209209
210− let mode = if self.content_type.is_binary() {
211− " <BINARY>"
212− } else {
213− ""
210+ let mode = match self.content_type {
211+ ContentType::BINARY => " <BINARY>",
212+ ContentType::UTF_16LE => " <UTF-16LE>",
213+ ContentType::UTF_16BE => " <UTF-16BE>",
214+ _ => ""
214215 };
215216
216217 writeln!(
@@ -247,7 +248,18 @@ impl<'a> Printer for InteractivePrinter<'a> {
247248 line_number: usize,
248249 line_buffer: &[u8],
249250 ) -> Result<()> {
250− let line = String::from_utf8_lossy(&line_buffer).to_string();
251+ let line = match self.content_type {
252+ ContentType::BINARY => {
253+ return Ok(());
254+ }
255+ ContentType::UTF_16LE => UTF_16LE
256+ .decode(&line_buffer, DecoderTrap::Strict)
257+ .unwrap_or("Invalid UTF-16LE".into()),
258+ ContentType::UTF_16BE => UTF_16BE
259+ .decode(&line_buffer, DecoderTrap::Strict)
260+ .unwrap_or("Invalid UTF-16BE".into()),
261+ _ => String::from_utf8_lossy(&line_buffer).to_string(),
262+ };
251263 let regions = {
252264 let highlighter = match self.highlighter {
253265 Some(ref mut highlighter) => highlighter,