Accept input lines as raw bytes and decode in main loop

Cargo.lock

@@ -141,6 +141,11 @@ name = "box_drawing"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "bytelines"
+version = "2.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "byteorder"
 version = "1.3.2"
@@ -336,6 +341,7 @@ dependencies = [
  "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "box_drawing 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bytelines 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "console 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "error-chain 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -943,6 +949,7 @@ dependencies = [
 "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
 "checksum blake2b_simd 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b83b7baab1e671718d78204225800d6b170e648188ac7dc992e9d6bddf87d0c0"
 "checksum box_drawing 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ea27d8d5fd867b17523bf6788b1175fa9867f34669d057e9adaf76e27bcea44b"
+"checksum bytelines 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "791e4e40d13e1463dee537b254225c12c46ec7328f1817c6264873bc166f615f"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum cc 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)" = "ce400c638d48ee0e9ab75aef7997609ec57367ccfe1463f21bf53c3eca67bf46"
 "checksum cexpr 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"

Cargo.toml

@@ -20,6 +20,7 @@ ansi_colours = "1.0.1"
 ansi_term = "0.12.1"
 atty = "0.2.14"
 box_drawing = "0.1.2"
+bytelines = "2.2.2"
 console = "0.11.2"
 dirs = "2.0"
 lazy_static = "1.4"

src/delta.rs

@@ -51,13 +51,13 @@ impl State {
 // | HunkPlus    | flush, emit | flush, emit | flush, emit | flush, emit | flush, push | push     |
 
 pub fn delta<I>(
-    lines: I,
+    mut lines: I,
     config: &Config,
     assets: &HighlightingAssets,
     writer: &mut dyn Write,
 ) -> std::io::Result<()>
 where
-    I: Iterator<Item = String>,
+    I: Iterator<Item = std::io::Result<Vec<u8>>>,
 {
     let mut painter = Painter::new(writer, config, assets);
     let mut minus_file = "".to_string();
@@ -65,7 +65,8 @@ where
     let mut state = State::Unknown;
     let mut source = Source::Unknown;
 
-    for raw_line in lines {
+    while let Some(Ok(raw_line_bytes)) = lines.next() {
+        let raw_line = String::from_utf8_lossy(&raw_line_bytes);
         let line = strip_ansi_codes(&raw_line).to_string();
         if source == Source::Unknown {
             source = detect_source(&line);
@@ -633,7 +634,7 @@ mod tests {
         let config = cli::process_command_line_arguments(&assets, &options);
 
         delta(
-            input.split("\n").map(String::from),
+            input.split("\n").map(|s| Ok(s.as_bytes().to_vec())),
             &config,
             &assets,
             &mut writer,

src/main.rs

@@ -13,11 +13,12 @@ mod paint;
 mod parse;
 mod style;
 
-use std::io::{self, BufRead, ErrorKind, Read, Write};
+use std::io::{self, ErrorKind, Read, Write};
 use std::process;
 
 use ansi_term;
 use atty;
+use bytelines::ByteLinesReader;
 use structopt::StructOpt;
 use syntect::highlighting::{Color, FontStyle, Style};
 
@@ -62,10 +63,7 @@ fn main() -> std::io::Result<()> {
     let mut writer = output_type.handle().unwrap();
 
     if let Err(error) = delta(
-        io::stdin()
-            .lock()
-            .lines()
-            .map(|l| l.unwrap_or_else(|_| "<delta: invalid utf-8 data>".to_string())),
+        io::stdin().lock().byte_lines_iter(),
         &config,
         &assets,
         &mut writer,
@@ -123,9 +121,9 @@ fn get_painted_rgb_string(color: Color, true_color: bool) -> String {
 
 fn list_themes(assets: &HighlightingAssets) -> std::io::Result<()> {
     let opt = cli::Opt::from_args();
-    let mut input = String::new();
+    let mut input = Vec::new();
     if atty::is(atty::Stream::Stdin) {
-        input = "\
+        input = b"\
 diff --git a/example.rs b/example.rs
 index f38589a..0f1bb83 100644
 --- a/example.rs
@@ -140,9 +138,9 @@ index f38589a..0f1bb83 100644
 +    let result = f64::powf(num, 3.0);
 +    println!(\"The cube of {:.2} is {:.2}.\", num, result);
  }"
-        .to_string()
+        .to_vec()
     } else {
-        io::stdin().read_to_string(&mut input)?;
+        io::stdin().read_to_end(&mut input)?;
     }
 
     let stdout = io::stdout();
@@ -166,7 +164,7 @@ index f38589a..0f1bb83 100644
         let mut writer = output_type.handle().unwrap();
 
         if let Err(error) = delta(
-            input.split('\n').map(String::from),
+            input.split(|&b| b == b'\n').map(|line| Ok(line.to_vec())),
             &config,
             &assets,
             &mut writer,