Merge pull request #152 from Orange-OpenSource/feature/support-utf8-with-bom

Support Hurl File encoded with UTF8 BOM

integration/test_echo.py

@@ -4,13 +4,14 @@
 #
 import sys
 import subprocess
+import codecs
 
 def test(hurl_file):
     cmd = ['hurlfmt', '--no-format', hurl_file]
     print(' '.join(cmd))
     result = subprocess.run(cmd, stdout=subprocess.PIPE)
-    expected = open(hurl_file).read()
-    actual = result.stdout.decode("utf-8") 
+    expected = codecs.open(hurl_file, encoding='utf-8-sig').read()  # Input file can be saved with a BOM
+    actual = result.stdout.decode("utf-8")
     if actual != expected:
         print('>>> error in stdout')
         print(f'actual: <{actual}>\nexpected: <{expected}>')

integration/tests/bom.exit

@@ -0,0 +1 @@
+0

integration/tests/bom.html

@@ -0,0 +1 @@
+<div class="hurl-file"><div class="hurl-entry"><div class="request"><span class="line"><span class="method">GET</span> <span class="url">http://localhost:8000/utf8_bom</span></span></div><div class="response"><span class="line"></span><span class="line"><span class="version">HTTP/1.0</span> <span class="status">200</span></span><span class="line">```Hello World!```</span></div></div><span class="line"></span></div>

integration/tests/bom.hurl

@@ -0,0 +1,5 @@
+GET http://localhost:8000/utf8_bom
+
+HTTP/1.0 200
+```Hello World!```
+

integration/tests/bom.json

@@ -0,0 +1 @@
+{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/utf8_bom"},"response":{"version":"HTTP/1.0","status":200,"body":{"type":"raw-string","value":"Hello World!"}}}]}

integration/tests/bom.py

@@ -0,0 +1,7 @@
+from tests import app
+from flask import request
+
+@app.route("/utf8_bom")
+def utf8_bom():
+    return 'Hello World!'
+

packages/hurl/src/cli/fs.rs

@@ -0,0 +1,102 @@
+/*
+ * hurl (https://hurl.dev)
+ * Copyright (C) 2020 Orange
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+use crate::cli::CLIError;
+use std::fs;
+use std::fs::File;
+use std::io::prelude::*;
+
+/// Remove BOM from the input bytes
+fn strip_bom(bytes: &mut Vec<u8>) {
+    if bytes.starts_with(&[0xefu8, 0xbb, 0xbf]) {
+        bytes.drain(0..3);
+    }
+}
+
+/// Similar to the standard read_to_string()
+/// But remove any existing BOM
+pub fn read_to_string(filename: &str) -> Result<String, CLIError> {
+    let mut f = match File::open(&filename) {
+        Ok(f) => f,
+        Err(e) => {
+            return Err(CLIError {
+                message: e.to_string(),
+            })
+        }
+    };
+    let metadata = fs::metadata(&filename).expect("unable to read metadata");
+    let mut buffer = vec![0; metadata.len() as usize];
+    if let Err(e) = f.read(&mut buffer) {
+        return Err(CLIError {
+            message: e.to_string(),
+        });
+    }
+    string_from_utf8(buffer)
+}
+
+pub fn string_from_utf8(buffer: Vec<u8>) -> Result<String, CLIError> {
+    let mut buffer = buffer;
+    strip_bom(&mut buffer);
+    match String::from_utf8(buffer) {
+        Ok(s) => Ok(s),
+        Err(e) => Err(CLIError {
+            message: e.to_string(),
+        }),
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+
+    #[test]
+    fn test_strip_bom() {
+        let mut bytes = vec![];
+        strip_bom(&mut bytes);
+        assert!(bytes.is_empty());
+
+        let mut bytes = vec![0xef, 0xbb, 0xbf, 0x68, 0x65, 0x6c, 0x6c, 0x6f];
+        strip_bom(&mut bytes);
+        assert_eq!(bytes, vec![0x68, 0x65, 0x6c, 0x6c, 0x6f]);
+
+        let mut bytes = vec![0x68, 0x65, 0x6c, 0x6c, 0x6f];
+        strip_bom(&mut bytes);
+        assert_eq!(bytes, vec![0x68, 0x65, 0x6c, 0x6c, 0x6f]);
+    }
+
+    #[test]
+    fn test_string_from_utf8_bom() {
+        let mut bytes = vec![];
+        strip_bom(&mut bytes);
+        assert_eq!(string_from_utf8(vec![]).unwrap(), "");
+        assert_eq!(
+            string_from_utf8(vec![0xef, 0xbb, 0xbf, 0x68, 0x65, 0x6c, 0x6c, 0x6f]).unwrap(),
+            "hello"
+        );
+        assert_eq!(
+            string_from_utf8(vec![0x68, 0x65, 0x6c, 0x6c, 0x6f]).unwrap(),
+            "hello"
+        );
+        assert_eq!(
+            string_from_utf8(vec![0xef]).err().unwrap(),
+            CLIError {
+                message: "incomplete utf-8 byte sequence from index 0".to_string()
+            }
+        );
+    }
+}

packages/hurl/src/cli/mod.rs

@@ -16,11 +16,18 @@
  *
  */
 
+pub use self::fs::read_to_string;
 pub use self::logger::{
     log_info, make_logger_error_message, make_logger_parser_error, make_logger_runner_error,
     make_logger_verbose,
 };
 
 mod color;
+mod fs;
 pub mod interactive;
 mod logger;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CLIError {
+    pub message: String,
+}

packages/hurl/src/main.rs

@@ -18,7 +18,6 @@
 
 use std::collections::HashMap;
 use std::env;
-use std::fs;
 use std::io::prelude::*;
 use std::io::{self, BufReader, Read};
 use std::path::{Path, PathBuf};
@@ -30,6 +29,7 @@ use clap::{AppSettings, ArgMatches};
 
 use hurl::cli;
 use hurl::cli::interactive;
+use hurl::cli::CLIError;
 use hurl::html;
 use hurl::http;
 use hurl::runner;
@@ -59,10 +59,6 @@ pub struct CLIOptions {
     pub user: Option<String>,
 }
 
-pub struct CLIError {
-    pub message: String,
-}
-
 fn execute(
     filename: &str,
     contents: String,
@@ -234,7 +230,7 @@ fn json_file(
         let results = if matches.is_present("append") && std::path::Path::new(&path).exists() {
             log_verbose(format!("Appending session to {}", path.display()).as_str());
 
-            let data = fs::read_to_string(path).unwrap();
+            let data = std::fs::read_to_string(path).unwrap();
             let v: serde_json::Value = match serde_json::from_str(data.as_str()) {
                 Ok(val) => val,
                 Err(_) => {
@@ -243,6 +239,7 @@ fn json_file(
                     });
                 }
             };
+
             match runner::deserialize_results(v) {
                 Err(msg) => {
                     return Err(CLIError {
@@ -669,17 +666,12 @@ fn main() {
             }
             contents
         } else {
-            match fs::read_to_string(filename) {
+            match cli::read_to_string(&filename) {
                 Ok(s) => s,
                 Err(e) => {
                     log_error_message(
                         false,
-                        format!(
-                            "Input file {} can not be read - {}",
-                            filename,
-                            e.to_string()
-                        )
-                        .as_str(),
+                        format!("Input stream can not be read - {}", e.message).as_str(),
                     );
                     std::process::exit(2);
                 }

packages/hurl/tests/runner.rs

@@ -17,6 +17,7 @@
  */
 extern crate hurl;
 
+use hurl::cli;
 use hurl::http;
 use hurl::runner;
 use hurl::runner::RunnerOptions;
@@ -37,8 +38,8 @@ pub fn log_runner_error(error: &runner::Error, _warning: bool) {
 // can be used for debugging
 #[test]
 fn test_hurl_file() {
-    let filename = "../../integration/tests/post_json.hurl";
-    let content = std::fs::read_to_string(filename).expect("Something went wrong reading the file");
+    let filename = "../../integration/tests/bom.hurl";
+    let content = cli::read_to_string(filename).expect("Something went wrong reading the file");
     let hurl_file = parser::parse_hurl_file(content.as_str()).unwrap();
     let variables = HashMap::new();
     let options = http::ClientOptions {

packages/hurlfmt/src/cli/fs.rs

@@ -0,0 +1,102 @@
+/*
+ * hurl (https://hurl.dev)
+ * Copyright (C) 2020 Orange
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+use crate::cli::CLIError;
+use std::fs;
+use std::fs::File;
+use std::io::prelude::*;
+
+/// Remove BOM from the input bytes
+fn strip_bom(bytes: &mut Vec<u8>) {
+    if bytes.starts_with(&[0xefu8, 0xbb, 0xbf]) {
+        bytes.drain(0..3);
+    }
+}
+
+/// Similar to the standard read_to_string()
+/// But remove any existing BOM
+pub fn read_to_string(filename: &str) -> Result<String, CLIError> {
+    let mut f = match File::open(&filename) {
+        Ok(f) => f,
+        Err(e) => {
+            return Err(CLIError {
+                message: e.to_string(),
+            })
+        }
+    };
+    let metadata = fs::metadata(&filename).expect("unable to read metadata");
+    let mut buffer = vec![0; metadata.len() as usize];
+    if let Err(e) = f.read(&mut buffer) {
+        return Err(CLIError {
+            message: e.to_string(),
+        });
+    }
+    string_from_utf8(buffer)
+}
+
+pub fn string_from_utf8(buffer: Vec<u8>) -> Result<String, CLIError> {
+    let mut buffer = buffer;
+    strip_bom(&mut buffer);
+    match String::from_utf8(buffer) {
+        Ok(s) => Ok(s),
+        Err(e) => Err(CLIError {
+            message: e.to_string(),
+        }),
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+
+    #[test]
+    fn test_strip_bom() {
+        let mut bytes = vec![];
+        strip_bom(&mut bytes);
+        assert!(bytes.is_empty());
+
+        let mut bytes = vec![0xef, 0xbb, 0xbf, 0x68, 0x65, 0x6c, 0x6c, 0x6f];
+        strip_bom(&mut bytes);
+        assert_eq!(bytes, vec![0x68, 0x65, 0x6c, 0x6c, 0x6f]);
+
+        let mut bytes = vec![0x68, 0x65, 0x6c, 0x6c, 0x6f];
+        strip_bom(&mut bytes);
+        assert_eq!(bytes, vec![0x68, 0x65, 0x6c, 0x6c, 0x6f]);
+    }
+
+    #[test]
+    fn test_string_from_utf8_bom() {
+        let mut bytes = vec![];
+        strip_bom(&mut bytes);
+        assert_eq!(string_from_utf8(vec![]).unwrap(), "");
+        assert_eq!(
+            string_from_utf8(vec![0xef, 0xbb, 0xbf, 0x68, 0x65, 0x6c, 0x6c, 0x6f]).unwrap(),
+            "hello"
+        );
+        assert_eq!(
+            string_from_utf8(vec![0x68, 0x65, 0x6c, 0x6c, 0x6f]).unwrap(),
+            "hello"
+        );
+        assert_eq!(
+            string_from_utf8(vec![0xef]).err().unwrap(),
+            CLIError {
+                message: "incomplete utf-8 byte sequence from index 0".to_string()
+            }
+        );
+    }
+}

packages/hurlfmt/src/cli/mod.rs

@@ -16,12 +16,18 @@
  *
  */
 
+pub use self::color::TerminalColor;
+pub use self::fs::read_to_string;
 pub use self::logger::{
     log_info, make_logger_error_message, make_logger_linter_error, make_logger_parser_error,
     make_logger_verbose,
 };
 
-pub use self::color::TerminalColor;
-
 mod color;
+mod fs;
 mod logger;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CLIError {
+    pub message: String,
+}

packages/hurlfmt/src/main.rs

@@ -17,7 +17,6 @@
  */
 extern crate clap;
 
-use std::fs;
 use std::io::Write;
 use std::io::{self, Read};
 use std::path::Path;
@@ -153,17 +152,12 @@ fn main() {
         }
         contents
     } else {
-        match fs::read_to_string(filename) {
+        match cli::read_to_string(&filename) {
             Ok(s) => s,
             Err(e) => {
                 log_error_message(
                     false,
-                    format!(
-                        "Input file {} can not be read - {}",
-                        filename,
-                        e.to_string()
-                    )
-                    .as_str(),
+                    format!("Input stream can not be read - {}", e.message).as_str(),
                 );
                 std::process::exit(2);
             }