Merge pull request #38 from Orange-OpenSource/feature/decompress

Decompress response body

Cargo.toml

@@ -31,6 +31,9 @@ float-cmp = "0.6.0"
 encoding = "0.2"
 chrono = "0.4.11"
 curl = "0.4.33"
+brotli="3.3.0"
+libflate = "1.0.2"
+
 
 #[dev-dependencies]
 proptest = "0.9.4"

integration/tests/compressed.hurl

@@ -1,3 +1,10 @@
+# -- COMPRESSED HAS NO EFFECT ON NON-COMPRESSED
+GET http://localhost:8000/compressed/none
+HTTP/1.0 200
+Content-Length: 12
+Content-Type: text/html; charset=utf-8
+```Hello World!```
+
 GET http://localhost:8000/compressed/gzip
 HTTP/1.0 200
 Content-Length: 32
@@ -5,9 +12,16 @@ Content-Encoding: gzip
 Content-Type: text/html; charset=utf-8
 ```Hello World!```
 
-# -- COMPRESSED HAS NO EFFECT ON NON-COMPRESSED
-GET http://localhost:8000/compressed/none
+GET http://localhost:8000/compressed/zlib
 HTTP/1.0 200
-Content-Length: 12
+Content-Length: 20
+Content-Encoding: deflate
+Content-Type: text/html; charset=utf-8
+```Hello World!```
+
+GET http://localhost:8000/compressed/brotli
+HTTP/1.0 200
+Content-Length: 16
+Content-Encoding: br
 Content-Type: text/html; charset=utf-8
 ```Hello World!```

integration/tests/compressed.py

@@ -5,16 +5,31 @@ from io import BytesIO
 @app.route("/compressed/gzip")
 def compressed_gzip():
     result = BytesIO()
-    # echo -n 'Hello World!' | gzip -f | hexdump -C
-    #result.write(b'\x1f\x8b\x08\x00\x0e\x2e\x83\x5f\x00\x03\xf3\x48\xcd\xc9\xc9\x57\x08\xcf\x2f\xca\x49\x51\xe4\x02\x00\xdd\xdd\x14\x7d\x0d')
-    #                1f 8b 08 00 ed 0c 84 5f 00 03 f3 48 cd c9 c9 57 08 cf 2f ca 49 51 04 00  a3 1c 29 1c 0c 00 00 00
     result.write(b'\x1f\x8b\x08\x00\xed\x0c\x84\x5f\x00\x03\xf3\x48\xcd\xc9\xc9\x57\x08\xcf\x2f\xca\x49\x51\x04\x00\xa3\x1c\x29\x1c\x0c\x00\x00\x00')
-
     data = result.getvalue()
     resp = make_response(data)
     resp.headers['Content-Encoding'] = 'gzip'
     return resp
 
+@app.route("/compressed/zlib")
+def compressed_zlib():
+    result = BytesIO()
+    result.write(b'\x78\x9c\xf3\x48\xcd\xc9\xc9\x57\x08\xcf\x2f\xca\x49\x51\x04\x00\x1c\x49\x04\x3e')
+    data = result.getvalue()
+    resp = make_response(data)
+    resp.headers['Content-Encoding'] = 'deflate'
+    return resp
+
+@app.route("/compressed/brotli")
+def compressed_brotli():
+    result = BytesIO()
+
+    result.write(b'\x21\x2c\x00\x04\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64\x21')
+    data = result.getvalue()
+    resp = make_response(data)
+    resp.headers['Content-Encoding'] = 'br'
+    return resp
+
 @app.route("/compressed/none")
 def compressed_none():
     return 'Hello World!'

integration/tests/error_assert_decompress.err

@@ -0,0 +1,7 @@
+error: Decompression Error
+  --> tests/error_assert_decompress.hurl:1:1
+   |
+ 1 | GET http://localhost:8000/error-assert-decompress
+   | ^ Could not uncompress response with gzip
+   |
+

integration/tests/error_assert_decompress.exit

@@ -0,0 +1 @@
+4

integration/tests/error_assert_decompress.hurl

@@ -0,0 +1,4 @@
+GET http://localhost:8000/error-assert-decompress
+HTTP/1.0 200
+```Hello World!```
+

integration/tests/error_assert_decompress.options

@@ -0,0 +1 @@
+--compressed

integration/tests/error_assert_decompressed.py

@@ -0,0 +1,8 @@
+from tests import app
+from flask import Response
+
+@app.route("/error-assert-decompress")
+def error_assert_decompress():
+    headers = {}
+    headers['Content-Encoding'] = 'gzip'
+    return Response('Hello', headers=headers)

integration/tests/error_output_decompress.err

@@ -0,0 +1 @@
+warning: Could not uncompress response with gzip

integration/tests/error_output_decompress.exit

@@ -0,0 +1 @@
+3

integration/tests/error_output_decompress.hurl

@@ -0,0 +1,4 @@
+GET http://localhost:8000/error-output-decompress
+HTTP/1.0 200
+
+

integration/tests/error_output_decompress.options

@@ -0,0 +1 @@
+--compressed

integration/tests/error_output_decompress.py

@@ -0,0 +1,8 @@
+from tests import app
+from flask import Response
+
+@app.route("/error-output-decompress")
+def error_output_decompress():
+    headers = {}
+    headers['Content-Encoding'] = 'gzip'
+    return Response('Hello', headers=headers)

integration/tests/error_output_decompressed.options

@@ -0,0 +1 @@
+--compressed

src/bin/hurl.rs

@@ -27,7 +27,9 @@ use atty::Stream;
 use chrono::{DateTime, Local};
 use clap::{AppSettings, ArgMatches};
 
+use hurl::ast::{Pos, SourceInfo};
 use hurl::cli;
+use hurl::cli::Error;
 use hurl::html;
 use hurl::http;
 use hurl::parser;
@@ -124,7 +126,6 @@ fn execute(
 
             let timeout = cli_options.timeout;
             let connect_timeout = cli_options.connect_timeout;
-            let compressed = cli_options.compressed;
             let options = http::ClientOptions {
                 follow_location,
                 max_redirect,
@@ -135,7 +136,6 @@ fn execute(
                 insecure,
                 timeout,
                 connect_timeout,
-                compressed,
             };
             let mut client = http::Client::init(options);
 
@@ -588,6 +588,7 @@ fn main() {
             }
             fs::read_to_string(filename).expect("Something went wrong reading the file")
         };
+
         let hurl_result = execute(
             filename,
             contents,
@@ -617,10 +618,32 @@ fn main() {
                         }
                         cli::log_info("");
                     }
-
+                    let body = if cli_options.compressed {
+                        match response.uncompress_body() {
+                            Ok(bytes) => bytes,
+                            Err(e) => {
+                                log_error_message(
+                                    false,
+                                    runner::Error {
+                                        source_info: SourceInfo {
+                                            start: Pos { line: 0, column: 0 },
+                                            end: Pos { line: 0, column: 0 },
+                                        },
+                                        inner: e,
+                                        assert: false,
+                                    }
+                                    .fixme()
+                                    .as_str(),
+                                );
+                                std::process::exit(3);
+                            }
+                        }
+                    } else {
+                        response.body
+                    };
                     unwrap_or_exit(
                         &log_error_message,
-                        write_output(response.body, matches.value_of("output")),
+                        write_output(body, matches.value_of("output")),
                     );
                 } else {
                     cli::log_info("no response has been received");

src/cli/error.rs

@@ -124,6 +124,8 @@ impl Error for runner::Error {
             RunnerError::AssertFailure { .. } => "Assert Failure".to_string(),
             RunnerError::UnrenderableVariable { .. } => "Unrenderable Variable".to_string(),
             RunnerError::NoQueryResult { .. } => "No query result".to_string(),
+            RunnerError::UnsupportedContentEncoding(..) => "Decompression Error".to_string(),
+            RunnerError::CouldNotUncompressResponse(..) => "Decompression Error".to_string(),
         }
     }
 
@@ -196,6 +198,12 @@ impl Error for runner::Error {
                 format!("value {} can not be rendered", value)
             }
             RunnerError::NoQueryResult { .. } => "The query didn't return any result".to_string(),
+            RunnerError::UnsupportedContentEncoding(algorithm) => {
+                format!("Compression {} is not supported", algorithm)
+            }
+            RunnerError::CouldNotUncompressResponse(algorithm) => {
+                format!("Could not uncompress response with {}", algorithm)
+            }
         }
     }
 }

src/cli/mod.rs

@@ -16,6 +16,7 @@
  *
  */
 
+pub use self::error::Error;
 pub use self::logger::{
     log_info, make_logger_error_message, make_logger_linter_error, make_logger_parser_error,
     make_logger_runner_error, make_logger_verbose,

src/http/client.rs

@@ -65,7 +65,6 @@ pub struct ClientOptions {
     pub insecure: bool,
     pub timeout: Duration,
     pub connect_timeout: Duration,
-    pub compressed: bool,
 }
 
 impl Client {
@@ -98,9 +97,6 @@ impl Client {
         h.timeout(options.timeout).unwrap();
         h.connect_timeout(options.connect_timeout).unwrap();
 
-        if options.compressed {
-            h.accept_encoding("br, gzip, deflate").unwrap();
-        }
         Client {
             handle: Box::new(h),
             follow_location: options.follow_location,

src/runner/content_decoding.rs

@@ -0,0 +1,211 @@
+/*
+ * hurl (https://hurl.dev)
+ * Copyright (C) 2020 Orange
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+///
+/// Uncompress body response
+/// using the Content-Encoding response header
+///
+use std::io::prelude::*;
+
+use crate::http;
+
+use super::core::RunnerError;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Encoding {
+    Brotli,
+    Gzip,
+    Deflate,
+    Identity,
+}
+
+impl http::Response {
+    fn content_encoding(&self) -> Result<Option<Encoding>, RunnerError> {
+        for header in self.headers.clone() {
+            if header.name.as_str().to_ascii_lowercase() == "content-encoding" {
+                return match header.value.as_str() {
+                    "br" => Ok(Some(Encoding::Brotli)),
+                    "gzip" => Ok(Some(Encoding::Gzip)),
+                    "deflate" => Ok(Some(Encoding::Deflate)),
+                    "identity" => Ok(Some(Encoding::Identity)),
+                    v => Err(RunnerError::UnsupportedContentEncoding(v.to_string())),
+                };
+            }
+        }
+        Ok(None)
+    }
+
+    pub fn uncompress_body(&self) -> Result<Vec<u8>, RunnerError> {
+        let encoding = self.content_encoding()?;
+        match encoding {
+            Some(Encoding::Identity) => Ok(self.body.clone()),
+            Some(Encoding::Gzip) => uncompress_gzip(&self.body[..]),
+            Some(Encoding::Deflate) => uncompress_zlib(&self.body[..]),
+            Some(Encoding::Brotli) => uncompress_brotli(&self.body[..]),
+            None => Ok(self.body.clone()),
+        }
+    }
+}
+
+fn uncompress_brotli(data: &[u8]) -> Result<Vec<u8>, RunnerError> {
+    let mut reader = brotli::Decompressor::new(data, 4096);
+    let mut buf = [0u8; 4096];
+    let n = match reader.read(&mut buf[..]) {
+        Err(_) => {
+            return Err(RunnerError::CouldNotUncompressResponse(
+                "brotli".to_string(),
+            ));
+        }
+        Ok(size) => size,
+    };
+    Ok(buf[..n].to_vec())
+}
+
+fn uncompress_gzip(data: &[u8]) -> Result<Vec<u8>, RunnerError> {
+    let mut decoder = match libflate::gzip::Decoder::new(data) {
+        Ok(v) => v,
+        Err(_) => return Err(RunnerError::CouldNotUncompressResponse("gzip".to_string())),
+    };
+    let mut buf = Vec::new();
+    match decoder.read_to_end(&mut buf) {
+        Ok(_) => Ok(buf),
+        Err(_) => Err(RunnerError::CouldNotUncompressResponse("gzip".to_string())),
+    }
+}
+
+fn uncompress_zlib(data: &[u8]) -> Result<Vec<u8>, RunnerError> {
+    let mut decoder = match libflate::zlib::Decoder::new(data) {
+        Ok(v) => v,
+        Err(_) => return Err(RunnerError::CouldNotUncompressResponse("zlib".to_string())),
+    };
+    let mut buf = Vec::new();
+    match decoder.read_to_end(&mut buf) {
+        Ok(_) => Ok(buf),
+        Err(_) => Err(RunnerError::CouldNotUncompressResponse("zlib".to_string())),
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+
+    #[test]
+    fn test_content_encoding() {
+        let response = http::Response {
+            version: http::Version::Http10,
+            status: 200,
+            headers: vec![],
+            body: vec![],
+        };
+        assert_eq!(response.content_encoding().unwrap(), None);
+
+        let response = http::Response {
+            version: http::Version::Http10,
+            status: 200,
+            headers: vec![http::Header {
+                name: "Content-Encoding".to_string(),
+                value: "xx".to_string(),
+            }],
+            body: vec![],
+        };
+        assert_eq!(
+            response.content_encoding().err().unwrap(),
+            RunnerError::UnsupportedContentEncoding("xx".to_string())
+        );
+
+        let response = http::Response {
+            version: http::Version::Http10,
+            status: 200,
+            headers: vec![http::Header {
+                name: "Content-Encoding".to_string(),
+                value: "br".to_string(),
+            }],
+            body: vec![],
+        };
+        assert_eq!(
+            response.content_encoding().unwrap().unwrap(),
+            Encoding::Brotli
+        );
+    }
+
+    #[test]
+    fn test_uncompress_body() {
+        let response = http::Response {
+            version: http::Version::Http10,
+            status: 200,
+            headers: vec![http::Header {
+                name: "Content-Encoding".to_string(),
+                value: "br".to_string(),
+            }],
+            body: vec![
+                0x21, 0x2c, 0x00, 0x04, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c,
+                0x64, 0x21,
+            ],
+        };
+        assert_eq!(response.uncompress_body().unwrap(), b"Hello World!");
+
+        let response = http::Response {
+            version: http::Version::Http10,
+            status: 200,
+            headers: vec![],
+            body: b"Hello World!".to_vec(),
+        };
+        assert_eq!(response.uncompress_body().unwrap(), b"Hello World!");
+    }
+
+    #[test]
+    fn test_uncompress_brotli() {
+        let data = vec![
+            0x21, 0x2c, 0x00, 0x04, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c,
+            0x64, 0x21,
+        ];
+        assert_eq!(uncompress_brotli(&data[..]).unwrap(), b"Hello World!");
+    }
+
+    #[test]
+    fn test_uncompress_gzip() {
+        let data = vec![
+            0x1f, 0x8b, 0x08, 0x08, 0xa7, 0x52, 0x85, 0x5f, 0x00, 0x03, 0x64, 0x61, 0x74, 0x61,
+            0x2e, 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x08, 0xcf, 0x2f,
+            0xca, 0x49, 0x51, 0x04, 0x00, 0xa3, 0x1c, 0x29, 0x1c, 0x0c, 0x00, 0x00, 0x00,
+        ];
+        assert_eq!(uncompress_gzip(&data[..]).unwrap(), b"Hello World!");
+    }
+
+    #[test]
+    fn test_uncompress_zlib() {
+        let data = vec![
+            0x78, 0x9c, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x08, 0xcf, 0x2f, 0xca, 0x49, 0x51,
+            0x04, 0x00, 0x1c, 0x49, 0x04, 0x3e,
+        ];
+        assert_eq!(uncompress_zlib(&data[..]).unwrap(), b"Hello World!");
+    }
+
+    #[test]
+    fn test_uncompress_error() {
+        let data = vec![0x21];
+        assert_eq!(
+            uncompress_brotli(&data[..]).err().unwrap(),
+            RunnerError::CouldNotUncompressResponse("brotli".to_string())
+        );
+        assert_eq!(
+            uncompress_gzip(&data[..]).err().unwrap(),
+            RunnerError::CouldNotUncompressResponse("gzip".to_string())
+        );
+    }
+}

src/runner/core.rs

@@ -130,6 +130,9 @@ pub enum RunnerError {
     CouldNotParseResponse,
     SSLCertificate,
 
+    UnsupportedContentEncoding(String),
+    CouldNotUncompressResponse(String),
+
     FileReadAccess {
         value: String,
     },

src/runner/http_response.rs

@@ -55,7 +55,8 @@ impl Response {
     ///
     pub fn text(&self) -> Result<String, RunnerError> {
         let encoding = self.encoding()?;
-        match encoding.decode(&self.body, DecoderTrap::Strict) {
+        let body = &self.uncompress_body()?;
+        match encoding.decode(body, DecoderTrap::Strict) {
             Ok(s) => Ok(s),
             Err(_) => Err(RunnerError::InvalidDecoding {
                 charset: encoding.name().to_string(),

src/runner/hurl_file.rs

@@ -61,7 +61,6 @@ use super::value::Value;
 ///        insecure: false,
 ///        timeout: Default::default(),
 ///        connect_timeout: Default::default(),
-///        compressed: false,
 /// };
 /// let mut client = http::Client::init(options);
 ///

src/runner/mod.rs

@@ -30,6 +30,7 @@ pub use self::log_deserialize::parse_results as deserialize_results;
 mod assert;
 mod body;
 mod capture;
+mod content_decoding;
 mod cookie;
 mod core;
 mod entry;

tests/libcurl.rs

@@ -65,7 +65,6 @@ fn default_client() -> Client {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Duration::from_secs(300),
-        compressed: false,
     };
     Client::init(options)
 }
@@ -307,7 +306,6 @@ fn test_follow_location() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let response = client.execute(&request, 0).unwrap();
@@ -341,7 +339,6 @@ fn test_max_redirect() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let request = default_get_request("http://localhost:8000/redirect".to_string());
@@ -430,67 +427,6 @@ fn test_post_bytes() {
 
 // endregion
 
-// region compressed
-#[test]
-fn test_compressed() {
-    let options = ClientOptions {
-        follow_location: false,
-        max_redirect: None,
-        cookie_input_file: None,
-        proxy: None,
-        no_proxy: None,
-        verbose: true,
-        insecure: false,
-        timeout: Default::default(),
-        connect_timeout: Duration::from_secs(300),
-        compressed: true,
-    };
-    let mut client = Client::init(options);
-    let request = Request {
-        method: Method::Get,
-        url: "http://localhost:8000/compressed/gzip".to_string(),
-        headers: vec![],
-        querystring: vec![],
-        form: vec![],
-        multipart: vec![],
-        cookies: vec![],
-        body: vec![],
-        content_type: None,
-    };
-    let response = client.execute(&request, 0).unwrap();
-    assert_eq!(response.status, 200);
-    assert_eq!(response.body, b"Hello World!".to_vec());
-    assert!(response.headers.contains(&Header {
-        name: "Content-Length".to_string(),
-        value: "32".to_string()
-    }));
-    assert!(response.headers.contains(&Header {
-        name: "Content-Encoding".to_string(),
-        value: "gzip".to_string()
-    }));
-
-    let request = Request {
-        method: Method::Get,
-        url: "http://localhost:8000/compressed/none".to_string(),
-        headers: vec![],
-        querystring: vec![],
-        form: vec![],
-        multipart: vec![],
-        cookies: vec![],
-        body: vec![],
-        content_type: None,
-    };
-    let response = client.execute(&request, 0).unwrap();
-    assert_eq!(response.status, 200);
-    assert_eq!(response.body, b"Hello World!".to_vec());
-    assert!(response.headers.contains(&Header {
-        name: "Content-Length".to_string(),
-        value: "12".to_string()
-    }));
-}
-
-// endregion
-
 // region error
 
 #[test]
@@ -519,7 +455,6 @@ fn test_error_fail_to_connect() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let request = default_get_request("http://localhost:8000/hello".to_string());
@@ -539,7 +474,6 @@ fn test_error_could_not_resolve_proxy_name() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let request = default_get_request("http://localhost:8000/hello".to_string());
@@ -559,7 +493,6 @@ fn test_timeout() {
         insecure: false,
         timeout: Duration::from_millis(100),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let request = default_get_request("http://localhost:8000/timeout".to_string());
@@ -581,7 +514,6 @@ fn test_connect_timeout() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Duration::from_secs(1),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let request = default_get_request("http://example.com:81".to_string());
@@ -684,7 +616,6 @@ fn test_cookie_file() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let request = default_get_request(
@@ -712,7 +643,6 @@ fn test_proxy() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = Client::init(options);
     let request = default_get_request("http://localhost:8000/hello".to_string());

tests/runner.rs

@@ -53,7 +53,6 @@ fn test_hurl_file() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = http::Client::init(options);
     let mut lines: Vec<&str> = regex::Regex::new(r"\n|\r\n")
@@ -160,7 +159,6 @@ fn test_hello() {
         insecure: false,
         timeout: Default::default(),
         connect_timeout: Default::default(),
-        compressed: false,
     };
     let mut client = http::Client::init(options);
     let source_info = SourceInfo {