Add decode filter

docs/spec/hurl.grammar

@@ -404,8 +404,9 @@ variable-name: [A-Za-z] [A-Za-z_-0-9]*
 
 filter:
     count-filter
-  | days-after-now
-  | days-before-now
+  | days-after-now-filter
+  | days-before-now-filter
+  | decode-filter
   | format-filter
   | html-escape-filter
   | html-unescape-filter
@@ -420,9 +421,11 @@ filter:
 
 count-filter: "count"
 
-days-after-now: "daysAfterNow"
+days-after-now-filter: "daysAfterNow"
 
-days-before-now: "daysBeforeNow"
+days-before-now-filter: "daysBeforeNow"
+
+decode-filter: "decode"
 
 format-filter: "format"
 

integration/tests_failed/filter_decode.err

@@ -0,0 +1,22 @@
+error: Filter Error
+  --> tests_failed/filter_decode.hurl:6:7
+   |
+ 6 | bytes decode "unknown" == "café"      # <unknown> encoding is not supported
+   |       ^^^^^^^^^^^^^^^^ <unknown> encoding is not supported
+   |
+
+error: Filter Error
+  --> tests_failed/filter_decode.hurl:7:7
+   |
+ 7 | bytes decode "arabic" == "café"       # value can not be decoded with <arabic> encoding
+   |       ^^^^^^^^^^^^^^^ value can not be decoded with <arabic> encoding
+   |
+
+error: Assert failure
+  --> tests_failed/filter_decode.hurl:8:0
+   |
+ 8 | bytes decode "iso-8859-1" == "café"   # value can be decoded but to an invalid string café
+   |   actual:   string <café>
+   |   expected: string <café>
+   |
+

integration/tests_failed/filter_decode.exit

@@ -0,0 +1 @@
+4

integration/tests_failed/filter_decode.hurl

@@ -0,0 +1,8 @@
+# café is utf8-encoded by the server
+GET http://localhost:8000/filter-decode
+HTTP 200
+[Asserts]
+bytes decode "utf-8" == "café"        # value is decoded with success
+bytes decode "unknown" == "café"      # <unknown> encoding is not supported
+bytes decode "arabic" == "café"       # value can not be decoded with <arabic> encoding
+bytes decode "iso-8859-1" == "café"   # value can be decoded but to an invalid string café

integration/tests_failed/filter_decode.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl tests_failed/filter_decode.hurl

integration/tests_failed/filter_decode.py

@@ -0,0 +1,8 @@
+from app import app
+from flask import Response
+
+
+@app.route("/filter-decode")
+def filter_decode():
+    data = """café""".encode("utf8")
+    return Response(data)

integration/tests_failed/filter_decode.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl tests_failed/filter_decode.hurl

integration/tests_ok/gb2312.exit

@@ -0,0 +1 @@
+0

integration/tests_ok/gb2312.html

@@ -0,0 +1,11 @@
+<pre><code class="language-hurl"><span class="hurl-entry"><span class="request"><span class="line"></span><span class="comment"># In this test, the data returned by the server is encoded using GB2312.</span>
+<span class="line"></span><span class="comment"># Meanwhile, the 'Content-Type' HTTP response header doesn't precise</span>
+<span class="line"></span><span class="comment"># any charset.</span>
+<span class="line"><span class="method">GET</span> <span class="url">http://localhost:8000/gb2312</span></span>
+</span><span class="response"><span class="line"><span class="version">HTTP</span> <span class="number">200</span></span>
+<span class="line"><span class="section-header">[Asserts]</span></span>
+<span class="line"><span class="query-type">header</span> <span class="string">"Content-Type"</span> <span class="predicate-type">==</span> <span class="string">"text/html"</span></span>
+<span class="line"><span class="query-type">bytes</span> <span class="predicate-type">contains</span> hex,<span class="hex">c4e3bac3cac0bde7</span>;</span> <span class="comment"># 你好世界 encoded in GB2312</span>
+<span class="line"><span class="query-type">bytes</span> <span class="filter-type">decode</span> <span class="string">"gb2312"</span> <span class="predicate-type">contains</span> <span class="string">"你好"</span></span>
+</span></span><span class="line"></span>
+</code></pre>

integration/tests_ok/gb2312.hurl

@@ -0,0 +1,10 @@
+# In this test, the data returned by the server is encoded using GB2312.
+# Meanwhile, the 'Content-Type' HTTP response header doesn't precise
+# any charset.
+GET http://localhost:8000/gb2312
+HTTP 200
+[Asserts]
+header "Content-Type" == "text/html"
+bytes contains hex,c4e3bac3cac0bde7; # 你好世界 encoded in GB2312
+bytes decode "gb2312" contains "你好"
+

integration/tests_ok/gb2312.json

@@ -0,0 +1 @@
+{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/gb2312"},"response":{"status":200,"asserts":[{"query":{"type":"header","name":"Content-Type"},"predicate":{"type":"equal","value":"text/html"}},{"query":{"type":"bytes"},"predicate":{"type":"contain","value":"xOO6w8rAvec=","encoding":"base64"}},{"query":{"type":"bytes"},"filters":[{"type":"decode","encoding":"gb2312"}],"predicate":{"type":"contain","value":"你好"}}]}}]}

integration/tests_ok/gb2312.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl tests_ok/gb2312.hurl

integration/tests_ok/gb2312.py

@@ -0,0 +1,18 @@
+from app import app
+from flask import Response
+
+
+@app.route("/gb2312")
+def gb2312():
+    headers = {"Content-Type": "text/html"}
+    data = """<!DOCTYPE html>
+<html>
+    <head>
+        <meta http-equiv='Content-Type' content='text/html; charset=gb2312'>
+    </head>
+    <body>你好世界</body>
+</html>
+""".encode(
+        "gb2312"
+    )
+    return Response(data, headers=headers)

integration/tests_ok/gb2312.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl tests_ok/gb2312.hurl

packages/hurl/src/runner/core.rs

@@ -192,4 +192,6 @@ pub enum RunnerError {
     FilterMissingInput {},
     FilterInvalidInput(String),
     FilterRegexNoCapture {},
+    FilterInvalidEncoding(String),
+    FilterDecode(String),
 }

packages/hurl/src/runner/error.rs

@@ -67,6 +67,8 @@ impl Error for runner::Error {
             RunnerError::FilterMissingInput { .. } => "Filter Error".to_string(),
             RunnerError::FilterInvalidInput { .. } => "Filter Error".to_string(),
             RunnerError::FilterRegexNoCapture { .. } => "Filter Error".to_string(),
+            RunnerError::FilterInvalidEncoding { .. } => "Filter Error".to_string(),
+            RunnerError::FilterDecode { .. } => "Filter Error".to_string(),
         }
     }
 
@@ -164,6 +166,12 @@ impl Error for runner::Error {
                 format!("invalid filter input: {message}")
             }
             RunnerError::FilterRegexNoCapture { .. } => "capture not found".to_string(),
+            RunnerError::FilterInvalidEncoding(encoding) => {
+                format!("<{encoding}> encoding is not supported")
+            }
+            RunnerError::FilterDecode(encoding) => {
+                format!("value can not be decoded with <{encoding}> encoding")
+            }
         }
     }
 }

packages/hurl/src/runner/filter.rs

@@ -18,6 +18,8 @@
 use std::collections::HashMap;
 
 use chrono::{NaiveDateTime, Utc};
+use encoding;
+use encoding::DecoderTrap;
 use hurl_core::ast::{Filter, FilterValue, RegexValue, SourceInfo, Template};
 use percent_encoding::AsciiSet;
 
@@ -51,6 +53,9 @@ fn eval_filter(
         FilterValue::Count => eval_count(value, &filter.source_info, in_assert),
         FilterValue::DaysAfterNow => eval_days_after_now(value, &filter.source_info, in_assert),
         FilterValue::DaysBeforeNow => eval_days_before_now(value, &filter.source_info, in_assert),
+        FilterValue::Decode { encoding, .. } => {
+            eval_decode(value, encoding, variables, &filter.source_info, in_assert)
+        }
         FilterValue::Format { fmt, .. } => {
             eval_format(value, fmt, variables, &filter.source_info, in_assert)
         }
@@ -171,6 +176,40 @@ fn eval_days_before_now(
     }
 }
 
+fn eval_decode(
+    value: &Value,
+    encoding_value: &Template,
+    variables: &HashMap<String, Value>,
+    source_info: &SourceInfo,
+    assert: bool,
+) -> Result<Value, Error> {
+    let encoding_value = eval_template(encoding_value, variables)?;
+    match value {
+        Value::Bytes(value) => {
+            match encoding::label::encoding_from_whatwg_label(encoding_value.as_str()) {
+                None => Err(Error {
+                    source_info: source_info.clone(),
+                    inner: RunnerError::FilterInvalidEncoding(encoding_value),
+                    assert,
+                }),
+                Some(enc) => match enc.decode(value, DecoderTrap::Strict) {
+                    Ok(decoded) => Ok(Value::String(decoded)),
+                    Err(_) => Err(Error {
+                        source_info: source_info.clone(),
+                        inner: RunnerError::FilterDecode(encoding_value),
+                        assert,
+                    }),
+                },
+            }
+        }
+        v => Err(Error {
+            source_info: source_info.clone(),
+            inner: RunnerError::FilterInvalidInput(v._type()),
+            assert,
+        }),
+    }
+}
+
 fn eval_format(
     value: &Value,
     fmt: &Template,

packages/hurl_core/src/ast/core.rs

@@ -861,6 +861,10 @@ pub enum FilterValue {
     Count,
     DaysAfterNow,
     DaysBeforeNow,
+    Decode {
+        space0: Whitespace,
+        encoding: Template,
+    },
     Format {
         space0: Whitespace,
         fmt: Template,

packages/hurl_core/src/format/html.rs

@@ -874,6 +874,11 @@ impl HtmlFormatter {
             FilterValue::Count => self.fmt_span("filter-type", "count"),
             FilterValue::DaysAfterNow => self.fmt_span("filter-type", "daysAfterNow"),
             FilterValue::DaysBeforeNow => self.fmt_span("filter-type", "daysBeforeNow"),
+            FilterValue::Decode { space0, encoding } => {
+                self.fmt_span("filter-type", "decode");
+                self.fmt_space(space0);
+                self.fmt_template(encoding);
+            }
             FilterValue::Format { space0, fmt } => {
                 self.fmt_span("filter-type", "format");
                 self.fmt_space(space0);

packages/hurl_core/src/parser/filter.rs

@@ -54,6 +54,7 @@ pub fn filter(reader: &mut Reader) -> ParseResult<'static, Filter> {
             count_filter,
             days_after_now_filter,
             days_before_now_filter,
+            decode_filter,
             format_filter,
             html_decode_filter,
             html_encode_filter,
@@ -101,6 +102,13 @@ fn days_before_now_filter(reader: &mut Reader) -> ParseResult<'static, FilterVal
     Ok(FilterValue::DaysBeforeNow)
 }
 
+fn decode_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> {
+    try_literal("decode", reader)?;
+    let space0 = one_or_more_spaces(reader)?;
+    let encoding = quoted_template(reader)?;
+    Ok(FilterValue::Decode { space0, encoding })
+}
+
 fn format_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> {
     try_literal("format", reader)?;
     let space0 = one_or_more_spaces(reader)?;

packages/hurlfmt/src/format/json.rs

@@ -565,6 +565,10 @@ impl ToJson for FilterValue {
                     JValue::String("daysBeforeNow".to_string()),
                 ));
             }
+            FilterValue::Decode { encoding, .. } => {
+                attributes.push(("type".to_string(), JValue::String("decode".to_string())));
+                attributes.push(("encoding".to_string(), JValue::String(encoding.to_string())));
+            }
             FilterValue::Format { fmt, .. } => {
                 attributes.push(("type".to_string(), JValue::String("format".to_string())));
                 attributes.push(("fmt".to_string(), JValue::String(fmt.to_string())));

packages/hurlfmt/src/format/token.rs

@@ -1165,6 +1165,12 @@ impl Tokenizable for Filter {
             FilterValue::Count => vec![Token::FilterType(String::from("count"))],
             FilterValue::DaysAfterNow => vec![Token::FilterType(String::from("daysAfterNow"))],
             FilterValue::DaysBeforeNow => vec![Token::FilterType(String::from("daysBeforeNow"))],
+            FilterValue::Decode { space0, encoding } => {
+                let mut tokens: Vec<Token> = vec![Token::FilterType(String::from("decode"))];
+                tokens.append(&mut space0.tokenize());
+                tokens.append(&mut encoding.tokenize());
+                tokens
+            }
             FilterValue::Format { space0, fmt } => {
                 let mut tokens: Vec<Token> = vec![Token::FilterType(String::from("format"))];
                 tokens.append(&mut space0.tokenize());