Add xpath filter

2024-09-11 05:35:53 +03:00 · 2023-06-13 16:33:32 +02:00 · 2023-06-13 16:33:32 +02:00 · c407a51324
commit c407a51324
parent 18115d6427
12 changed files with 139 additions and 38 deletions
--- a/docs/spec/hurl.grammar
+++ b/docs/spec/hurl.grammar
@ -418,6 +418,7 @@ filter:
  | to-int-filter
  | url-decode-filter
  | url-encode-filter
+  | xpath-filter

 count-filter: "count"

@ -449,6 +450,8 @@ url-decode-filter: "urlDecode"

 url-encode-filter: "urlEncode"

+xpath-filter: "xpath" sp quoted-string
+

 # Lexical Grammar

--- a/integration/tests_ok/hello_gb2312.html
+++ b/integration/tests_ok/hello_gb2312.html
@ -1,6 +1,7 @@
 <pre><code class="language-hurl"><span class="hurl-entry"><span class="request"><span class="line"></span><span class="comment"># In this test, the data returned by the server is encoded using GB2312.</span>
-<span class="line"></span><span class="comment"># The 'Content-Type' HTTP response header precise the charset 'gb2312' so</span>
-<span class="line"></span><span class="comment"># any text based assert are using GB2312 and can be used.</span>
+<span class="line"></span>
+<span class="line"></span><span class="comment"># The 'Content-Type' HTTP response header precise the charset 'gb2312'</span>
+<span class="line"></span><span class="comment"># so any text based assert are using GB2312 and can be used.</span>
 <span class="line"></span><span class="comment"># See the sibling fail test where there is no charset =&gt; tests_failed/hello_gb2312_failed.hurl</span>
 <span class="line"><span class="method">GET</span> <span class="url">http://localhost:8000/hello_gb2312</span></span>
 </span><span class="response"><span class="line"><span class="version">HTTP</span> <span class="number">200</span></span>
@ -8,5 +9,14 @@
 <span class="line"><span class="query-type">header</span> <span class="string">"Content-Type"</span> <span class="predicate-type">==</span> <span class="string">"text/html; charset=gb2312"</span></span>
 <span class="line"><span class="query-type">bytes</span> <span class="predicate-type">contains</span> hex,<span class="hex">c4e3bac3cac0bde7</span>;</span> <span class="comment"># 你好世界 encoded in GB2312</span>
 <span class="line"><span class="query-type">xpath</span> <span class="string">"string(//body)"</span> <span class="predicate-type">==</span> <span class="string">"你好世界"</span></span>
-</span></span><span class="line"></span>
-</code></pre>
+</span></span><span class="hurl-entry"><span class="request"><span class="line"></span>
+<span class="line"></span>
+<span class="line"></span><span class="comment"># The 'Content-Type' HTTP response header does not precise the charset 'gb2312'</span>
+<span class="line"></span><span class="comment"># so body must be decoded explicitly by Hurl before processing any text based assert</span>
+<span class="line"><span class="method">GET</span> <span class="url">http://localhost:8000/hello_gb2312_implicit</span></span>
+</span><span class="response"><span class="line"><span class="version">HTTP</span> <span class="number">200</span></span>
+<span class="line"><span class="section-header">[Asserts]</span></span>
+<span class="line"><span class="query-type">header</span> <span class="string">"Content-Type"</span> <span class="predicate-type">==</span> <span class="string">"text/html"</span></span>
+<span class="line"><span class="query-type">bytes</span> <span class="predicate-type">contains</span> hex,<span class="hex">c4e3bac3cac0bde7</span>;</span> <span class="comment"># 你好世界 encoded in GB2312</span>
+<span class="line"><span class="query-type">bytes</span> <span class="filter-type">decode</span> <span class="string">"gb2312"</span> <span class="filter-type">xpath</span> <span class="string">"string(//body)"</span> <span class="predicate-type">==</span> <span class="string">"你好世界"</span></span>
+</span></span></code></pre>
--- a/integration/tests_ok/hello_gb2312.hurl
+++ b/integration/tests_ok/hello_gb2312.hurl
@ -1,6 +1,7 @@
 # In this test, the data returned by the server is encoded using GB2312.
-# The 'Content-Type' HTTP response header precise the charset 'gb2312' so
-# any text based assert are using GB2312 and can be used.
+
+# The 'Content-Type' HTTP response header precise the charset 'gb2312'
+# so any text based assert are using GB2312 and can be used.
 # See the sibling fail test where there is no charset => tests_failed/hello_gb2312_failed.hurl
 GET http://localhost:8000/hello_gb2312
 HTTP 200
@ -9,3 +10,12 @@ header "Content-Type" == "text/html; charset=gb2312"
 bytes contains hex,c4e3bac3cac0bde7; # 你好世界 encoded in GB2312
 xpath "string(//body)" == "你好世界"

+
+# The 'Content-Type' HTTP response header does not precise the charset 'gb2312'
+# so body must be decoded explicitly by Hurl before processing any text based assert
+GET http://localhost:8000/hello_gb2312_implicit
+HTTP 200
+[Asserts]
+header "Content-Type" == "text/html"
+bytes contains hex,c4e3bac3cac0bde7; # 你好世界 encoded in GB2312
+bytes decode "gb2312" xpath "string(//body)" == "你好世界"
--- a/integration/tests_ok/hello_gb2312.json
+++ b/integration/tests_ok/hello_gb2312.json
@ -1 +1 @@
-{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/hello_gb2312"},"response":{"status":200,"asserts":[{"query":{"type":"header","name":"Content-Type"},"predicate":{"type":"equal","value":"text/html; charset=gb2312"}},{"query":{"type":"bytes"},"predicate":{"type":"contain","value":"xOO6w8rAvec=","encoding":"base64"}},{"query":{"type":"xpath","expr":"string(//body)"},"predicate":{"type":"equal","value":"你好世界"}}]}}]}
+{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/hello_gb2312"},"response":{"status":200,"asserts":[{"query":{"type":"header","name":"Content-Type"},"predicate":{"type":"equal","value":"text/html; charset=gb2312"}},{"query":{"type":"bytes"},"predicate":{"type":"contain","value":"xOO6w8rAvec=","encoding":"base64"}},{"query":{"type":"xpath","expr":"string(//body)"},"predicate":{"type":"equal","value":"你好世界"}}]}},{"request":{"method":"GET","url":"http://localhost:8000/hello_gb2312_implicit"},"response":{"status":200,"asserts":[{"query":{"type":"header","name":"Content-Type"},"predicate":{"type":"equal","value":"text/html"}},{"query":{"type":"bytes"},"predicate":{"type":"contain","value":"xOO6w8rAvec=","encoding":"base64"}},{"query":{"type":"bytes"},"filters":[{"type":"decode","encoding":"gb2312"},{"type":"toDate","expr":"string(//body)"}],"predicate":{"type":"equal","value":"你好世界"}}]}}]}
--- a/integration/tests_ok/hello_gb2312.py
+++ b/integration/tests_ok/hello_gb2312.py
@ -16,3 +16,19 @@ def hello_gb2312():
        "gb2312"
    )
    return Response(data, headers=headers)
+
+
+@app.route("/hello_gb2312_implicit")
+def hello_gb2312_implicit():
+    headers = {"Content-Type": "text/html"}
+    data = """<!DOCTYPE html>
+<html>
+    <head>
+        <meta http-equiv='Content-Type' content='text/html; charset=gb2312'>
+    </head>
+    <body>你好世界</body>
+</html>
+""".encode(
+        "gb2312"
+    )
+    return Response(data, headers=headers)
--- a/packages/hurl/src/runner/filter.rs
+++ b/packages/hurl/src/runner/filter.rs
@ -26,6 +26,7 @@ use percent_encoding::AsciiSet;
 use crate::html;
 use crate::runner::regex::eval_regex_value;
 use crate::runner::template::eval_template;
+use crate::runner::xpath;
 use crate::runner::{Error, RunnerError, Value};

 /// Apply successive `filters` to an input `value`.
@ -100,6 +101,9 @@ fn eval_filter(
        FilterValue::ToInt => eval_to_int(value, &filter.source_info, in_assert),
        FilterValue::UrlDecode => eval_url_decode(value, &filter.source_info, in_assert),
        FilterValue::UrlEncode => eval_url_encode(value, &filter.source_info, in_assert),
+        FilterValue::XPath { expr, .. } => {
+            eval_xpath(value, expr, variables, &filter.source_info, in_assert)
+        }
    }
 }

@ -447,6 +451,63 @@ fn eval_to_int(
    }
 }

+pub fn eval_xpath(
+    value: &Value,
+    expr: &Template,
+    variables: &HashMap<String, Value>,
+    source_info: &SourceInfo,
+    assert: bool,
+) -> Result<Option<Value>, Error> {
+    match value {
+        Value::String(xml) => {
+            // The filter will use the HTML parser that should also work with XML input
+            let is_html = true;
+            eval_xpath_string(xml, expr, variables, source_info, is_html)
+        }
+        v => Err(Error {
+            source_info: source_info.clone(),
+            inner: RunnerError::FilterInvalidInput(v._type()),
+            assert,
+        }),
+    }
+}
+
+pub fn eval_xpath_string(
+    xml: &str,
+    expr_template: &Template,
+    variables: &HashMap<String, Value>,
+    source_info: &SourceInfo,
+    is_html: bool,
+) -> Result<Option<Value>, Error> {
+    let expr = eval_template(expr_template, variables)?;
+    let result = if is_html {
+        xpath::eval_html(xml, &expr)
+    } else {
+        xpath::eval_xml(xml, &expr)
+    };
+    match result {
+        Ok(value) => Ok(Some(value)),
+        Err(xpath::XpathError::InvalidXml {}) => Err(Error {
+            source_info: source_info.clone(),
+            inner: RunnerError::QueryInvalidXml,
+            assert: false,
+        }),
+        Err(xpath::XpathError::InvalidHtml {}) => Err(Error {
+            source_info: source_info.clone(),
+            inner: RunnerError::QueryInvalidXml,
+            assert: false,
+        }),
+        Err(xpath::XpathError::Eval {}) => Err(Error {
+            source_info: expr_template.source_info.clone(),
+            inner: RunnerError::QueryInvalidXpathEval,
+            assert: false,
+        }),
+        Err(xpath::XpathError::Unsupported {}) => {
+            panic!("Unsupported xpath {expr}"); // good usecase for panic - I could not reproqduce this usecase myself
+        }
+    }
+}
+
 #[cfg(test)]
 pub mod tests {
    use chrono::offset::Utc;
--- a/packages/hurl/src/runner/query.rs
+++ b/packages/hurl/src/runner/query.rs
@ -22,9 +22,10 @@ use regex::Regex;
 use sha2::Digest;

 use crate::runner::core::{Error, RunnerError};
+use crate::runner::filter;
 use crate::runner::template::eval_template;
 use crate::runner::value::Value;
-use crate::runner::xpath;
+
 use crate::{http, jsonpath};

 pub type QueryResult = Result<Option<Value>, Error>;
@ -130,43 +131,16 @@ fn eval_query_xpath(
    response: &http::Response,
    expr: &Template,
    variables: &HashMap<String, Value>,
-    query_source_info: &SourceInfo,
+    source_info: &SourceInfo,
 ) -> QueryResult {
-    let expr_source_info = &expr.source_info;
-    let value = eval_template(expr, variables)?;
    match response.text() {
        Err(inner) => Err(Error {
-            source_info: query_source_info.clone(),
+            source_info: source_info.clone(),
            inner: RunnerError::from(inner),
            assert: false,
        }),
        Ok(xml) => {
-            let result = if response.is_html() {
-                xpath::eval_html(&xml, &value)
-            } else {
-                xpath::eval_xml(&xml, &value)
-            };
-            match result {
-                Ok(value) => Ok(Some(value)),
-                Err(xpath::XpathError::InvalidXml {}) => Err(Error {
-                    source_info: query_source_info.clone(),
-                    inner: RunnerError::QueryInvalidXml,
-                    assert: false,
-                }),
-                Err(xpath::XpathError::InvalidHtml {}) => Err(Error {
-                    source_info: query_source_info.clone(),
-                    inner: RunnerError::QueryInvalidXml,
-                    assert: false,
-                }),
-                Err(xpath::XpathError::Eval {}) => Err(Error {
-                    source_info: expr_source_info.clone(),
-                    inner: RunnerError::QueryInvalidXpathEval,
-                    assert: false,
-                }),
-                Err(xpath::XpathError::Unsupported {}) => {
-                    panic!("Unsupported xpath {value}"); // good usecase for panic - I could nmot reporduce this usecase myself
-                }
-            }
+            filter::eval_xpath_string(&xml, expr, variables, source_info, response.is_html())
        }
    }
 }
--- a/packages/hurl_core/src/ast/core.rs
+++ b/packages/hurl_core/src/ast/core.rs
@ -907,6 +907,10 @@ pub enum FilterValue {
    ToInt,
    UrlDecode,
    UrlEncode,
+    XPath {
+        space0: Whitespace,
+        expr: Template,
+    },
 }

 #[derive(Clone, Debug, PartialEq, Eq, Copy)]
--- a/packages/hurl_core/src/format/html.rs
+++ b/packages/hurl_core/src/format/html.rs
@ -935,6 +935,11 @@ impl HtmlFormatter {
            FilterValue::ToInt => self.fmt_span("filter-type", "toInt"),
            FilterValue::UrlDecode => self.fmt_span("filter-type", "urlDecode"),
            FilterValue::UrlEncode => self.fmt_span("filter-type", "urlEncode"),
+            FilterValue::XPath { space0, expr } => {
+                self.fmt_span("filter-type", "xpath");
+                self.fmt_space(space0);
+                self.fmt_template(expr);
+            }
        };
    }

--- a/packages/hurl_core/src/parser/filter.rs
+++ b/packages/hurl_core/src/parser/filter.rs
@ -66,6 +66,7 @@ pub fn filter(reader: &mut Reader) -> ParseResult<'static, Filter> {
            to_date_filter,
            url_decode_filter,
            url_encode_filter,
+            xpath_filter,
        ],
        reader,
    )
@ -183,6 +184,13 @@ fn url_decode_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> {
    Ok(FilterValue::UrlDecode)
 }

+fn xpath_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> {
+    try_literal("xpath", reader)?;
+    let space0 = one_or_more_spaces(reader)?;
+    let expr = quoted_template(reader).map_err(|e| e.non_recoverable())?;
+    Ok(FilterValue::XPath { space0, expr })
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
--- a/packages/hurlfmt/src/format/json.rs
+++ b/packages/hurlfmt/src/format/json.rs
@ -619,6 +619,10 @@ impl ToJson for FilterValue {
            FilterValue::ToInt => {
                attributes.push(("type".to_string(), JValue::String("toInt".to_string())));
            }
+            FilterValue::XPath { expr, .. } => {
+                attributes.push(("type".to_string(), JValue::String("toDate".to_string())));
+                attributes.push(("expr".to_string(), JValue::String(expr.to_string())));
+            }
        }
        JValue::Object(attributes)
    }
--- a/packages/hurlfmt/src/format/token.rs
+++ b/packages/hurlfmt/src/format/token.rs
@ -1243,6 +1243,12 @@ impl Tokenizable for Filter {
                tokens
            }
            FilterValue::ToInt => vec![Token::FilterType(String::from("toInt"))],
+            FilterValue::XPath { space0, expr } => {
+                let mut tokens: Vec<Token> = vec![Token::FilterType(String::from("xpath"))];
+                tokens.append(&mut space0.tokenize());
+                tokens.append(&mut expr.tokenize());
+                tokens
+            }
        }
    }
 }