From 67d48bde222127514f3c432042e854696e167652 Mon Sep 17 00:00:00 2001 From: Fabrice Reix Date: Fri, 28 Oct 2022 07:58:18 +0200 Subject: [PATCH] Add filters urlEncode/urlDecode --- docs/spec/hurl.grammar | 8 +-- integration/tests_ok/filter.html | 19 ++++++ integration/tests_ok/filter.hurl | 18 ++++++ integration/tests_ok/filter.json | 1 + integration/tests_ok/filter.py | 11 ++++ integration/tests_ok/filter_count.html | 8 --- integration/tests_ok/filter_count.hurl | 7 -- integration/tests_ok/filter_count.json | 1 - integration/tests_ok/filter_count.py | 12 ---- packages/hurl/src/runner/filter.rs | 85 ++++++++++++++++++++++++- packages/hurl_core/src/ast/core.rs | 4 +- packages/hurl_core/src/format/html.rs | 6 +- packages/hurl_core/src/parser/filter.rs | 16 ++--- packages/hurl_core/src/parser/query.rs | 8 +-- packages/hurlfmt/src/format/json.rs | 11 ++-- packages/hurlfmt/src/format/token.rs | 4 +- 16 files changed, 158 insertions(+), 61 deletions(-) create mode 100644 integration/tests_ok/filter.html create mode 100644 integration/tests_ok/filter.hurl create mode 100644 integration/tests_ok/filter.json create mode 100644 integration/tests_ok/filter.py delete mode 100644 integration/tests_ok/filter_count.html delete mode 100644 integration/tests_ok/filter_count.hurl delete mode 100644 integration/tests_ok/filter_count.json delete mode 100644 integration/tests_ok/filter_count.py diff --git a/docs/spec/hurl.grammar b/docs/spec/hurl.grammar index 8be056973..08434fa85 100644 --- a/docs/spec/hurl.grammar +++ b/docs/spec/hurl.grammar @@ -409,16 +409,16 @@ variable-name: [A-Za-z] [A-Za-z_-0-9]* filter: regex-filter | count-filter - | escapeurl-filter - | unescapeurl-filter + | url-encode-filter + | url-decode-filter regex-filter: "regex" sp (quoted-string | regex) count-filter: "count" -escapeurl-filter: "escapeUrl" +url-encode-filter: "urlEncode" -unescapeurl-filter: "unescapeUrl" +url-decode-filter: "urlDecode" # Lexical Grammar diff --git a/integration/tests_ok/filter.html b/integration/tests_ok/filter.html new file mode 100644 index 000000000..5ff5f475c --- /dev/null +++ b/integration/tests_ok/filter.html @@ -0,0 +1,19 @@ +
GET http://localhost:8000/filter
+
+HTTP/1.0 200
+[Captures]
+url: jsonpath "$.url"
+[Asserts]
+jsonpath "$.list" count == 3
+jsonpath "$.message" regex /Hello (.*)!/ == "Bob"
+jsonpath "$.url" == "https://mozilla.org/?x=шеллы"
+jsonpath "$.url" urlEncode == "https%3A//mozilla.org/%3Fx%3D%D1%88%D0%B5%D0%BB%D0%BB%D1%8B"
+jsonpath "$.encoded_url" urlDecode == "https://mozilla.org/?x=шеллы"
+variable "url" urlEncode urlDecode == "{{url}}"
+{
+  "list": [1,2,3],
+  "message": "Hello Bob!",
+  "url": "https://mozilla.org/?x=шеллы",
+  "encoded_url": "https://mozilla.org/?x=%D1%88%D0%B5%D0%BB%D0%BB%D1%8B"
+}
+
diff --git a/integration/tests_ok/filter.hurl b/integration/tests_ok/filter.hurl new file mode 100644 index 000000000..8679f90d6 --- /dev/null +++ b/integration/tests_ok/filter.hurl @@ -0,0 +1,18 @@ +GET http://localhost:8000/filter + +HTTP/1.0 200 +[Captures] +url: jsonpath "$.url" +[Asserts] +jsonpath "$.list" count == 3 +jsonpath "$.message" regex /Hello (.*)!/ == "Bob" +jsonpath "$.url" == "https://mozilla.org/?x=шеллы" +jsonpath "$.url" urlEncode == "https%3A//mozilla.org/%3Fx%3D%D1%88%D0%B5%D0%BB%D0%BB%D1%8B" +jsonpath "$.encoded_url" urlDecode == "https://mozilla.org/?x=шеллы" +variable "url" urlEncode urlDecode == "{{url}}" +{ + "list": [1,2,3], + "message": "Hello Bob!", + "url": "https://mozilla.org/?x=шеллы", + "encoded_url": "https://mozilla.org/?x=%D1%88%D0%B5%D0%BB%D0%BB%D1%8B" +} diff --git a/integration/tests_ok/filter.json b/integration/tests_ok/filter.json new file mode 100644 index 000000000..4b09b1e80 --- /dev/null +++ b/integration/tests_ok/filter.json @@ -0,0 +1 @@ +{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/filter"},"response":{"version":"HTTP/1.0","status":200,"captures":[{"name":"url","query":{"type":"jsonpath","expr":"$.url"}}],"asserts":[{"query":{"type":"jsonpath","expr":"$.list"},"filters":[{"type":"count"}],"predicate":{"type":"equal","value":3}},{"query":{"type":"jsonpath","expr":"$.message"},"filters":[{"type":"regex","expr":{"type":"regex","value":"Hello (.*)!"}}],"predicate":{"type":"equal","value":"Bob"}},{"query":{"type":"jsonpath","expr":"$.url"},"predicate":{"type":"equal","value":"https://mozilla.org/?x=шеллы"}},{"query":{"type":"jsonpath","expr":"$.url"},"filters":[{"type":"urlEncode"}],"predicate":{"type":"equal","value":"https%3A//mozilla.org/%3Fx%3D%D1%88%D0%B5%D0%BB%D0%BB%D1%8B"}},{"query":{"type":"jsonpath","expr":"$.encoded_url"},"filters":[{"type":"urlDecode"}],"predicate":{"type":"equal","value":"https://mozilla.org/?x=шеллы"}},{"query":{"type":"variable","name":"url"},"filters":[{"type":"urlEncode"},{"type":"urlDecode"}],"predicate":{"type":"equal","value":"{{url}}"}}],"body":{"type":"json","value":{"list":[1,2,3],"message":"Hello Bob!","url":"https://mozilla.org/?x=шеллы","encoded_url":"https://mozilla.org/?x=%D1%88%D0%B5%D0%BB%D0%BB%D1%8B"}}}}]} diff --git a/integration/tests_ok/filter.py b/integration/tests_ok/filter.py new file mode 100644 index 000000000..5b8572708 --- /dev/null +++ b/integration/tests_ok/filter.py @@ -0,0 +1,11 @@ +from app import app + + +@app.route("/filter") +def filter(): + return """{ + "list": [1,2,3], + "message": "Hello Bob!", + "url": "https://mozilla.org/?x=шеллы", + "encoded_url": "https://mozilla.org/?x=%D1%88%D0%B5%D0%BB%D0%BB%D1%8B" +}""" diff --git a/integration/tests_ok/filter_count.html b/integration/tests_ok/filter_count.html deleted file mode 100644 index 67571fc59..000000000 --- a/integration/tests_ok/filter_count.html +++ /dev/null @@ -1,8 +0,0 @@ -
GET http://localhost:8000/filter-count
-HTTP/1.0 200
-[Asserts]
-jsonpath "$.users" count == 3
-jsonpath "$.users" count > 1
-jsonpath "$.users" count <= 10
-
-
diff --git a/integration/tests_ok/filter_count.hurl b/integration/tests_ok/filter_count.hurl deleted file mode 100644 index 8e5bb80ce..000000000 --- a/integration/tests_ok/filter_count.hurl +++ /dev/null @@ -1,7 +0,0 @@ -GET http://localhost:8000/filter-count -HTTP/1.0 200 -[Asserts] -jsonpath "$.users" count == 3 -jsonpath "$.users" count > 1 -jsonpath "$.users" count <= 10 - diff --git a/integration/tests_ok/filter_count.json b/integration/tests_ok/filter_count.json deleted file mode 100644 index e058ec942..000000000 --- a/integration/tests_ok/filter_count.json +++ /dev/null @@ -1 +0,0 @@ -{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/filter-count"},"response":{"version":"HTTP/1.0","status":200,"asserts":[{"query":{"type":"jsonpath","expr":"$.users"},"filters":[{"type":"count"}],"predicate":{"type":"equal","value":3}},{"query":{"type":"jsonpath","expr":"$.users"},"filters":[{"type":"count"}],"predicate":{"type":"greater","value":1}},{"query":{"type":"jsonpath","expr":"$.users"},"filters":[{"type":"count"}],"predicate":{"type":"less-or-equal","value":10}}]}}]} diff --git a/integration/tests_ok/filter_count.py b/integration/tests_ok/filter_count.py deleted file mode 100644 index b3ebdd1e8..000000000 --- a/integration/tests_ok/filter_count.py +++ /dev/null @@ -1,12 +0,0 @@ -from app import app -from flask import Response - - -@app.route("/filter-count") -def subquery_count(): - return Response( - """{ - "users": ["Bob", "Bill", "Bruce"] -}""", - mimetype="application/json", - ) diff --git a/packages/hurl/src/runner/filter.rs b/packages/hurl/src/runner/filter.rs index 90a8f590e..a510e6e63 100644 --- a/packages/hurl/src/runner/filter.rs +++ b/packages/hurl/src/runner/filter.rs @@ -18,6 +18,7 @@ use crate::runner::template::eval_template; use crate::runner::{Error, RunnerError, Value}; use hurl_core::ast::{Filter, FilterValue, RegexValue, SourceInfo}; +use percent_encoding::AsciiSet; use regex::Regex; use std::collections::HashMap; @@ -43,8 +44,8 @@ fn eval_filter( value: regex_value, .. } => eval_regex(value, regex_value, variables, &filter.source_info), FilterValue::Count {} => eval_count(value, &filter.source_info), - FilterValue::EscapeUrl { .. } => todo!(), - FilterValue::UnEscapeUrl { .. } => todo!(), + FilterValue::UrlEncode { .. } => eval_url_encode(value, &filter.source_info), + FilterValue::UrlDecode { .. } => eval_url_decode(value, &filter.source_info), } } @@ -108,6 +109,48 @@ fn eval_count(value: &Value, source_info: &SourceInfo) -> Result { } } +// does not encopde "/" +// like Jinja template (https://jinja.palletsprojects.com/en/3.1.x/templates/#jinja-filters.urlencode) +fn eval_url_encode(value: &Value, source_info: &SourceInfo) -> Result { + match value { + Value::String(value) => { + const FRAGMENT: &AsciiSet = &percent_encoding::NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~') + .remove(b'/'); + let encoded = percent_encoding::percent_encode(value.as_bytes(), FRAGMENT).to_string(); + Ok(Value::String(encoded)) + } + v => Err(Error { + source_info: source_info.clone(), + inner: RunnerError::FilterInvalidInput(v._type()), + assert: false, + }), + } +} + +fn eval_url_decode(value: &Value, source_info: &SourceInfo) -> Result { + match value { + Value::String(value) => { + match percent_encoding::percent_decode(value.as_bytes()).decode_utf8() { + Ok(decoded) => Ok(Value::String(decoded.to_string())), + Err(_) => Err(Error { + source_info: source_info.clone(), + inner: RunnerError::FilterInvalidInput("Invalid UTF8 stream".to_string()), + assert: false, + }), + } + } + v => Err(Error { + source_info: source_info.clone(), + inner: RunnerError::FilterInvalidInput(v._type()), + assert: false, + }), + } +} + #[cfg(test)] pub mod tests { use super::*; @@ -240,4 +283,42 @@ pub mod tests { assert_eq!(error.source_info, SourceInfo::new(1, 7, 1, 20)); assert_eq!(error.inner, RunnerError::InvalidRegex {}); } + + #[test] + pub fn eval_filter_url_encode() { + let variables = HashMap::new(); + let filter = Filter { + source_info: SourceInfo::new(0, 0, 0, 0), + value: FilterValue::UrlEncode {}, + }; + assert_eq!( + eval_filter( + &filter, + &Value::String("https://mozilla.org/?x=шеллы".to_string()), + &variables, + ) + .unwrap(), + Value::String( + "https%3A//mozilla.org/%3Fx%3D%D1%88%D0%B5%D0%BB%D0%BB%D1%8B".to_string() + ) + ); + } + + #[test] + pub fn eval_filter_url_decode() { + let variables = HashMap::new(); + let filter = Filter { + source_info: SourceInfo::new(0, 0, 0, 0), + value: FilterValue::UrlDecode {}, + }; + assert_eq!( + eval_filter( + &filter, + &Value::String("https://mozilla.org/?x=%D1%88%D0%B5%D0%BB%D0%BB%D1%8B".to_string()), + &variables, + ) + .unwrap(), + Value::String("https://mozilla.org/?x=шеллы".to_string()) + ); + } } diff --git a/packages/hurl_core/src/ast/core.rs b/packages/hurl_core/src/ast/core.rs index c932b554d..10b4e608e 100644 --- a/packages/hurl_core/src/ast/core.rs +++ b/packages/hurl_core/src/ast/core.rs @@ -810,6 +810,6 @@ pub enum FilterValue { space0: Whitespace, value: RegexValue, }, - EscapeUrl {}, - UnEscapeUrl {}, + UrlEncode {}, + UrlDecode {}, } diff --git a/packages/hurl_core/src/format/html.rs b/packages/hurl_core/src/format/html.rs index bf844be30..41378b598 100644 --- a/packages/hurl_core/src/format/html.rs +++ b/packages/hurl_core/src/format/html.rs @@ -1038,10 +1038,8 @@ impl Htmlable for FilterValue { buffer.push_str(value.to_html().as_str()); buffer } - FilterValue::EscapeUrl {} => "escapeUrl".to_string(), - FilterValue::UnEscapeUrl {} => { - "unescapeUrl".to_string() - } + FilterValue::UrlEncode {} => "urlEncode".to_string(), + FilterValue::UrlDecode {} => "urlDecode".to_string(), } } } diff --git a/packages/hurl_core/src/parser/filter.rs b/packages/hurl_core/src/parser/filter.rs index 3b7707872..e0899b885 100644 --- a/packages/hurl_core/src/parser/filter.rs +++ b/packages/hurl_core/src/parser/filter.rs @@ -52,8 +52,8 @@ pub fn filter(reader: &mut Reader) -> ParseResult<'static, Filter> { vec![ count_filter, regex_filter, - escape_url_filter, - unescape_url_filter, + url_encode_filter, + url_decode_filter, ], reader, ) @@ -87,14 +87,14 @@ fn regex_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> { Ok(FilterValue::Regex { space0, value }) } -fn escape_url_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> { - try_literal("escapeUrl", reader)?; - Ok(FilterValue::EscapeUrl {}) +fn url_encode_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> { + try_literal("urlEncode", reader)?; + Ok(FilterValue::UrlEncode {}) } -fn unescape_url_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> { - try_literal("unescapeUrl", reader)?; - Ok(FilterValue::UnEscapeUrl {}) +fn url_decode_filter(reader: &mut Reader) -> ParseResult<'static, FilterValue> { + try_literal("urlDecode", reader)?; + Ok(FilterValue::UrlDecode {}) } #[cfg(test)] diff --git a/packages/hurl_core/src/parser/query.rs b/packages/hurl_core/src/parser/query.rs index f1735b673..cff8e8c99 100644 --- a/packages/hurl_core/src/parser/query.rs +++ b/packages/hurl_core/src/parser/query.rs @@ -353,7 +353,7 @@ mod tests { #[test] fn test_query_with_filters() { - let mut reader = Reader::init("body unescapeUrl "); + let mut reader = Reader::init("body urlDecode "); assert_eq!( query(&mut reader).unwrap(), Query { @@ -369,11 +369,11 @@ mod tests { source_info: SourceInfo::new(1, 5, 1, 6) }, Filter { - source_info: SourceInfo::new(1, 6, 1, 17), - value: FilterValue::UnEscapeUrl {}, + source_info: SourceInfo::new(1, 6, 1, 15), + value: FilterValue::UrlDecode {}, } )] ); - assert_eq!(reader.state.cursor, 16); + assert_eq!(reader.state.cursor, 14); } } diff --git a/packages/hurlfmt/src/format/json.rs b/packages/hurlfmt/src/format/json.rs index 48aa307f1..2446319bb 100644 --- a/packages/hurlfmt/src/format/json.rs +++ b/packages/hurlfmt/src/format/json.rs @@ -517,14 +517,11 @@ impl ToJson for FilterValue { FilterValue::Count { .. } => { attributes.push(("type".to_string(), JValue::String("count".to_string()))); } - FilterValue::EscapeUrl { .. } => { - attributes.push(("type".to_string(), JValue::String("escapeUrl".to_string()))); + FilterValue::UrlEncode { .. } => { + attributes.push(("type".to_string(), JValue::String("urlEncode".to_string()))); } - FilterValue::UnEscapeUrl { .. } => { - attributes.push(( - "type".to_string(), - JValue::String("unescapeUrl".to_string()), - )); + FilterValue::UrlDecode { .. } => { + attributes.push(("type".to_string(), JValue::String("urlDecode".to_string()))); } } JValue::Object(attributes) diff --git a/packages/hurlfmt/src/format/token.rs b/packages/hurlfmt/src/format/token.rs index dbc335ef9..4712bdf4f 100644 --- a/packages/hurlfmt/src/format/token.rs +++ b/packages/hurlfmt/src/format/token.rs @@ -1086,8 +1086,8 @@ impl Tokenizable for Filter { tokens } FilterValue::Count { .. } => vec![Token::FilterType(String::from("count"))], - FilterValue::EscapeUrl { .. } => vec![Token::FilterType(String::from("escapeUrl"))], - FilterValue::UnEscapeUrl { .. } => vec![Token::FilterType(String::from("unescapeUrl"))], + FilterValue::UrlEncode { .. } => vec![Token::FilterType(String::from("urlEncode"))], + FilterValue::UrlDecode { .. } => vec![Token::FilterType(String::from("urlDecode"))], } } }