Migrate test integ SSL

integration/integration.py

@@ -29,15 +29,15 @@ def main():
     # Dynamic run (with server)
     [
         test_hurl.test(f)
-        for f in get_files("tests_ok/*.hurl")
-        + get_files("tests_failed/*.hurl")
-        + get_files("ssl/*.hurl")
+        for f in get_files("tests_ok/*.hurl") + get_files("tests_failed/*.hurl")
         if accept(f)
     ]
     # Run test scripts
     extension = "ps1" if platform.system() == "Windows" else "sh"
-    script_files = get_files("tests_ok/*." + extension) + get_files(
-        "tests_failed/*." + extension
+    script_files = (
+        get_files("tests_ok/*." + extension)
+        + get_files("tests_failed/*." + extension)
+        + get_files("ssl/*." + extension)
     )
     for f in sorted(script_files):
         test_script.test(f)

integration/ssl/cacert.options

@@ -1,4 +0,0 @@
---cacert
-ssl/ca/cert.pem
---ssl-no-revoke
-

integration/ssl/cacert.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl ssl/cacert.hurl --cacert ssl/ca/cert.pem --ssl-no-revoke --verbose

integration/ssl/cacert.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl ssl/cacert.hurl --cacert ssl/ca/cert.pem --verbose

integration/ssl/cacert_selfsigned.options

@@ -1,2 +0,0 @@
---cacert
-ssl/server/cert.selfsigned.pem

integration/ssl/cacert_selfsigned.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl ssl/cacert.hurl --cacert ssl/ca/cert.pem --ssl-no-revoke --verbose

integration/ssl/cacert_selfsigned.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl ssl/cacert_selfsigned.hurl --cacert ssl/server/cert.selfsigned.pem --verbose

integration/ssl/client_authentication.options

@@ -1,6 +0,0 @@
---cacert
-ssl/server/cert.selfsigned.pem
---cert
-ssl/client/cert.pem
---key
-ssl/client/key.pem

integration/ssl/client_authentication.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl ssl/client_authentication.hurl --cacert ssl/server/cert.selfsigned.pem --cert ssl/client/cert.pem --key ssl/client/key.pem --verbose || true

integration/ssl/client_authentication.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl ssl/client_authentication.hurl --cacert ssl/server/cert.selfsigned.pem --cert ssl/client/cert.pem --key ssl/client/key.pem --verbose

integration/ssl/error_self_signed_certificate.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl ssl/error_self_signed_certificate.hurl

integration/ssl/error_self_signed_certificate.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl ssl/error_self_signed_certificate.hurl

integration/ssl/insecure.options

@@ -1 +0,0 @@
---insecure

integration/ssl/insecure.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl ssl/insecure.hurl --insecure --verbose

integration/ssl/insecure.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl ssl/insecure.hurl --insecure --verbose

integration/ssl/letsencrypt.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl ssl/letsencrypt.hurl --verbose

integration/ssl/letsencrypt.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl ssl/letsencrypt.hurl --verbose

integration/ssl/options.ps1

@@ -0,0 +1,3 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl ssl/options.hurl --verbose

integration/ssl/options.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl ssl/options.hurl --verbose

integration/test_script.py

@@ -6,7 +6,7 @@ import sys
 import subprocess
 import os
 import argparse
-from typing import Optional
+import re
 
 
 def decode_string(encoded: bytes) -> str:
@@ -36,8 +36,19 @@ def test(script_file: str):
     print(cmd)
     result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-    # exit code
-    expected = get_expected_exit_code(script_file)
+    test_exit_code(os.path.splitext(script_file)[0] + ".exit", result)
+    test_stdout(os.path.splitext(script_file)[0] + ".out", result)
+    test_stdout_pattern(os.path.splitext(script_file)[0] + ".out.pattern", result)
+    test_stderr(os.path.splitext(script_file)[0] + ".err", result)
+    test_stderr_pattern(os.path.splitext(script_file)[0] + ".err.pattern", result)
+
+
+def test_exit_code(f, result) -> int:
+    """test exit code"""
+    if os.path.exists(f):
+        expected = int(open(f, encoding="utf-8").read().strip())
+    else:
+        expected = 0
     if result.returncode != expected:
         print(">>> error in return code")
         print(f"expected: {expected}  actual:{result.returncode}")
@@ -51,8 +62,16 @@ def test(script_file: str):
 
         sys.exit(1)
 
-    # stdout
-    expected = get_expected_stdout(script_file)
+    return expected
+
+
+def test_stdout(f, result):
+    """test stdout"""
+
+    if not os.path.exists(f):
+        return
+
+    expected = open(f, "rb").read()
     actual = result.stdout
     if actual != expected:
         print(">>> error in stdout")
@@ -60,26 +79,103 @@ def test(script_file: str):
         sys.exit(1)
 
 
-def get_expected_exit_code(script_file: str) -> int:
-    """Runs expected exit code for a given test script"""
+def test_stdout_pattern(f, result):
+    """test stdout with pattern lines"""
+    if not os.path.exists(f):
+        return
 
-    f = os.path.splitext(script_file)[0] + ".exit"
-    if os.path.exists(f):
-        expected = int(open(f, encoding="utf-8").read().strip())
-    else:
-        expected = 0
-    return expected
+    expected = open(f, encoding="utf-8").read()
+
+    # curl debug logs are too dependent on the context, so we filter
+    # them and not take them into account for testing differences.
+    expected = remove_curl_debug_lines(expected)
+    expected_lines = expected.split("\n")
+    expected_pattern_lines = [parse_pattern(line) for line in expected_lines]
+
+    actual = decode_string(result.stdout)
+    actual = remove_curl_debug_lines(actual)
+    actual_lines = re.split(r"\r?\n", actual)
+
+    if len(actual_lines) != len(expected_pattern_lines):
+        print(">>> error in stout / mismatch in number of lines")
+        print(
+            f"actual: {len(actual_lines)} lines\nexpected: {len(expected_pattern_lines)} lines"
+        )
+        sys.exit(1)
+    for i in range(len(expected_pattern_lines)):
+        if not re.match(expected_pattern_lines[i], actual_lines[i]):
+            print(f">>> error in stout in line {i+1}")
+            print(f"actual: <{actual_lines[i]}>")
+            print(
+                f"expected: <{expected_lines[i]}> (translated to regex <{expected_pattern_lines[i]}>)"
+            )
+            sys.exit(1)
 
 
-def get_expected_stdout(script_file: str) -> Optional[str]:
-    """Runs expected stdout for a given test script"""
+def test_stderr(f, result):
+    """test stderr"""
 
-    f = os.path.splitext(script_file)[0] + ".out"
-    if os.path.exists(f):
-        value = open(f, "rb").read()
-    else:
-        value = None
-    return value
+    if not os.path.exists(f):
+        return
+
+    expected = open(f, encoding="utf-8").read()
+    actual = decode_string(result.stderr)
+    if actual != expected:
+        print(">>> error in stderr")
+        print(f"actual  : <{actual}>\nexpected: <{expected}>")
+        sys.exit(1)
+
+
+def test_stderr_pattern(f, result):
+    """test stderr with pattern lines"""
+
+    if not os.path.exists(f):
+        return
+
+    expected = open(f, encoding="utf-8").read()
+
+    # curl debug logs are too dependent on the context, so we filter
+    # them and not take them into account for testing differences.
+    expected = remove_curl_debug_lines(expected)
+    expected_lines = expected.split("\n")
+    expected_pattern_lines = [parse_pattern(line) for line in expected_lines]
+
+    actual = decode_string(result.stderr)
+    actual = remove_curl_debug_lines(actual)
+    actual_lines = re.split(r"\r?\n", actual)
+
+    if len(actual_lines) != len(expected_pattern_lines):
+        print(">>> error in stderr / mismatch in number of lines")
+        print(
+            f"actual: {len(actual_lines)} lines\nexpected: {len(expected_pattern_lines)} lines"
+        )
+        sys.exit(1)
+    for i in range(len(expected_pattern_lines)):
+        if not re.match(expected_pattern_lines[i], actual_lines[i]):
+            print(f">>> error in stderr in line {i+1}")
+            print(f"actual: <{actual_lines[i]}>")
+            print(
+                f"expected: <{expected_lines[i]}> (translated to regex <{expected_pattern_lines[i]}>)"
+            )
+            sys.exit(1)
+
+
+def parse_pattern(s: str) -> str:
+    """Transform a stderr pattern to a regex"""
+    # Escape regex metacharacters
+    for c in ["\\", ".", "(", ")", "[", "]", "^", "$", "*", "+", "?", "|"]:
+        s = s.replace(c, "\\" + c)
+
+    s = re.sub("~+", ".*", s)
+    s = "^" + s + "$"
+    return s
+
+
+def remove_curl_debug_lines(text: str) -> str:
+    """Removes curl debug logs from text and returns the new text."""
+    lines = text.split("\n")
+    lines = [line for line in lines if not line.startswith("**")]
+    return "\n".join(lines)
 
 
 def main():