Fix netcat and pip install

.github/workflows/check.yml

@@ -43,6 +43,14 @@ jobs:
     - name: Install prerequisites
       run: bin/check/install_prerequisites.sh
 
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
+
     - name: Check Rust version
       run: bin/check/rust_version.py 7
     

.github/workflows/package.yml

@@ -26,6 +26,13 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install
       run: |
         bin/install_prerequisites_ubuntu.sh
@@ -77,20 +84,27 @@ jobs:
         run: |
           set -e
           echo "::group::Install system prerequisites"
-          bin/install_prerequisites_docker_ubuntu.sh
+            bin/install_prerequisites_docker_ubuntu.sh
+          echo "::endgroup::"
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Environment"
-          bin/environment.sh
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Install generic linux package"
-          bin/release/install_generic_linux_package.sh
+            bin/release/install_generic_linux_package.sh
           export PATH="/tmp/hurl-generic-linux:$PATH"
           echo "::endgroup::"
           echo "::group::Install tests prerequisites"
-          bin/test/test_prerequisites.sh
+            bin/test/test_prerequisites.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test_integ.sh
+            bin/test/test_integ.sh
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -121,20 +135,27 @@ jobs:
         run: |
           set -e
           echo "::group::Install system prerequisites"
-          bin/install_prerequisites_docker_archlinux.sh
+            bin/install_prerequisites_docker_archlinux.sh
+          echo "::endgroup::"
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Environment"
-          bin/environment.sh
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Install generic linux package"
-          bin/release/install_generic_linux_package.sh
+            bin/release/install_generic_linux_package.sh
           export PATH="/tmp/hurl-generic-linux:$PATH"
           echo "::endgroup::"
           echo "::group::Install tests prerequisites"
-          bin/test/test_prerequisites.sh
+            bin/test/test_prerequisites.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test_integ.sh
+            bin/test/test_integ.sh
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -165,20 +186,27 @@ jobs:
         run: |
           set -e
           echo "::group::Install system prerequisites"
-          bin/install_prerequisites_docker_fedora.sh
+            bin/install_prerequisites_docker_fedora.sh
+          echo "::endgroup::"
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Environment"
-          bin/environment.sh
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Install generic linux package"
-          bin/release/install_generic_linux_package.sh
+            bin/release/install_generic_linux_package.sh
           export PATH="/tmp/hurl-generic-linux:$PATH"
           echo "::group::Install tests prerequisites"
           echo "::endgroup::"
-          bin/test/test_prerequisites.sh
+            bin/test/test_prerequisites.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test_integ.sh || true # to be fixed => "error in stderr actual: <hurl: /lib64/libcurl.so.4: no version information available (required by hurl)"
+            bin/test/test_integ.sh || true # to be fixed => "error in stderr actual: <hurl: /lib64/libcurl.so.4: no version information available (required by hurl)"
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -199,10 +227,17 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
-    - name: Install
+    - name: Install Prerequisites
+      run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
       run: |
-        bin/install_prerequisites_ubuntu.sh
-        bin/install_rust.sh
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
+    - name: Install Rust
+      run: bin/install_rust.sh
     - name: Environment
       run: bin/environment.sh
     - name: Build
@@ -240,8 +275,17 @@ jobs:
         ref: ${{ inputs.branch }}
     - name: Install Prerequisites
       run: bin/install_prerequisites_macos.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Build
       run: |
         bin/release/release.sh

.github/workflows/test.yml

@@ -22,12 +22,19 @@ jobs:
       uses: actions/checkout@v3
       with:
         ref: ${{ inputs.branch }}
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH 
+        which python3
+        python3 --version
+        pip --version 
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Build
       run: |
         bin/release/release.sh
@@ -56,12 +63,19 @@ jobs:
       uses: actions/setup-python@v4                               
       with:                                                       
         python-version: '3.10'
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Test Prerequisites
       run: bin/test/test_prerequisites.sh
     - name: Tests units
@@ -95,12 +109,19 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Test Prerequisites
       run: bin/test/test_prerequisites.sh
     - name: Tests units
@@ -135,16 +156,23 @@ jobs:
         run: |
           set -e
           echo "::group::Install Prerequisites"
-          bin/install_prerequisites_docker_archlinux.sh
+            bin/install_prerequisites_docker_archlinux.sh
           echo "::endgroup::"
-          echo "::group::Environment"
-          bin/environment.sh
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Install Rust"
-          bin/install_rust.sh
+            bin/install_rust.sh
+          echo "::endgroup::"
+          echo "::group::Environment"
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test.sh
+            bin/test/test.sh
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -169,16 +197,23 @@ jobs:
         run: |
           set -e
           echo "::group::Install Prerequisites"
-          bin/install_prerequisites_docker_fedora.sh
+            bin/install_prerequisites_docker_fedora.sh
           echo "::endgroup::"
-          echo "::group::Environment"
-          bin/environment.sh
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Install Rust"
-          bin/install_rust.sh
+            bin/install_rust.sh
+          echo "::endgroup::"
+          echo "::group::Environment"
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test.sh
+            bin/test/test.sh
           echo "::endgroup::"
     - name: find artifacts
       run: |
@@ -209,16 +244,23 @@ jobs:
           apk update --quiet
           apk add --quiet bash
           echo "::group::Install Prerequisites"
-          bin/install_prerequisites_alpine.sh
+            bin/install_prerequisites_alpine.sh
           echo "::endgroup::"
-          echo "::group::Environment"
-          bin/environment.sh
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Install Rust"
-          bin/install_rust.sh
+            bin/install_rust.sh
+          echo "::endgroup::"
+          echo "::group::Environment"
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test.sh
+            bin/test/test.sh
           echo "::endgroup::"
     - name: find artifacts
       run: |
@@ -242,12 +284,19 @@ jobs:
       uses: actions/checkout@v3
       with:
         ref: ${{ inputs.branch }}
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_macos.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Test Prerequisites
       run: |
         bin/test/test_prerequisites.sh

CONTRIBUTING.md

@@ -67,7 +67,9 @@ by the tests suite:
 ```shell
 $ python3 -m venv .venv
 $ source .venv/bin/activate
-$ pip3 install --requirement bin/requirements-frozen.txt
+$ PATH=/tmp/hurl-python3-venv/bin:${PATH}
+$ export PATH
+$ pip install --break-system-packages --requirement bin/requirements-frozen.txt 
 ```
 
 Then, you can launch our local server (used to test Hurl features):

bin/check/install_prerequisites.sh

@@ -2,6 +2,6 @@
 set -Eeuo pipefail
 
 bin/install_rust.sh
-pip3 install -r bin/requirements-frozen.txt
+python3 -m pip install --requirement bin/requirements-frozen.txt
 sudo apt-get update && sudo apt-get install -y libxml2-utils
 

bin/install_prerequisites_alpine.sh

@@ -4,7 +4,6 @@ set -Eeuo pipefail
 echo "----- install prerequisite packages -----"
 apk update --quiet
 apk add --quiet sudo bash sudo curl curl-dev build-base libffi-dev libxml2-dev libxml2-utils openssl-dev python3 python3-dev py3-pip cargo squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_archlinux.sh

@@ -3,7 +3,6 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 pacman -Syy --noconfirm
-pacman -Sy --noconfirm bash sudo curl icu base-devel libxml2 python3 glibc openbsd-netcat squid
-python3 get-pip.py
+pacman -Sy --noconfirm bash sudo curl icu base-devel libxml2 python3 python3-venv glibc openbsd-netcat squid
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true

bin/install_prerequisites_docker_archlinux.sh

@@ -4,7 +4,6 @@ set -Eeuo pipefail
 echo "----- install prerequisite packages -----"
 pacman -Syy --noconfirm
 pacman -Sy --noconfirm bash sudo python3 python-pip icu base-devel libxml2 glibc openbsd-netcat squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_docker_fedora.sh

@@ -2,10 +2,7 @@
 set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
-yum install -y sudo python38 procps gcc libxml2-devel openssl-devel libcurl-devel nc squid
-curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-USER="$(whoami)"
-export USER
-python3 /tmp/get-pip.py
+yum install -y sudo which python3 python3-pip procps gcc libxml2-devel openssl-devel libcurl-devel nc squid
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
+

bin/install_prerequisites_docker_ubuntu.sh

@@ -3,8 +3,7 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 apt-get update
-apt-get -y install bash sudo curl libcurl4-openssl-dev libxml2-utils libxml2-dev libssl-dev python3 python3-pip netcat squid
-python3 -m pip install --upgrade pip --quiet
+apt-get -y install bash sudo curl libcurl4-openssl-dev libxml2-utils libxml2-dev libssl-dev python3 python3-pip python3-venv python3-full netcat-openbsd squid
 sudo service squid stop
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true

bin/install_prerequisites_fedora.sh

@@ -3,7 +3,6 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 yum install -y bash sudo procps gcc libxml2-devel openssl-devel libcurl-devel python3-devel python3-pip nc squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_macos.sh

@@ -3,7 +3,6 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 brew install curl pkg-config squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_ubuntu.sh

@@ -3,8 +3,7 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 sudo apt-get update
-sudo apt-get -y install bash libcurl4-openssl-dev libxml2-utils netcat python3 python3-pip net-tools squid
-python3 -m pip install --upgrade pip --quiet
+sudo apt-get -y install bash libcurl4-openssl-dev libxml2-utils netcat-openbsd python3 python3-pip python3-venv net-tools squid
 sudo service squid stop || true
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true

bin/install_python3_venv.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# shellcheck source=/dev/null
+set -Eeuo pipefail
+
+echo "----- install python3 venv -----"
+python3 -m venv /tmp/hurl-python3-venv
+source /tmp/hurl-python3-venv/bin/activate
+python3 -m pip install --upgrade pip --quiet
+

bin/test/bench.sh

@@ -10,7 +10,7 @@ hurl --version
 hurlfmt --version
 
 # bench
-pip3 install --requirement bin/requirements-frozen.txt
+python3 -m pip install --requirement bin/requirements-frozen.txt
 cd bench
 python3 server.py > server.log 2>&1 &
 sleep 5

bin/test/test_prerequisites.ps1

@@ -7,7 +7,7 @@ $actual_dir=(Get-Location).Path
 $project_root_path=(Resolve-Path -LiteralPath $PSScriptRoot\..\..).path
 
 # install python libs
-pip3 install --requirement $project_root_path\bin\requirements-frozen.txt
+python3 -m pip install --requirement $project_root_path\bin\requirements-frozen.txt
 if ($LASTEXITCODE) { Throw }
 
 # start mock servers

bin/test/test_prerequisites.sh

@@ -34,7 +34,7 @@ function cat_and_exit_err() {
 }
 
 echo "----- install servers prerequisites -----"
-pip3 install --requirement bin/requirements-frozen.txt
+python3 -m pip install --requirement bin/requirements-frozen.txt
 
 echo "----- start servers -----"
 cd integration