From 9ec8e4ece692a569a9cb06ff16e7086b9a25e627 Mon Sep 17 00:00:00 2001
From: lepapareil <filipe.pinto@orange.com>
Date: Tue, 13 Jun 2023 11:27:02 +0200
Subject: [PATCH] Fix netcat and pip install

---
 .github/workflows/check.yml                   |  8 ++
 .github/workflows/package.yml                 | 80 ++++++++++++----
 .github/workflows/test.yml                    | 95 ++++++++++++++-----
 CONTRIBUTING.md                               |  4 +-
 bin/check/install_prerequisites.sh            |  2 +-
 bin/install_prerequisites_alpine.sh           |  1 -
 bin/install_prerequisites_archlinux.sh        |  3 +-
 bin/install_prerequisites_docker_archlinux.sh |  1 -
 bin/install_prerequisites_docker_fedora.sh    |  7 +-
 bin/install_prerequisites_docker_ubuntu.sh    |  3 +-
 bin/install_prerequisites_fedora.sh           |  1 -
 bin/install_prerequisites_macos.sh            |  1 -
 bin/install_prerequisites_ubuntu.sh           |  3 +-
 bin/install_python3_venv.sh                   |  9 ++
 bin/test/bench.sh                             |  2 +-
 bin/test/test_prerequisites.ps1               |  2 +-
 bin/test/test_prerequisites.sh                |  2 +-
 17 files changed, 163 insertions(+), 61 deletions(-)
 create mode 100755 bin/install_python3_venv.sh

diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 09c1421de..3e9b27e87 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -43,6 +43,14 @@ jobs:
     - name: Install prerequisites
       run: bin/check/install_prerequisites.sh
 
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
+
     - name: Check Rust version
       run: bin/check/rust_version.py 7
     
diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
index 29a7f2368..75b120274 100644
--- a/.github/workflows/package.yml
+++ b/.github/workflows/package.yml
@@ -26,6 +26,13 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install
       run: |
         bin/install_prerequisites_ubuntu.sh
@@ -77,20 +84,27 @@ jobs:
         run: |
           set -e
           echo "::group::Install system prerequisites"
-          bin/install_prerequisites_docker_ubuntu.sh
+            bin/install_prerequisites_docker_ubuntu.sh
+          echo "::endgroup::"
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Environment"
-          bin/environment.sh
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Install generic linux package"
-          bin/release/install_generic_linux_package.sh
+            bin/release/install_generic_linux_package.sh
           export PATH="/tmp/hurl-generic-linux:$PATH"
           echo "::endgroup::"
           echo "::group::Install tests prerequisites"
-          bin/test/test_prerequisites.sh
+            bin/test/test_prerequisites.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test_integ.sh
+            bin/test/test_integ.sh
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -121,20 +135,27 @@ jobs:
         run: |
           set -e
           echo "::group::Install system prerequisites"
-          bin/install_prerequisites_docker_archlinux.sh
+            bin/install_prerequisites_docker_archlinux.sh
+          echo "::endgroup::"
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Environment"
-          bin/environment.sh
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Install generic linux package"
-          bin/release/install_generic_linux_package.sh
+            bin/release/install_generic_linux_package.sh
           export PATH="/tmp/hurl-generic-linux:$PATH"
           echo "::endgroup::"
           echo "::group::Install tests prerequisites"
-          bin/test/test_prerequisites.sh
+            bin/test/test_prerequisites.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test_integ.sh
+            bin/test/test_integ.sh
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -165,20 +186,27 @@ jobs:
         run: |
           set -e
           echo "::group::Install system prerequisites"
-          bin/install_prerequisites_docker_fedora.sh
+            bin/install_prerequisites_docker_fedora.sh
+          echo "::endgroup::"
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Environment"
-          bin/environment.sh
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Install generic linux package"
-          bin/release/install_generic_linux_package.sh
+            bin/release/install_generic_linux_package.sh
           export PATH="/tmp/hurl-generic-linux:$PATH"
           echo "::group::Install tests prerequisites"
           echo "::endgroup::"
-          bin/test/test_prerequisites.sh
+            bin/test/test_prerequisites.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test_integ.sh || true # to be fixed => "error in stderr actual: <hurl: /lib64/libcurl.so.4: no version information available (required by hurl)"
+            bin/test/test_integ.sh || true # to be fixed => "error in stderr actual: <hurl: /lib64/libcurl.so.4: no version information available (required by hurl)"
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -199,10 +227,17 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
-    - name: Install
+    - name: Install Prerequisites
+      run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
       run: |
-        bin/install_prerequisites_ubuntu.sh
-        bin/install_rust.sh
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
+    - name: Install Rust
+      run: bin/install_rust.sh
     - name: Environment
       run: bin/environment.sh
     - name: Build
@@ -240,8 +275,17 @@ jobs:
         ref: ${{ inputs.branch }}
     - name: Install Prerequisites
       run: bin/install_prerequisites_macos.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Build
       run: |
         bin/release/release.sh
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7a33d6ec2..b2aa37ad0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -22,12 +22,19 @@ jobs:
       uses: actions/checkout@v3
       with:
         ref: ${{ inputs.branch }}
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH 
+        which python3
+        python3 --version
+        pip --version 
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Build
       run: |
         bin/release/release.sh
@@ -56,12 +63,19 @@ jobs:
       uses: actions/setup-python@v4                               
       with:                                                       
         python-version: '3.10'
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Test Prerequisites
       run: bin/test/test_prerequisites.sh
     - name: Tests units
@@ -95,12 +109,19 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.10'
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_ubuntu.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Test Prerequisites
       run: bin/test/test_prerequisites.sh
     - name: Tests units
@@ -135,16 +156,23 @@ jobs:
         run: |
           set -e
           echo "::group::Install Prerequisites"
-          bin/install_prerequisites_docker_archlinux.sh
+            bin/install_prerequisites_docker_archlinux.sh
           echo "::endgroup::"
-          echo "::group::Environment"
-          bin/environment.sh
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Install Rust"
-          bin/install_rust.sh
+            bin/install_rust.sh
+          echo "::endgroup::"
+          echo "::group::Environment"
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test.sh
+            bin/test/test.sh
           echo "::endgroup::"
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
@@ -169,16 +197,23 @@ jobs:
         run: |
           set -e
           echo "::group::Install Prerequisites"
-          bin/install_prerequisites_docker_fedora.sh
+            bin/install_prerequisites_docker_fedora.sh
           echo "::endgroup::"
-          echo "::group::Environment"
-          bin/environment.sh
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Install Rust"
-          bin/install_rust.sh
+            bin/install_rust.sh
+          echo "::endgroup::"
+          echo "::group::Environment"
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test.sh
+            bin/test/test.sh
           echo "::endgroup::"
     - name: find artifacts
       run: |
@@ -209,16 +244,23 @@ jobs:
           apk update --quiet
           apk add --quiet bash
           echo "::group::Install Prerequisites"
-          bin/install_prerequisites_alpine.sh
+            bin/install_prerequisites_alpine.sh
           echo "::endgroup::"
-          echo "::group::Environment"
-          bin/environment.sh
+          echo "::group::Install python3 venv"
+            bin/install_python3_venv.sh
+            export PATH=/tmp/hurl-python3-venv/bin:$PATH
+            which python3
+            python3 --version
+            pip --version
           echo "::endgroup::"
           echo "::group::Install Rust"
-          bin/install_rust.sh
+            bin/install_rust.sh
+          echo "::endgroup::"
+          echo "::group::Environment"
+            bin/environment.sh
           echo "::endgroup::"
           echo "::group::Tests"
-          bin/test/test.sh
+            bin/test/test.sh
           echo "::endgroup::"
     - name: find artifacts
       run: |
@@ -242,12 +284,19 @@ jobs:
       uses: actions/checkout@v3
       with:
         ref: ${{ inputs.branch }}
-    - name: Environment
-      run: bin/environment.sh
     - name: Install Prerequisites
       run: bin/install_prerequisites_macos.sh
+    - name: Install python3 venv
+      run: |
+        bin/install_python3_venv.sh
+        export PATH=/tmp/hurl-python3-venv/bin:$PATH
+        which python3
+        python3 --version
+        pip --version
     - name: Install Rust
       run: bin/install_rust.sh
+    - name: Environment
+      run: bin/environment.sh
     - name: Test Prerequisites
       run: |
         bin/test/test_prerequisites.sh
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 60eb5fc31..8b58d3417 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -67,7 +67,9 @@ by the tests suite:
 ```shell
 $ python3 -m venv .venv
 $ source .venv/bin/activate
-$ pip3 install --requirement bin/requirements-frozen.txt
+$ PATH=/tmp/hurl-python3-venv/bin:${PATH}
+$ export PATH
+$ pip install --break-system-packages --requirement bin/requirements-frozen.txt 
 ```
 
 Then, you can launch our local server (used to test Hurl features):
diff --git a/bin/check/install_prerequisites.sh b/bin/check/install_prerequisites.sh
index b8cf766e2..702d15de9 100755
--- a/bin/check/install_prerequisites.sh
+++ b/bin/check/install_prerequisites.sh
@@ -2,6 +2,6 @@
 set -Eeuo pipefail
 
 bin/install_rust.sh
-pip3 install -r bin/requirements-frozen.txt
+python3 -m pip install --requirement bin/requirements-frozen.txt
 sudo apt-get update && sudo apt-get install -y libxml2-utils
 
diff --git a/bin/install_prerequisites_alpine.sh b/bin/install_prerequisites_alpine.sh
index 5128c31e3..515a711ed 100755
--- a/bin/install_prerequisites_alpine.sh
+++ b/bin/install_prerequisites_alpine.sh
@@ -4,7 +4,6 @@ set -Eeuo pipefail
 echo "----- install prerequisite packages -----"
 apk update --quiet
 apk add --quiet sudo bash sudo curl curl-dev build-base libffi-dev libxml2-dev libxml2-utils openssl-dev python3 python3-dev py3-pip cargo squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 
diff --git a/bin/install_prerequisites_archlinux.sh b/bin/install_prerequisites_archlinux.sh
index a9244b000..b0db4ac86 100755
--- a/bin/install_prerequisites_archlinux.sh
+++ b/bin/install_prerequisites_archlinux.sh
@@ -3,7 +3,6 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 pacman -Syy --noconfirm
-pacman -Sy --noconfirm bash sudo curl icu base-devel libxml2 python3 glibc openbsd-netcat squid
-python3 get-pip.py
+pacman -Sy --noconfirm bash sudo curl icu base-devel libxml2 python3 python3-venv glibc openbsd-netcat squid
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
diff --git a/bin/install_prerequisites_docker_archlinux.sh b/bin/install_prerequisites_docker_archlinux.sh
index b0ed481e9..c5e4e45a8 100755
--- a/bin/install_prerequisites_docker_archlinux.sh
+++ b/bin/install_prerequisites_docker_archlinux.sh
@@ -4,7 +4,6 @@ set -Eeuo pipefail
 echo "----- install prerequisite packages -----"
 pacman -Syy --noconfirm
 pacman -Sy --noconfirm bash sudo python3 python-pip icu base-devel libxml2 glibc openbsd-netcat squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 
diff --git a/bin/install_prerequisites_docker_fedora.sh b/bin/install_prerequisites_docker_fedora.sh
index bc01c6524..e4175cdc9 100755
--- a/bin/install_prerequisites_docker_fedora.sh
+++ b/bin/install_prerequisites_docker_fedora.sh
@@ -2,10 +2,7 @@
 set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
-yum install -y sudo python38 procps gcc libxml2-devel openssl-devel libcurl-devel nc squid
-curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
-USER="$(whoami)"
-export USER
-python3 /tmp/get-pip.py
+yum install -y sudo which python3 python3-pip procps gcc libxml2-devel openssl-devel libcurl-devel nc squid
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
+
diff --git a/bin/install_prerequisites_docker_ubuntu.sh b/bin/install_prerequisites_docker_ubuntu.sh
index a4f5867db..2f995a68b 100755
--- a/bin/install_prerequisites_docker_ubuntu.sh
+++ b/bin/install_prerequisites_docker_ubuntu.sh
@@ -3,8 +3,7 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 apt-get update
-apt-get -y install bash sudo curl libcurl4-openssl-dev libxml2-utils libxml2-dev libssl-dev python3 python3-pip netcat squid
-python3 -m pip install --upgrade pip --quiet
+apt-get -y install bash sudo curl libcurl4-openssl-dev libxml2-utils libxml2-dev libssl-dev python3 python3-pip python3-venv python3-full netcat-openbsd squid
 sudo service squid stop
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
diff --git a/bin/install_prerequisites_fedora.sh b/bin/install_prerequisites_fedora.sh
index 12a786410..3b8429461 100755
--- a/bin/install_prerequisites_fedora.sh
+++ b/bin/install_prerequisites_fedora.sh
@@ -3,7 +3,6 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 yum install -y bash sudo procps gcc libxml2-devel openssl-devel libcurl-devel python3-devel python3-pip nc squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 
diff --git a/bin/install_prerequisites_macos.sh b/bin/install_prerequisites_macos.sh
index abc8c0993..7b64830b5 100755
--- a/bin/install_prerequisites_macos.sh
+++ b/bin/install_prerequisites_macos.sh
@@ -3,7 +3,6 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 brew install curl pkg-config squid
-python3 -m pip install --upgrade pip --quiet
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 
diff --git a/bin/install_prerequisites_ubuntu.sh b/bin/install_prerequisites_ubuntu.sh
index 1a4975f7e..bbea4cb9c 100755
--- a/bin/install_prerequisites_ubuntu.sh
+++ b/bin/install_prerequisites_ubuntu.sh
@@ -3,8 +3,7 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 sudo apt-get update
-sudo apt-get -y install bash libcurl4-openssl-dev libxml2-utils netcat python3 python3-pip net-tools squid
-python3 -m pip install --upgrade pip --quiet
+sudo apt-get -y install bash libcurl4-openssl-dev libxml2-utils netcat-openbsd python3 python3-pip python3-venv net-tools squid
 sudo service squid stop || true
 sudo squid -k shutdown || true
 sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
diff --git a/bin/install_python3_venv.sh b/bin/install_python3_venv.sh
new file mode 100755
index 000000000..901d0c2dd
--- /dev/null
+++ b/bin/install_python3_venv.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# shellcheck source=/dev/null
+set -Eeuo pipefail
+
+echo "----- install python3 venv -----"
+python3 -m venv /tmp/hurl-python3-venv
+source /tmp/hurl-python3-venv/bin/activate
+python3 -m pip install --upgrade pip --quiet
+
diff --git a/bin/test/bench.sh b/bin/test/bench.sh
index 4fcdb2fd8..d59bd0521 100755
--- a/bin/test/bench.sh
+++ b/bin/test/bench.sh
@@ -10,7 +10,7 @@ hurl --version
 hurlfmt --version
 
 # bench
-pip3 install --requirement bin/requirements-frozen.txt
+python3 -m pip install --requirement bin/requirements-frozen.txt
 cd bench
 python3 server.py > server.log 2>&1 &
 sleep 5
diff --git a/bin/test/test_prerequisites.ps1 b/bin/test/test_prerequisites.ps1
index 82ff4c322..3e0155983 100644
--- a/bin/test/test_prerequisites.ps1
+++ b/bin/test/test_prerequisites.ps1
@@ -7,7 +7,7 @@ $actual_dir=(Get-Location).Path
 $project_root_path=(Resolve-Path -LiteralPath $PSScriptRoot\..\..).path
 
 # install python libs
-pip3 install --requirement $project_root_path\bin\requirements-frozen.txt
+python3 -m pip install --requirement $project_root_path\bin\requirements-frozen.txt
 if ($LASTEXITCODE) { Throw }
 
 # start mock servers
diff --git a/bin/test/test_prerequisites.sh b/bin/test/test_prerequisites.sh
index de97eac68..d57c0874a 100755
--- a/bin/test/test_prerequisites.sh
+++ b/bin/test/test_prerequisites.sh
@@ -34,7 +34,7 @@ function cat_and_exit_err() {
 }
 
 echo "----- install servers prerequisites -----"
-pip3 install --requirement bin/requirements-frozen.txt
+python3 -m pip install --requirement bin/requirements-frozen.txt
 
 echo "----- start servers -----"
 cd integration