Use squid instead of mitmproxy on ci

.circleci/config.yml

@@ -8,49 +8,37 @@ jobs:
     steps:
       - checkout
       - run:
-          name: environment
-          command: bin/environment.sh
+          name: Environment
+          command: |
+            bin/environment.sh
       - run:
-          name: Build
+          name: Install prerequisites
           command: |
-            export CARGO_NET_GIT_FETCH_WITH_CLI=true
-            uname -a
-            lsb_release -a
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-            source $HOME/.cargo/env
-            rustc --version
-            cargo --version
+            bin/install_prerequisites_ubuntu.sh
             sudo apt update -qq
-            sudo apt install -y -qq libxml2-dev libxslt-dev libxml2-utils libcurl4-openssl-dev 
-            cargo build --release --verbose --locked --color always
-            target/release/hurl --version
-            curl --version
+            sudo apt install -y -qq libxml2-dev libxslt-dev libxml2-utils libcurl4-openssl-dev
+      - run:
+          name: Install rust
+          command: |
+            bin/install_rust.sh
       - run:
-          name: Tests units and integration tests
+          name: Tests units
           command: |
-            python3 -m pip install --upgrade pip --quiet
             bin/test/test_prerequisites.sh
-            echo "================= Tests units ================="
-            cargo build --verbose --locked --color always
-            cargo test --features strict --verbose --color always
-            mitmdump --version
-            pip3 install lxml bs4
-            echo "================= Integration tests ================="
-            export PATH="$PWD/target/debug:$PATH"
-            cd integration
-            python3 ./integration.py
-            ./test_curl_commands.sh $(find ./tests_ok -maxdepth 1 -type f -name '*.curl' ! -name '*windows*')
-            python3 ./test_html_output.py tests_ok/*.html
-            ./test_curl_commands.sh $(find ./tests_failed -maxdepth 1 -type f -name '*.curl' ! -name '*windows*')
-            python3 ./test_html_output.py tests_failed/*.html
-      - store_artifacts:
-          path: integration/mitmdump.log
-
-      - store_artifacts:
-          path: integration/server.log
-
+            bin/test/test_unit.sh
+      - run:
+          name: Build release
+          command: |
+            bin/release/release.sh
+      - run:
+          name: Integration tests
+          command: |
+            PATH="${PWD}/target/release:${PATH}"
+            export PATH
+            bin/test/test_prerequisites.sh
+            bin/test/test_integ.sh
       - store_artifacts:
-          path: integration/server-ssl.log
+          path: integration/build/*.log
 
 workflows:
   tests-arm64:

.github/workflows/package.yml

@@ -238,11 +238,12 @@ jobs:
       uses: actions/checkout@v3
       with:
         ref: ${{ inputs.branch }}
+    - name: Install Prerequisites
+      run: bin/install_prerequisites_macos.sh
     - name: Install Rust
       run: bin/install_rust.sh
     - name: Build
       run: |
-        brew install pkg-config
         bin/release/release.sh
         echo "PATH=:${PWD}/target/release:$PATH" >> "${GITHUB_ENV}"
     - name: Get version
@@ -284,8 +285,8 @@ jobs:
       uses: actions/cache@v3
       with:
         path: C:\vcpkg\installed
-        key: ${{ runner.os }}-release-windows-x64-${{ github.sha }}
-        restore-keys: ${{ runner.os }}-release-windows-x64-
+        key: ${{ runner.os }}-release-windows-x64
+        restore-keys: ${{ runner.os }}-release-windows-x64
     - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:
@@ -313,12 +314,6 @@ jobs:
         .\bin\release\install_windows64_installer.ps1
         .\bin\test\test_prerequisites.ps1
         .\bin\test\test_integ.ps1
-    - name: Archive logs artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: log-windows-x64-artifacts
-        path: |
-          .\**\*.log
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
       with:

.github/workflows/test.yml

@@ -52,6 +52,10 @@ jobs:
       uses: actions/checkout@v3
       with:
         ref: ${{ inputs.branch }}
+    - name: Install Python 3.10                                   
+      uses: actions/setup-python@v4                               
+      with:                                                       
+        python-version: '3.10'
     - name: Environment
       run: bin/environment.sh
     - name: Install Prerequisites
@@ -71,7 +75,7 @@ jobs:
       uses: actions/upload-artifact@v3
       if: ${{ always() }}
       with:
-        name: tests-ubuntu-${{ matrix.rust }}-artifacts
+        name: tests-ubuntu-latest-${{ matrix.rust }}-artifacts
         path: |
           ./**/*.log
           ./**/report/*.json
@@ -110,7 +114,7 @@ jobs:
       uses: actions/upload-artifact@v3
       if: ${{ always() }}
       with:
-        name: tests-ubuntu-${{ matrix.rust }}-artifacts
+        name: tests-ubuntu-20.04-${{ matrix.rust }}-artifacts
         path: |
           ./**/*.log
           ./**/report/*.json
@@ -188,45 +192,45 @@ jobs:
         path: |
           ./**/*.log
 
-#  test-docker-alpine-x64:
-#    runs-on: ubuntu-latest
-#    steps:
-#    - name: Checkout repository
-#      uses: actions/checkout@v3
-#      with:
-#        ref: ${{ inputs.branch }}
-#    - name: Build, Test units and Integration tests
-#      uses: addnab/docker-run-action@v3
-#      with:
-#        image: alpine
-#        options: --volume ${{ github.workspace }}:/work --workdir /work --privileged --env CARGO_TERM_COLOR=always
-#        run: |
-#          set -e
-#          apk update --quiet
-#          apk add --quiet bash
-#          echo "::group::Install Prerequisites"
-#          bin/install_prerequisites_alpine.sh
-#          echo "::endgroup::"
-#          echo "::group::Environment"
-#          bin/environment.sh
-#          echo "::endgroup::"
-#          echo "::group::Install Rust"
-#          bin/install_rust.sh
-#          echo "::endgroup::"
-#          echo "::group::Tests"
-#          bin/test/test.sh
-#          echo "::endgroup::"
-#    - name: find artifacts
-#      run: |
-#        pwd
-#        find . -name "*.log"
-#    - name: Archive production artifacts
-#      uses: actions/upload-artifact@v3
-#      if: ${{ always() }}
-#      with:
-#        name: tests-alpine-x64-artifacts
-#        path: |
-#          ./**/*.log
+  test-docker-alpine-x64:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ inputs.branch }}
+    - name: Build, Test units and Integration tests
+      uses: addnab/docker-run-action@v3
+      with:
+        image: alpine
+        options: --volume ${{ github.workspace }}:/work --workdir /work --privileged --env CARGO_TERM_COLOR=always
+        run: |
+          set -e
+          apk update --quiet
+          apk add --quiet bash
+          echo "::group::Install Prerequisites"
+          bin/install_prerequisites_alpine.sh
+          echo "::endgroup::"
+          echo "::group::Environment"
+          bin/environment.sh
+          echo "::endgroup::"
+          echo "::group::Install Rust"
+          bin/install_rust.sh
+          echo "::endgroup::"
+          echo "::group::Tests"
+          bin/test/test.sh
+          echo "::endgroup::"
+    - name: find artifacts
+      run: |
+        pwd
+        find . -name "*.log"
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v3
+      if: ${{ always() }}
+      with:
+        name: tests-alpine-x64-artifacts
+        path: |
+          ./**/*.log
 
   test-macos-x64:
     runs-on: macos-latest
@@ -240,10 +244,13 @@ jobs:
         ref: ${{ inputs.branch }}
     - name: Environment
       run: bin/environment.sh
+    - name: Install Prerequisites
+      run: bin/install_prerequisites_macos.sh
     - name: Install Rust
       run: bin/install_rust.sh
     - name: Test Prerequisites
-      run: bin/test/test_prerequisites.sh
+      run: |
+        bin/test/test_prerequisites.sh
     - name: Tests units
       run: |
         bin/test/test_unit.sh
@@ -283,8 +290,8 @@ jobs:
       uses: actions/cache@v3
       with:
         path: C:\vcpkg\installed
-        key: ${{ runner.os }}-test-windows-x64-${{ github.sha }}
-        restore-keys: ${{ runner.os }}-test-windows-x64-
+        key: ${{ runner.os }}-test-windows-x64
+        restore-keys: ${{ runner.os }}-test-windows-x64
     - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:

CONTRIBUTING.md

@@ -69,11 +69,12 @@ Then, you can launch our local server (used to test Hurl features):
 $ cd integration
 $ python3 server.py >server.log 2>&1 &
 $ python3 ssl/server.py >server-ssl.log 2>&1 &
-$ mitmdump --listen-host 127.0.0.1 --listen-port 8888 --modify-header "/From-Proxy/Hello" >mitmdump.log 2>&1 &
+$ squid_conf="http_access allow all\nhttp_port 3128\nrequest_header_add From-Proxy Hello\nreply_header_add From-Proxy Hello"
+$ (echo "${squid_conf}" | squid -d 2 -N -f /dev/stdin > proxy.log 2>&1) &
 $ jobs
 [1]    running    python3 server.py > server.log 2>&1
 [2]  - running    python3 ssl/server.py > server-ssl.log 2>&1
-[3]  + running    mitmdump --listen-host 127.0.0.1 --listen-port 8888 --modify-header  >  2>&1
+[3]  + running    echo "${squid_conf}" | squid -d 2 -N -f /dev/stdin > proxy.log 2>&1
 ```
 
 You can check [`bin/test/test_prerequisites.sh`] and [`bin/test/test_prerequisites.ps1`] for more details.

bin/check/install_prerequisites.sh

@@ -3,5 +3,5 @@ set -Eeuo pipefail
 
 bin/install_rust.sh
 pip3 install -r bin/requirements-frozen.txt
-sudo apt update && sudo apt install -y libxml2-utils
+sudo apt-get update && sudo apt-get install -y libxml2-utils
 

bin/environment.ps1

@@ -1,12 +1,12 @@
 Set-StrictMode -Version latest
 $ErrorActionPreference = 'Stop'
 
-powershell write-host -foregroundcolor Cyan "----- context -----"
+write-host -foregroundcolor Cyan "----- context -----"
 
 # get windows infos
 Get-ComputerInfo -Property WindowsProductName,WindowsVersion,OsHardwareAbstractionLayer
 
-# get powershell infos
+# get pwsh infos
 (Get-Variable PSVersionTable -ValueOnly).PSVersion
 
 # get vcpkg infos
@@ -20,3 +20,4 @@ if ($LASTEXITCODE) { Throw }
 # get cargo info
 cargo --version
 if ($LASTEXITCODE) { Throw }
+

bin/install_prerequisites_alpine.sh

@@ -3,7 +3,8 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 apk update --quiet
-apk add --quiet bash curl curl-dev build-base libffi-dev libxml2-dev libxml2-utils openssl-dev python3 python3-dev py3-pip cargo
-
+apk add --quiet sudo bash sudo curl curl-dev build-base libffi-dev libxml2-dev libxml2-utils openssl-dev python3 python3-dev py3-pip cargo squid
 python3 -m pip install --upgrade pip --quiet
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_archlinux.sh

@@ -3,7 +3,7 @@ set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
 pacman -Syy --noconfirm
-pacman -Sy --noconfirm bash curl icu base-devel libxml2 python3 glibc openbsd-netcat
-
+pacman -Sy --noconfirm bash sudo curl icu base-devel libxml2 python3 glibc openbsd-netcat squid
 python3 get-pip.py
-
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true

bin/install_prerequisites_docker_archlinux.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
 set -Eeuo pipefail
 
-# Install packages
+echo "----- install prerequisite packages -----"
 pacman -Syy --noconfirm
-pacman -Sy --noconfirm bash python3 python-pip icu base-devel libxml2 glibc openbsd-netcat
+pacman -Sy --noconfirm bash sudo python3 python-pip icu base-devel libxml2 glibc openbsd-netcat squid
 python3 -m pip install --upgrade pip --quiet
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_docker_fedora.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 set -Eeuo pipefail
 
-# Install packages
-yum install -y python38 procps gcc libxml2-devel openssl-devel libcurl-devel nc
+echo "----- install prerequisite packages -----"
+yum install -y sudo python38 procps gcc libxml2-devel openssl-devel libcurl-devel nc squid
 curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
 USER="$(whoami)"
 export USER
 python3 /tmp/get-pip.py
-
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true

bin/install_prerequisites_docker_ubuntu.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 set -Eeuo pipefail
 
-# Install packages
-apt update
-apt -y install bash curl sudo libcurl4-openssl-dev libxml2-utils libxml2-dev libssl-dev python3 python3-pip netcat
+echo "----- install prerequisite packages -----"
+apt-get update
+apt-get -y install bash sudo curl libcurl4-openssl-dev libxml2-utils libxml2-dev libssl-dev python3 python3-pip netcat squid
 python3 -m pip install --upgrade pip --quiet
+sudo service squid stop
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_fedora.sh

@@ -2,6 +2,8 @@
 set -Eeuo pipefail
 
 echo "----- install prerequisite packages -----"
-yum install -y bash procps gcc libxml2-devel openssl-devel libcurl-devel python3-devel python3-pip nc
+yum install -y bash sudo procps gcc libxml2-devel openssl-devel libcurl-devel python3-devel python3-pip nc squid
 python3 -m pip install --upgrade pip --quiet
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
 

bin/install_prerequisites_macos.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+echo "----- install prerequisite packages -----"
+brew install curl pkg-config squid
+python3 -m pip install --upgrade pip --quiet
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true
+

bin/install_prerequisites_ubuntu.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 set -Eeuo pipefail
 
-sudo apt update
-# Install libcurl dev so that hurl can be built dynamically with libcurl
-sudo apt install bash libcurl4-openssl-dev libxml2-utils netcat
+echo "----- install prerequisite packages -----"
+sudo apt-get update
+sudo apt-get -y install bash libcurl4-openssl-dev libxml2-utils netcat python3 python3-pip net-tools squid
 python3 -m pip install --upgrade pip --quiet
+sudo service squid stop || true
+sudo squid -k shutdown || true
+sudo rm -v /dev/shm/squid*.shm >/dev/null 2>&1 || true