run e2e tests in parallel

.github/workflows/e2e_tests_parallel.yml

@@ -0,0 +1,82 @@
+name: e2e tests (parallel)
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches:
+      - main  # Run tests on pushes to the main branch
+  pull_request:
+
+jobs:
+  e2e_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false  # Continue running other jobs even if one fails
+      matrix:
+        test-suite:
+          - name: "Prompt Gateway"
+            script: "run_prompt_gateway_tests.sh"
+          - name: "Model Alias Routing"
+            script: "run_model_alias_tests.sh"
+          - name: "OpenAI Responses API"
+            script: "run_openai_responses_tests.sh"
+          - name: "OpenAI Responses API with State"
+            script: "run_openai_responses_with_state_tests.sh"
+
+    name: ${{ matrix.test-suite.name }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # --- Disk inspection & cleanup section (added to free space on GitHub runner) ---
+      - name: Check disk usage before cleanup
+        run: |
+          echo "=== Disk usage before cleanup ==="
+          df -h
+          echo "=== Repo size ==="
+          du -sh .
+
+      - name: Free disk space on runner
+        run: |
+          echo "=== Cleaning preinstalled SDKs and toolchains to free space ==="
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          # If you still hit disk issues, uncomment this to free more space.
+          # It just removes cached tool versions; setup-python will re-download what it needs.
+          # sudo rm -rf /opt/hostedtoolcache || true
+
+          echo "=== Docker cleanup (before our builds/compose) ==="
+          docker system prune -af || true
+          docker volume prune -f || true
+
+          echo "=== Disk usage after cleanup ==="
+          df -h
+      # --- End disk cleanup section ---
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: |
+          export POETRY_VERSION=2.2.1
+          curl -sSL https://install.python-poetry.org | python3 -
+          export PATH="$HOME/.local/bin:$PATH"
+
+      - name: Run ${{ matrix.test-suite.name }} tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
+          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+          GROK_API_KEY : ${{ secrets.GROK_API_KEY }}
+        run: |
+          python -mvenv venv
+          source venv/bin/activate && cd tests/e2e && bash ${{ matrix.test-suite.script }}

tests/e2e/run_model_alias_tests.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Run model alias routing e2e tests
+# if any of the commands fail, the script will exit
+set -e
+
+. ./common_scripts.sh
+
+print_disk_usage
+
+mkdir -p ~/plano_logs
+touch ~/plano_logs/modelserver.log
+
+print_debug() {
+  log "Received signal to stop"
+  log "Printing debug logs for docker"
+  log "===================================="
+  tail -n 100 ../build.log
+  planoai logs --debug | tail -n 100
+}
+
+trap 'print_debug' INT TERM ERR
+
+log starting > ../build.log
+
+log building and installing plano cli
+log ==================================
+cd ../../cli
+poetry install
+cd -
+
+log building docker image for arch gateway
+log ======================================
+cd ../../
+planoai build
+cd -
+
+# Once we build plano we have to install the dependencies again to a new virtual environment.
+poetry install
+
+log startup arch gateway with model alias routing demo
+cd ../../
+planoai down
+planoai up demos/use_cases/model_alias_routing/config_with_aliases.yaml
+cd -
+
+log running e2e tests for model alias routing
+log ========================================
+poetry run pytest test_model_alias_routing.py
+
+log shutting down the arch gateway service
+log =======================================
+planoai down

tests/e2e/run_openai_responses_tests.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Run OpenAI responses API e2e tests
+# if any of the commands fail, the script will exit
+set -e
+
+. ./common_scripts.sh
+
+print_disk_usage
+
+mkdir -p ~/plano_logs
+touch ~/plano_logs/modelserver.log
+
+print_debug() {
+  log "Received signal to stop"
+  log "Printing debug logs for docker"
+  log "===================================="
+  tail -n 100 ../build.log
+  planoai logs --debug | tail -n 100
+}
+
+trap 'print_debug' INT TERM ERR
+
+log starting > ../build.log
+
+log building and installing plano cli
+log ==================================
+cd ../../cli
+poetry install
+cd -
+
+log building docker image for arch gateway
+log ======================================
+cd ../../
+planoai build
+cd -
+
+# Once we build plano we have to install the dependencies again to a new virtual environment.
+poetry install
+
+log startup arch gateway with model alias routing demo
+cd ../../
+planoai down
+planoai up demos/use_cases/model_alias_routing/config_with_aliases.yaml
+cd -
+
+log running e2e tests for openai responses api client
+log ================================================
+poetry run pytest test_openai_responses_api_client.py
+
+log shutting down the arch gateway service
+log =======================================
+planoai down

tests/e2e/run_openai_responses_with_state_tests.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Run OpenAI responses API with state storage e2e tests
+# if any of the commands fail, the script will exit
+set -e
+
+. ./common_scripts.sh
+
+print_disk_usage
+
+mkdir -p ~/plano_logs
+touch ~/plano_logs/modelserver.log
+
+print_debug() {
+  log "Received signal to stop"
+  log "Printing debug logs for docker"
+  log "===================================="
+  tail -n 100 ../build.log
+  planoai logs --debug | tail -n 100
+}
+
+trap 'print_debug' INT TERM ERR
+
+log starting > ../build.log
+
+log building and installing plano cli
+log ==================================
+cd ../../cli
+poetry install
+cd -
+
+log building docker image for arch gateway
+log ======================================
+cd ../../
+planoai build
+cd -
+
+# Once we build plano we have to install the dependencies again to a new virtual environment.
+poetry install
+
+log startup arch gateway with state storage for openai responses api client demo
+cd ../../
+planoai down
+planoai up tests/e2e/config_memory_state_v1_responses.yaml
+cd -
+
+log running e2e tests for openai responses api client with state
+log ============================================================
+poetry run pytest test_openai_responses_api_client_with_state.py
+
+log shutting down the arch gateway service
+log =======================================
+planoai down

tests/e2e/run_prompt_gateway_tests.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Run prompt gateway e2e tests
+# if any of the commands fail, the script will exit
+set -e
+
+. ./common_scripts.sh
+
+print_disk_usage
+
+mkdir -p ~/plano_logs
+touch ~/plano_logs/modelserver.log
+
+print_debug() {
+  log "Received signal to stop"
+  log "Printing debug logs for docker"
+  log "===================================="
+  tail -n 100 ../build.log
+  planoai logs --debug | tail -n 100
+}
+
+trap 'print_debug' INT TERM ERR
+
+log starting > ../build.log
+
+log building and running function_calling demo
+log ===========================================
+cd ../../demos/samples_python/weather_forecast/
+docker compose up weather_forecast_service --build -d
+cd -
+
+log building and installing plano cli
+log ==================================
+cd ../../cli
+poetry install
+cd -
+
+log building docker image for arch gateway
+log ======================================
+cd ../../
+planoai build
+cd -
+
+# Once we build plano we have to install the dependencies again to a new virtual environment.
+poetry install
+
+log startup arch gateway with function calling demo
+cd ../../
+planoai down
+planoai up demos/samples_python/weather_forecast/config.yaml
+cd -
+
+log running e2e tests for prompt gateway
+log ====================================
+poetry run pytest test_prompt_gateway.py
+
+log shutting down the arch gateway service for prompt_gateway demo
+log ===============================================================
+planoai down
+
+log shutting down the weather_forecast demo
+log =======================================
+cd ../../demos/samples_python/weather_forecast
+docker compose down
+cd -