mirror of
https://github.com/katanemo/plano.git
synced 2026-06-17 15:25:17 +02:00
run e2e tests in parallel
This commit is contained in:
Adil Hafeez
parent
88d14a205b
commit
d594526a60
5 changed files with 302 additions and 0 deletions
82
.github/workflows/e2e_tests_parallel.yml
vendored
Normal file
82
.github/workflows/e2e_tests_parallel.yml
vendored
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
name: e2e tests (parallel)
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main # Run tests on pushes to the main branch
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
e2e_tests:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false # Continue running other jobs even if one fails
|
||||
matrix:
|
||||
test-suite:
|
||||
- name: "Prompt Gateway"
|
||||
script: "run_prompt_gateway_tests.sh"
|
||||
- name: "Model Alias Routing"
|
||||
script: "run_model_alias_tests.sh"
|
||||
- name: "OpenAI Responses API"
|
||||
script: "run_openai_responses_tests.sh"
|
||||
- name: "OpenAI Responses API with State"
|
||||
script: "run_openai_responses_with_state_tests.sh"
|
||||
|
||||
name: ${{ matrix.test-suite.name }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
# --- Disk inspection & cleanup section (added to free space on GitHub runner) ---
|
||||
- name: Check disk usage before cleanup
|
||||
run: |
|
||||
echo "=== Disk usage before cleanup ==="
|
||||
df -h
|
||||
echo "=== Repo size ==="
|
||||
du -sh .
|
||||
|
||||
- name: Free disk space on runner
|
||||
run: |
|
||||
echo "=== Cleaning preinstalled SDKs and toolchains to free space ==="
|
||||
sudo rm -rf /usr/share/dotnet
|
||||
sudo rm -rf /usr/local/lib/android
|
||||
sudo rm -rf /opt/ghc
|
||||
# If you still hit disk issues, uncomment this to free more space.
|
||||
# It just removes cached tool versions; setup-python will re-download what it needs.
|
||||
# sudo rm -rf /opt/hostedtoolcache || true
|
||||
|
||||
echo "=== Docker cleanup (before our builds/compose) ==="
|
||||
docker system prune -af || true
|
||||
docker volume prune -f || true
|
||||
|
||||
echo "=== Disk usage after cleanup ==="
|
||||
df -h
|
||||
# --- End disk cleanup section ---
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install Poetry
|
||||
run: |
|
||||
export POETRY_VERSION=2.2.1
|
||||
curl -sSL https://install.python-poetry.org | python3 -
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
|
||||
- name: Run ${{ matrix.test-suite.name }} tests
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
|
||||
AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
|
||||
GROK_API_KEY : ${{ secrets.GROK_API_KEY }}
|
||||
run: |
|
||||
python -mvenv venv
|
||||
source venv/bin/activate && cd tests/e2e && bash ${{ matrix.test-suite.script }}
|
||||
52
tests/e2e/run_model_alias_tests.sh
Executable file
52
tests/e2e/run_model_alias_tests.sh
Executable file
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash
|
||||
# Run model alias routing e2e tests
|
||||
# if any of the commands fail, the script will exit
|
||||
set -e
|
||||
|
||||
. ./common_scripts.sh
|
||||
|
||||
print_disk_usage
|
||||
|
||||
mkdir -p ~/plano_logs
|
||||
touch ~/plano_logs/modelserver.log
|
||||
|
||||
print_debug() {
|
||||
log "Received signal to stop"
|
||||
log "Printing debug logs for docker"
|
||||
log "===================================="
|
||||
tail -n 100 ../build.log
|
||||
planoai logs --debug | tail -n 100
|
||||
}
|
||||
|
||||
trap 'print_debug' INT TERM ERR
|
||||
|
||||
log starting > ../build.log
|
||||
|
||||
log building and installing plano cli
|
||||
log ==================================
|
||||
cd ../../cli
|
||||
poetry install
|
||||
cd -
|
||||
|
||||
log building docker image for arch gateway
|
||||
log ======================================
|
||||
cd ../../
|
||||
planoai build
|
||||
cd -
|
||||
|
||||
# Once we build plano we have to install the dependencies again to a new virtual environment.
|
||||
poetry install
|
||||
|
||||
log startup arch gateway with model alias routing demo
|
||||
cd ../../
|
||||
planoai down
|
||||
planoai up demos/use_cases/model_alias_routing/config_with_aliases.yaml
|
||||
cd -
|
||||
|
||||
log running e2e tests for model alias routing
|
||||
log ========================================
|
||||
poetry run pytest test_model_alias_routing.py
|
||||
|
||||
log shutting down the arch gateway service
|
||||
log =======================================
|
||||
planoai down
|
||||
52
tests/e2e/run_openai_responses_tests.sh
Executable file
52
tests/e2e/run_openai_responses_tests.sh
Executable file
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash
|
||||
# Run OpenAI responses API e2e tests
|
||||
# if any of the commands fail, the script will exit
|
||||
set -e
|
||||
|
||||
. ./common_scripts.sh
|
||||
|
||||
print_disk_usage
|
||||
|
||||
mkdir -p ~/plano_logs
|
||||
touch ~/plano_logs/modelserver.log
|
||||
|
||||
print_debug() {
|
||||
log "Received signal to stop"
|
||||
log "Printing debug logs for docker"
|
||||
log "===================================="
|
||||
tail -n 100 ../build.log
|
||||
planoai logs --debug | tail -n 100
|
||||
}
|
||||
|
||||
trap 'print_debug' INT TERM ERR
|
||||
|
||||
log starting > ../build.log
|
||||
|
||||
log building and installing plano cli
|
||||
log ==================================
|
||||
cd ../../cli
|
||||
poetry install
|
||||
cd -
|
||||
|
||||
log building docker image for arch gateway
|
||||
log ======================================
|
||||
cd ../../
|
||||
planoai build
|
||||
cd -
|
||||
|
||||
# Once we build plano we have to install the dependencies again to a new virtual environment.
|
||||
poetry install
|
||||
|
||||
log startup arch gateway with model alias routing demo
|
||||
cd ../../
|
||||
planoai down
|
||||
planoai up demos/use_cases/model_alias_routing/config_with_aliases.yaml
|
||||
cd -
|
||||
|
||||
log running e2e tests for openai responses api client
|
||||
log ================================================
|
||||
poetry run pytest test_openai_responses_api_client.py
|
||||
|
||||
log shutting down the arch gateway service
|
||||
log =======================================
|
||||
planoai down
|
||||
52
tests/e2e/run_openai_responses_with_state_tests.sh
Executable file
52
tests/e2e/run_openai_responses_with_state_tests.sh
Executable file
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash
|
||||
# Run OpenAI responses API with state storage e2e tests
|
||||
# if any of the commands fail, the script will exit
|
||||
set -e
|
||||
|
||||
. ./common_scripts.sh
|
||||
|
||||
print_disk_usage
|
||||
|
||||
mkdir -p ~/plano_logs
|
||||
touch ~/plano_logs/modelserver.log
|
||||
|
||||
print_debug() {
|
||||
log "Received signal to stop"
|
||||
log "Printing debug logs for docker"
|
||||
log "===================================="
|
||||
tail -n 100 ../build.log
|
||||
planoai logs --debug | tail -n 100
|
||||
}
|
||||
|
||||
trap 'print_debug' INT TERM ERR
|
||||
|
||||
log starting > ../build.log
|
||||
|
||||
log building and installing plano cli
|
||||
log ==================================
|
||||
cd ../../cli
|
||||
poetry install
|
||||
cd -
|
||||
|
||||
log building docker image for arch gateway
|
||||
log ======================================
|
||||
cd ../../
|
||||
planoai build
|
||||
cd -
|
||||
|
||||
# Once we build plano we have to install the dependencies again to a new virtual environment.
|
||||
poetry install
|
||||
|
||||
log startup arch gateway with state storage for openai responses api client demo
|
||||
cd ../../
|
||||
planoai down
|
||||
planoai up tests/e2e/config_memory_state_v1_responses.yaml
|
||||
cd -
|
||||
|
||||
log running e2e tests for openai responses api client with state
|
||||
log ============================================================
|
||||
poetry run pytest test_openai_responses_api_client_with_state.py
|
||||
|
||||
log shutting down the arch gateway service
|
||||
log =======================================
|
||||
planoai down
|
||||
64
tests/e2e/run_prompt_gateway_tests.sh
Executable file
64
tests/e2e/run_prompt_gateway_tests.sh
Executable file
|
|
@ -0,0 +1,64 @@
|
|||
#!/bin/bash
|
||||
# Run prompt gateway e2e tests
|
||||
# if any of the commands fail, the script will exit
|
||||
set -e
|
||||
|
||||
. ./common_scripts.sh
|
||||
|
||||
print_disk_usage
|
||||
|
||||
mkdir -p ~/plano_logs
|
||||
touch ~/plano_logs/modelserver.log
|
||||
|
||||
print_debug() {
|
||||
log "Received signal to stop"
|
||||
log "Printing debug logs for docker"
|
||||
log "===================================="
|
||||
tail -n 100 ../build.log
|
||||
planoai logs --debug | tail -n 100
|
||||
}
|
||||
|
||||
trap 'print_debug' INT TERM ERR
|
||||
|
||||
log starting > ../build.log
|
||||
|
||||
log building and running function_calling demo
|
||||
log ===========================================
|
||||
cd ../../demos/samples_python/weather_forecast/
|
||||
docker compose up weather_forecast_service --build -d
|
||||
cd -
|
||||
|
||||
log building and installing plano cli
|
||||
log ==================================
|
||||
cd ../../cli
|
||||
poetry install
|
||||
cd -
|
||||
|
||||
log building docker image for arch gateway
|
||||
log ======================================
|
||||
cd ../../
|
||||
planoai build
|
||||
cd -
|
||||
|
||||
# Once we build plano we have to install the dependencies again to a new virtual environment.
|
||||
poetry install
|
||||
|
||||
log startup arch gateway with function calling demo
|
||||
cd ../../
|
||||
planoai down
|
||||
planoai up demos/samples_python/weather_forecast/config.yaml
|
||||
cd -
|
||||
|
||||
log running e2e tests for prompt gateway
|
||||
log ====================================
|
||||
poetry run pytest test_prompt_gateway.py
|
||||
|
||||
log shutting down the arch gateway service for prompt_gateway demo
|
||||
log ===============================================================
|
||||
planoai down
|
||||
|
||||
log shutting down the weather_forecast demo
|
||||
log =======================================
|
||||
cd ../../demos/samples_python/weather_forecast
|
||||
docker compose down
|
||||
cd -
|
||||
Loading…
Add table
Add a link
Reference in a new issue