apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-30 19:36:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

improve e2e tests (#731)

Browse source 
* fix build break

docs build was breaking because requirements file was getting ignored from .dockerignore

* improve e2e tests time

* fix: bump GH Actions to latest versions (checkout@v4, setup-python@v5, build-push-action@v6)

* more improvements

* fix perm

* more improvements

* parallel runs
This commit is contained in:
Adil Hafeez 2026-02-09 13:20:06 -08:00 • committed by GitHub
parent 99077d83fb
commit 4d9ed74b68
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 327 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/e2e/pyproject.toml
View file

@ -13,6 +13,7 @@ dependencies = [
"pytest-sugar>=1.0.0",
"deepdiff>=8.0.1",
"pytest-retry>=1.6.3",
"pytest-xdist>=3.5.0",
"anthropic>=0.66.0",
"openai>=1.0.0",
]

49
tests/e2e/run_model_alias_tests.sh Executable file
View file

@ -0,0 +1,49 @@
#!/bin/bash
# Runs the model_alias_routing + openai responses API e2e test suites.
# These share the same gateway config so they run together.
# Requires the plano Docker image to already be built/loaded.
set -e
. ./common_scripts.sh
print_disk_usage
mkdir -p ~/plano_logs
touch ~/plano_logs/modelserver.log
print_debug() {
log "Received signal to stop"
log "Printing debug logs for docker"
log "===================================="
tail -n 100 ../build.log 2>/dev/null || true
planoai logs --debug 2>/dev/null | tail -n 100 || true
}
trap 'print_debug' INT TERM ERR
log starting > ../build.log
# Install plano CLI
log "building and installing plano cli"
cd ../../cli
uv sync
uv tool install .
cd -
# Re-sync e2e deps
uv sync
# Start gateway with model alias routing config
log "startup arch gateway with model alias routing demo"
cd ../../
planoai down || true
planoai up demos/use_cases/model_alias_routing/config_with_aliases.yaml
cd -
# Run both test suites that share this config in a single pytest invocation
log "running e2e tests for model alias routing + openai responses api"
uv run pytest -n auto test_model_alias_routing.py test_openai_responses_api_client.py
# Cleanup
log "shutting down"
planoai down || true

57
tests/e2e/run_prompt_gateway_tests.sh Executable file
View file

@ -0,0 +1,57 @@
#!/bin/bash
# Runs the prompt_gateway e2e test suite.
# Requires the plano Docker image to already be built/loaded.
set -e
. ./common_scripts.sh
print_disk_usage
mkdir -p ~/plano_logs
touch ~/plano_logs/modelserver.log
print_debug() {
log "Received signal to stop"
log "Printing debug logs for docker"
log "===================================="
tail -n 100 ../build.log 2>/dev/null || true
planoai logs --debug 2>/dev/null | tail -n 100 || true
}
trap 'print_debug' INT TERM ERR
log starting > ../build.log
# Install plano CLI
log "building and installing plano cli"
cd ../../cli
uv sync
uv tool install .
cd -
# Re-sync e2e deps
uv sync
# Start weather_forecast service (needed for prompt_gateway tests)
log "building and running weather_forecast service"
cd ../../demos/samples_python/weather_forecast/
docker compose up weather_forecast_service --build -d
cd -
# Start gateway with prompt_gateway config
log "startup arch gateway with function calling demo"
cd ../../
planoai down || true
planoai up demos/samples_python/weather_forecast/config.yaml
cd -
# Run tests
log "running e2e tests for prompt gateway"
uv run pytest test_prompt_gateway.py
# Cleanup
log "shutting down"
planoai down || true
cd ../../demos/samples_python/weather_forecast
docker compose down
cd -

48
tests/e2e/run_responses_state_tests.sh Executable file
View file

@ -0,0 +1,48 @@
#!/bin/bash
# Runs the openai responses API with state storage e2e test suite.
# Requires the plano Docker image to already be built/loaded.
set -e
. ./common_scripts.sh
print_disk_usage
mkdir -p ~/plano_logs
touch ~/plano_logs/modelserver.log
print_debug() {
log "Received signal to stop"
log "Printing debug logs for docker"
log "===================================="
tail -n 100 ../build.log 2>/dev/null || true
planoai logs --debug 2>/dev/null | tail -n 100 || true
}
trap 'print_debug' INT TERM ERR
log starting > ../build.log
# Install plano CLI
log "building and installing plano cli"
cd ../../cli
uv sync
uv tool install .
cd -
# Re-sync e2e deps
uv sync
# Start gateway with state storage config
log "startup arch gateway with state storage config"
cd ../../
planoai down || true
planoai up tests/e2e/config_memory_state_v1_responses.yaml
cd -
# Run tests
log "running e2e tests for openai responses api with state"
uv run pytest test_openai_responses_api_client_with_state.py
# Cleanup
log "shutting down"
planoai down || true