mirror of
https://github.com/flakestorm/flakestorm.git
synced 2026-04-24 16:26:35 +02:00
61652be09b
- Rename all instances of Entropix to FlakeStorm - Rename package from entropix to flakestorm - Update all class names (EntropixConfig -> FlakeStormConfig, EntropixRunner -> FlakeStormRunner) - Update Rust module from entropix_rust to flakestorm_rust - Update README: remove cloud comparison, update links to flakestorm.com - Update .gitignore to allow docs files referenced in README - Add origin remote for VS Code compatibility - Fix missing imports and type references - All imports and references updated throughout codebase
159 lines
4.9 KiB
Python
159 lines
4.9 KiB
Python
"""Tests for CLI commands."""
|
|
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from typer.testing import CliRunner
|
|
|
|
from flakestorm.cli.main import app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
class TestHelpCommand:
|
|
"""Tests for help output."""
|
|
|
|
def test_main_help(self):
|
|
"""Main help displays correctly."""
|
|
result = runner.invoke(app, ["--help"])
|
|
assert result.exit_code == 0
|
|
assert "run" in result.output.lower() or "flakestorm" in result.output.lower()
|
|
|
|
def test_run_help(self):
|
|
"""Run command help displays options."""
|
|
result = runner.invoke(app, ["run", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "--config" in result.output or "config" in result.output.lower()
|
|
|
|
def test_init_help(self):
|
|
"""Init command help displays."""
|
|
result = runner.invoke(app, ["init", "--help"])
|
|
assert result.exit_code == 0
|
|
|
|
def test_verify_help(self):
|
|
"""Verify command help displays."""
|
|
result = runner.invoke(app, ["verify", "--help"])
|
|
assert result.exit_code == 0
|
|
|
|
|
|
class TestInitCommand:
|
|
"""Tests for `flakestorm init`."""
|
|
|
|
def test_init_creates_config(self):
|
|
"""init creates flakestorm.yaml."""
|
|
with tempfile.TemporaryDirectory():
|
|
# Change to temp directory context
|
|
result = runner.invoke(app, ["init"], catch_exceptions=False)
|
|
|
|
# The command might create in current dir or specified dir
|
|
# Check the output for success indicators
|
|
assert (
|
|
result.exit_code == 0
|
|
or "created" in result.output.lower()
|
|
or "exists" in result.output.lower()
|
|
)
|
|
|
|
|
|
class TestVerifyCommand:
|
|
"""Tests for `flakestorm verify`."""
|
|
|
|
def test_verify_valid_config(self):
|
|
"""verify accepts valid config."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
config_path = Path(tmpdir) / "flakestorm.yaml"
|
|
config_path.write_text(
|
|
"""
|
|
agent:
|
|
endpoint: "http://localhost:8000/chat"
|
|
type: http
|
|
|
|
golden_prompts:
|
|
- "Test prompt"
|
|
|
|
mutations:
|
|
count: 5
|
|
types:
|
|
- paraphrase
|
|
|
|
invariants: []
|
|
"""
|
|
)
|
|
result = runner.invoke(app, ["verify", "--config", str(config_path)])
|
|
# The verify command should at least run (exit 0 or 1)
|
|
# On Python 3.9, there may be type annotation issues
|
|
assert result.exit_code in (0, 1)
|
|
|
|
def test_verify_missing_config(self):
|
|
"""verify handles missing config file."""
|
|
result = runner.invoke(app, ["verify", "--config", "/nonexistent/path.yaml"])
|
|
# Should show error about missing file
|
|
assert (
|
|
result.exit_code != 0
|
|
or "not found" in result.output.lower()
|
|
or "error" in result.output.lower()
|
|
)
|
|
|
|
def test_verify_invalid_yaml(self):
|
|
"""verify rejects invalid YAML syntax."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
config_path = Path(tmpdir) / "flakestorm.yaml"
|
|
config_path.write_text("invalid: yaml: : content")
|
|
|
|
result = runner.invoke(app, ["verify", "--config", str(config_path)])
|
|
# Should fail or show error
|
|
assert result.exit_code != 0 or "error" in result.output.lower()
|
|
|
|
|
|
class TestRunCommand:
|
|
"""Tests for `flakestorm run`."""
|
|
|
|
def test_run_missing_config(self):
|
|
"""run handles missing config."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
result = runner.invoke(
|
|
app, ["run", "--config", f"{tmpdir}/nonexistent.yaml"]
|
|
)
|
|
# Should show error about missing file
|
|
assert (
|
|
result.exit_code != 0
|
|
or "not found" in result.output.lower()
|
|
or "error" in result.output.lower()
|
|
)
|
|
|
|
def test_run_with_ci_flag(self):
|
|
"""run accepts --ci flag."""
|
|
result = runner.invoke(app, ["run", "--help"])
|
|
assert "--ci" in result.output
|
|
|
|
def test_run_with_min_score(self):
|
|
"""run accepts --min-score flag."""
|
|
result = runner.invoke(app, ["run", "--help"])
|
|
assert "--min-score" in result.output or "min" in result.output.lower()
|
|
|
|
|
|
class TestReportCommand:
|
|
"""Tests for `flakestorm report`."""
|
|
|
|
def test_report_help(self):
|
|
"""report command has help."""
|
|
result = runner.invoke(app, ["report", "--help"])
|
|
assert result.exit_code == 0
|
|
|
|
|
|
class TestScoreCommand:
|
|
"""Tests for `flakestorm score`."""
|
|
|
|
def test_score_help(self):
|
|
"""score command has help."""
|
|
result = runner.invoke(app, ["score", "--help"])
|
|
assert result.exit_code == 0
|
|
|
|
|
|
class TestVersionFlag:
|
|
"""Tests for --version flag."""
|
|
|
|
def test_version_displays(self):
|
|
"""--version shows version number."""
|
|
result = runner.invoke(app, ["--version"])
|
|
# Should show version or be a recognized command
|
|
assert result.exit_code == 0 or "version" in result.output.lower()
|