mirror of
https://github.com/flakestorm/flakestorm.git
synced 2026-04-25 00:36:54 +02:00
Implement Open Source edition limits and feature restrictions
- Add 5 mutation types (paraphrase, noise, tone_shift, prompt_injection, custom) - Cap mutations at 50 per test run - Force sequential execution only - Disable GitHub Actions integration (Cloud feature) - Add upgrade prompts throughout CLI - Update README with feature comparison - Add limits.py module for centralized limit management - Add cloud and limits CLI commands - Update all documentation with Cloud upgrade messaging
This commit is contained in:
parent
2016be238d
commit
7b75fc9530
47 changed files with 3560 additions and 1012 deletions
159
tests/test_cli.py
Normal file
159
tests/test_cli.py
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
"""Tests for CLI commands."""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from entropix.cli.main import app
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
class TestHelpCommand:
|
||||
"""Tests for help output."""
|
||||
|
||||
def test_main_help(self):
|
||||
"""Main help displays correctly."""
|
||||
result = runner.invoke(app, ["--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "run" in result.output.lower() or "entropix" in result.output.lower()
|
||||
|
||||
def test_run_help(self):
|
||||
"""Run command help displays options."""
|
||||
result = runner.invoke(app, ["run", "--help"])
|
||||
assert result.exit_code == 0
|
||||
assert "--config" in result.output or "config" in result.output.lower()
|
||||
|
||||
def test_init_help(self):
|
||||
"""Init command help displays."""
|
||||
result = runner.invoke(app, ["init", "--help"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
def test_verify_help(self):
|
||||
"""Verify command help displays."""
|
||||
result = runner.invoke(app, ["verify", "--help"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
class TestInitCommand:
|
||||
"""Tests for `entropix init`."""
|
||||
|
||||
def test_init_creates_config(self):
|
||||
"""init creates entropix.yaml."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Change to temp directory context
|
||||
result = runner.invoke(app, ["init"], catch_exceptions=False)
|
||||
|
||||
# The command might create in current dir or specified dir
|
||||
# Check the output for success indicators
|
||||
assert (
|
||||
result.exit_code == 0
|
||||
or "created" in result.output.lower()
|
||||
or "exists" in result.output.lower()
|
||||
)
|
||||
|
||||
|
||||
class TestVerifyCommand:
|
||||
"""Tests for `entropix verify`."""
|
||||
|
||||
def test_verify_valid_config(self):
|
||||
"""verify accepts valid config."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "entropix.yaml"
|
||||
config_path.write_text(
|
||||
"""
|
||||
agent:
|
||||
endpoint: "http://localhost:8000/chat"
|
||||
type: http
|
||||
|
||||
golden_prompts:
|
||||
- "Test prompt"
|
||||
|
||||
mutations:
|
||||
count: 5
|
||||
types:
|
||||
- paraphrase
|
||||
|
||||
invariants: []
|
||||
"""
|
||||
)
|
||||
result = runner.invoke(app, ["verify", "--config", str(config_path)])
|
||||
# The verify command should at least run (exit 0 or 1)
|
||||
# On Python 3.9, there may be type annotation issues
|
||||
assert result.exit_code in (0, 1)
|
||||
|
||||
def test_verify_missing_config(self):
|
||||
"""verify handles missing config file."""
|
||||
result = runner.invoke(app, ["verify", "--config", "/nonexistent/path.yaml"])
|
||||
# Should show error about missing file
|
||||
assert (
|
||||
result.exit_code != 0
|
||||
or "not found" in result.output.lower()
|
||||
or "error" in result.output.lower()
|
||||
)
|
||||
|
||||
def test_verify_invalid_yaml(self):
|
||||
"""verify rejects invalid YAML syntax."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
config_path = Path(tmpdir) / "entropix.yaml"
|
||||
config_path.write_text("invalid: yaml: : content")
|
||||
|
||||
result = runner.invoke(app, ["verify", "--config", str(config_path)])
|
||||
# Should fail or show error
|
||||
assert result.exit_code != 0 or "error" in result.output.lower()
|
||||
|
||||
|
||||
class TestRunCommand:
|
||||
"""Tests for `entropix run`."""
|
||||
|
||||
def test_run_missing_config(self):
|
||||
"""run handles missing config."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = runner.invoke(
|
||||
app, ["run", "--config", f"{tmpdir}/nonexistent.yaml"]
|
||||
)
|
||||
# Should show error about missing file
|
||||
assert (
|
||||
result.exit_code != 0
|
||||
or "not found" in result.output.lower()
|
||||
or "error" in result.output.lower()
|
||||
)
|
||||
|
||||
def test_run_with_ci_flag(self):
|
||||
"""run accepts --ci flag."""
|
||||
result = runner.invoke(app, ["run", "--help"])
|
||||
assert "--ci" in result.output
|
||||
|
||||
def test_run_with_min_score(self):
|
||||
"""run accepts --min-score flag."""
|
||||
result = runner.invoke(app, ["run", "--help"])
|
||||
assert "--min-score" in result.output or "min" in result.output.lower()
|
||||
|
||||
|
||||
class TestReportCommand:
|
||||
"""Tests for `entropix report`."""
|
||||
|
||||
def test_report_help(self):
|
||||
"""report command has help."""
|
||||
result = runner.invoke(app, ["report", "--help"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
class TestScoreCommand:
|
||||
"""Tests for `entropix score`."""
|
||||
|
||||
def test_score_help(self):
|
||||
"""score command has help."""
|
||||
result = runner.invoke(app, ["score", "--help"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
class TestVersionFlag:
|
||||
"""Tests for --version flag."""
|
||||
|
||||
def test_version_displays(self):
|
||||
"""--version shows version number."""
|
||||
result = runner.invoke(app, ["--version"])
|
||||
# Should show version or be a recognized command
|
||||
assert result.exit_code == 0 or "version" in result.output.lower()
|
||||
Loading…
Add table
Add a link
Reference in a new issue