mirror of
https://github.com/flakestorm/flakestorm.git
synced 2026-04-25 00:36:54 +02:00
- Updated class names and import statements to align with the new naming convention.
- Adjusted test commands and report references to use FlakeStorm terminology. - Ensured consistency in configuration and runner references throughout the documentation.
This commit is contained in:
parent
0c986e268a
commit
1d45fb2981
10 changed files with 37 additions and 37 deletions
|
|
@ -6,11 +6,11 @@
|
|||
|
||||
```python
|
||||
import asyncio
|
||||
from flakestorm import flakestormRunner, load_config
|
||||
from flakestorm import FlakeStormRunner, load_config
|
||||
|
||||
async def main():
|
||||
config = load_config("flakestorm.yaml")
|
||||
runner = EntropixRunner(config)
|
||||
runner = FlakeStormRunner(config)
|
||||
results = await runner.run()
|
||||
print(f"Robustness Score: {results.statistics.robustness_score:.1%}")
|
||||
|
||||
|
|
@ -21,12 +21,12 @@ asyncio.run(main())
|
|||
|
||||
## Core Classes
|
||||
|
||||
### EntropixConfig
|
||||
### FlakeStormConfig
|
||||
|
||||
Configuration container for all flakestorm settings.
|
||||
|
||||
```python
|
||||
from flakestorm import flakestormConfig, load_config
|
||||
from flakestorm import FlakeStormConfig, load_config
|
||||
|
||||
# Load from file
|
||||
config = load_config("flakestorm.yaml")
|
||||
|
|
@ -41,7 +41,7 @@ config.invariants # list[InvariantConfig]
|
|||
yaml_str = config.to_yaml()
|
||||
|
||||
# Parse from string
|
||||
config = EntropixConfig.from_yaml(yaml_content)
|
||||
config = FlakeStormConfig.from_yaml(yaml_content)
|
||||
```
|
||||
|
||||
#### Properties
|
||||
|
|
@ -59,15 +59,15 @@ config = EntropixConfig.from_yaml(yaml_content)
|
|||
|
||||
---
|
||||
|
||||
### EntropixRunner
|
||||
### FlakeStormRunner
|
||||
|
||||
Main test runner class.
|
||||
|
||||
```python
|
||||
from flakestorm import flakestormRunner
|
||||
from flakestorm import FlakeStormRunner
|
||||
|
||||
runner = EntropixRunner(
|
||||
config="flakestorm.yaml", # or EntropixConfig object
|
||||
runner = FlakeStormRunner(
|
||||
config="flakestorm.yaml", # or FlakeStormConfig object
|
||||
agent=None, # optional: pre-configured adapter
|
||||
console=None, # optional: Rich console
|
||||
show_progress=True, # show progress bars
|
||||
|
|
@ -434,7 +434,6 @@ fi
|
|||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `ENTROPIX_CONFIG` | Default config file path |
|
||||
| `OLLAMA_HOST` | Override Ollama server URL |
|
||||
| Custom headers | Expanded in config via `${VAR}` syntax |
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ pytest --cov=src/flakestorm --cov-report=html
|
|||
pytest tests/test_config.py
|
||||
|
||||
# Run specific test
|
||||
pytest tests/test_config.py::TestEntropixConfig::test_create_default_config
|
||||
pytest tests/test_config.py::TestFlakeStormConfig::test_create_default_config
|
||||
```
|
||||
|
||||
### Code Style
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ async def _run_single_mutation(self, mutation):
|
|||
|
||||
**A:** They serve different purposes:
|
||||
|
||||
- **`runner.py`**: High-level API for users - simple `EntropixRunner.run()` interface
|
||||
- **`runner.py`**: High-level API for users - simple `FlakeStormRunner.run()` interface
|
||||
- **`orchestrator.py`**: Internal coordination logic - handles the complex flow
|
||||
|
||||
This separation allows:
|
||||
|
|
@ -359,7 +359,7 @@ fn levenshtein_distance(s1: &str, s2: &str) -> usize {
|
|||
}
|
||||
|
||||
#[pymodule]
|
||||
fn entropix_rust(m: &PyModule) -> PyResult<()> {
|
||||
fn flakestorm_rust(m: &PyModule) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(levenshtein_distance, m)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -375,7 +375,7 @@ except ImportError:
|
|||
|
||||
def levenshtein_distance(s1: str, s2: str) -> int:
|
||||
if _RUST_AVAILABLE:
|
||||
return entropix_rust.levenshtein_distance(s1, s2)
|
||||
return flakestorm_rust.levenshtein_distance(s1, s2)
|
||||
# Pure Python fallback
|
||||
...
|
||||
```
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ This document tracks the implementation progress of flakestorm - The Agent Relia
|
|||
### Phase 3: Runner & Assertions (Week 3-4)
|
||||
|
||||
#### Async Runner
|
||||
- [x] Create EntropixRunner class
|
||||
- [x] Create FlakeStormRunner class
|
||||
- [x] Implement orchestrator logic
|
||||
- [x] Add concurrency control with semaphores
|
||||
- [x] Implement progress tracking
|
||||
|
|
|
|||
|
|
@ -88,13 +88,14 @@ class AgentConfig(BaseModel):
|
|||
```
|
||||
|
||||
```python
|
||||
class EntropixConfig(BaseModel):
|
||||
class FlakeStormConfig(BaseModel):
|
||||
"""Root configuration model."""
|
||||
agent: AgentConfig
|
||||
golden_prompts: list[str]
|
||||
mutations: MutationConfig
|
||||
llm: LLMConfig
|
||||
model: ModelConfig
|
||||
invariants: list[InvariantConfig]
|
||||
output: OutputConfig
|
||||
advanced: AdvancedConfig
|
||||
```
|
||||
|
||||
|
|
@ -194,13 +195,13 @@ The adapter pattern was chosen because:
|
|||
**Key Components:**
|
||||
|
||||
```python
|
||||
class EntropixOrchestrator:
|
||||
class Orchestrator:
|
||||
"""Main orchestration class."""
|
||||
|
||||
async def run(self) -> TestResults:
|
||||
"""Execute the full test suite."""
|
||||
# 1. Generate mutations for all golden prompts
|
||||
# 2. Run mutations in parallel with semaphore
|
||||
# 2. Run mutations sequentially (open-source version)
|
||||
# 3. Verify responses against invariants
|
||||
# 4. Aggregate and score results
|
||||
# 5. Return comprehensive results
|
||||
|
|
@ -573,7 +574,7 @@ class MutationResult:
|
|||
@dataclass
|
||||
class TestResults:
|
||||
"""Complete test run results."""
|
||||
config: EntropixConfig
|
||||
config: FlakeStormConfig
|
||||
mutations: list[MutationResult]
|
||||
statistics: TestStatistics
|
||||
timestamp: datetime
|
||||
|
|
|
|||
|
|
@ -120,10 +120,10 @@ flakestorm/
|
|||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
from flakestorm.core.config import load_config, EntropixConfig
|
||||
from flakestorm.core.runner import flakestormRunner
|
||||
from flakestorm.core.config import load_config, FlakeStormConfig
|
||||
from flakestorm.core.runner import FlakeStormRunner
|
||||
|
||||
__all__ = ["load_config", "EntropixConfig", "EntropixRunner", "__version__"]
|
||||
__all__ = ["load_config", "FlakeStormConfig", "FlakeStormRunner", "__version__"]
|
||||
```
|
||||
|
||||
---
|
||||
|
|
@ -268,7 +268,7 @@ jobs:
|
|||
|
||||
## Publishing the Rust Extension
|
||||
|
||||
The Rust extension (`entropix_rust`) is published separately because it requires platform-specific binaries.
|
||||
The Rust extension (if implemented) would be published separately because it requires platform-specific binaries.
|
||||
|
||||
### Using `maturin`
|
||||
|
||||
|
|
@ -278,7 +278,7 @@ cd rust/
|
|||
# Build wheels for your current platform
|
||||
maturin build --release
|
||||
|
||||
# The wheel is in: ../target/wheels/entropix_rust-0.1.0-cp39-*.whl
|
||||
# The wheel would be in: ../target/wheels/flakestorm_rust-0.1.0-cp39-*.whl
|
||||
```
|
||||
|
||||
### Multi-Platform Publishing with GitHub Actions
|
||||
|
|
|
|||
|
|
@ -308,8 +308,8 @@ import pytest
|
|||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from datetime import datetime
|
||||
|
||||
from flakestorm.core.orchestrator import flakestormOrchestrator, OrchestratorState
|
||||
from flakestorm.core.config import flakestormConfig, AgentConfig, MutationConfig
|
||||
from flakestorm.core.orchestrator import Orchestrator, OrchestratorState
|
||||
from flakestorm.core.config import FlakeStormConfig, AgentConfig, MutationConfig
|
||||
from flakestorm.mutations.types import Mutation, MutationType
|
||||
from flakestorm.assertions.verifier import CheckResult
|
||||
|
||||
|
|
@ -332,13 +332,13 @@ class TestOrchestratorState:
|
|||
assert state.completed_mutations == 5
|
||||
|
||||
|
||||
class TestEntropixOrchestrator:
|
||||
class TestOrchestrator:
|
||||
"""Tests for main orchestrator."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config(self):
|
||||
"""Create a minimal test config."""
|
||||
return EntropixConfig(
|
||||
return FlakeStormConfig(
|
||||
agent=AgentConfig(
|
||||
endpoint="http://localhost:8000/chat",
|
||||
type="http",
|
||||
|
|
@ -388,7 +388,7 @@ class TestEntropixOrchestrator:
|
|||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||
):
|
||||
"""Orchestrator generates mutations for all golden prompts."""
|
||||
orchestrator = EntropixOrchestrator(
|
||||
orchestrator = Orchestrator(
|
||||
config=mock_config,
|
||||
agent=mock_agent,
|
||||
mutation_engine=mock_mutation_engine,
|
||||
|
|
@ -405,7 +405,7 @@ class TestEntropixOrchestrator:
|
|||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||
):
|
||||
"""Orchestrator invokes agent for each mutation."""
|
||||
orchestrator = EntropixOrchestrator(
|
||||
orchestrator = Orchestrator(
|
||||
config=mock_config,
|
||||
agent=mock_agent,
|
||||
mutation_engine=mock_mutation_engine,
|
||||
|
|
@ -423,7 +423,7 @@ class TestEntropixOrchestrator:
|
|||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||
):
|
||||
"""Orchestrator returns complete test results."""
|
||||
orchestrator = EntropixOrchestrator(
|
||||
orchestrator = Orchestrator(
|
||||
config=mock_config,
|
||||
agent=mock_agent,
|
||||
mutation_engine=mock_mutation_engine,
|
||||
|
|
@ -444,7 +444,7 @@ class TestEntropixOrchestrator:
|
|||
failing_agent = AsyncMock()
|
||||
failing_agent.invoke.side_effect = Exception("Agent error")
|
||||
|
||||
orchestrator = EntropixOrchestrator(
|
||||
orchestrator = Orchestrator(
|
||||
config=mock_config,
|
||||
agent=failing_agent,
|
||||
mutation_engine=mock_mutation_engine,
|
||||
|
|
|
|||
|
|
@ -674,7 +674,7 @@ invariants:
|
|||
flakestorm run --output html
|
||||
|
||||
# Review report
|
||||
open reports/entropix_report_*.html
|
||||
open reports/flakestorm-*.html
|
||||
|
||||
# Fix issues in your agent
|
||||
# ...
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ flakestorm run --ci --min-score 0.8
|
|||
|
||||
```bash
|
||||
# Open the generated report
|
||||
open reports/entropix_report_*.html
|
||||
open reports/flakestorm-*.html
|
||||
```
|
||||
|
||||
---
|
||||
|
|
@ -734,7 +734,7 @@ agent:
|
|||
flakestorm run --verbose
|
||||
|
||||
# Or set environment variable
|
||||
export ENTROPIX_DEBUG=1
|
||||
export FLAKESTORM_DEBUG=1
|
||||
flakestorm run
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Chaos Engineering for AI Agents. Apply adversarial fuzzing to prove
|
|||
your agents are production-ready before deployment.
|
||||
|
||||
Example:
|
||||
>>> from flakestorm import flakestormRunner, load_config
|
||||
>>> from flakestorm import FlakeStormRunner, load_config
|
||||
>>> config = load_config("flakestorm.yaml")
|
||||
>>> runner = FlakeStormRunner(config)
|
||||
>>> results = await runner.run()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue