mirror of
https://github.com/flakestorm/flakestorm.git
synced 2026-04-25 00:36:54 +02:00
Refactor Entropix to FlakeStorm
- Rename all instances of Entropix to FlakeStorm - Rename package from entropix to flakestorm - Update all class names (EntropixConfig -> FlakeStormConfig, EntropixRunner -> FlakeStormRunner) - Update Rust module from entropix_rust to flakestorm_rust - Update README: remove cloud comparison, update links to flakestorm.com - Update .gitignore to allow docs files referenced in README - Add origin remote for VS Code compatibility - Fix missing imports and type references - All imports and references updated throughout codebase
This commit is contained in:
Entropix
parent
7b75fc9530
commit
61652be09b
48 changed files with 586 additions and 425 deletions
17
.gitignore
vendored
17
.gitignore
vendored
|
|
@ -92,8 +92,8 @@ reports/
|
||||||
!docs/*.html
|
!docs/*.html
|
||||||
|
|
||||||
# Local configuration (may contain secrets)
|
# Local configuration (may contain secrets)
|
||||||
entropix.yaml
|
flakestorm.yaml
|
||||||
!entropix.yaml.example
|
!flakestorm.yaml.example
|
||||||
|
|
||||||
# Ollama models cache (optional, can be large)
|
# Ollama models cache (optional, can be large)
|
||||||
.ollama/
|
.ollama/
|
||||||
|
|
@ -108,5 +108,16 @@ entropix.yaml
|
||||||
.env.*.local
|
.env.*.local
|
||||||
secrets/
|
secrets/
|
||||||
|
|
||||||
# docs
|
# docs (exclude all, but allow specific files referenced in README)
|
||||||
docs/
|
docs/
|
||||||
|
# Allow docs files referenced in README.md
|
||||||
|
!docs/USAGE_GUIDE.md
|
||||||
|
!docs/CONFIGURATION_GUIDE.md
|
||||||
|
!docs/TEST_SCENARIOS.md
|
||||||
|
!docs/MODULES.md
|
||||||
|
!docs/DEVELOPER_FAQ.md
|
||||||
|
!docs/PUBLISHING.md
|
||||||
|
!docs/CONTRIBUTING.md
|
||||||
|
!docs/API_SPECIFICATION.md
|
||||||
|
!docs/TESTING_GUIDE.md
|
||||||
|
!docs/IMPLEMENTATION_CHECKLIST.md
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
# Pre-commit hooks for Entropix
|
# Pre-commit hooks for flakestorm
|
||||||
# Install: pip install pre-commit && pre-commit install
|
# Install: pip install pre-commit && pre-commit install
|
||||||
# Run manually: pre-commit run --all-files
|
# Run manually: pre-commit run --all-files
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,8 @@ resolver = "2"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
authors = ["Entropix Team"]
|
authors = ["flakestorm Team"]
|
||||||
repository = "https://github.com/entropix/entropix"
|
repository = "https://github.com/flakestorm/flakestorm"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
pyo3 = { version = "0.20", features = ["extension-module"] }
|
pyo3 = { version = "0.20", features = ["extension-module"] }
|
||||||
|
|
|
||||||
2
LICENSE
2
LICENSE
|
|
@ -175,7 +175,7 @@
|
||||||
|
|
||||||
END OF TERMS AND CONDITIONS
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
Copyright 2025 Entropix
|
Copyright 2025 flakestorm
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|
|
||||||
111
README.md
111
README.md
|
|
@ -1,4 +1,4 @@
|
||||||
# Entropix
|
# FlakeStorm
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<strong>The Agent Reliability Engine</strong><br>
|
<strong>The Agent Reliability Engine</strong><br>
|
||||||
|
|
@ -6,26 +6,22 @@
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://github.com/entropix/entropix/blob/main/LICENSE">
|
<a href="https://github.com/flakestorm/flakestorm/blob/main/LICENSE">
|
||||||
<img src="https://img.shields.io/badge/license-AGPLv3-blue.svg" alt="License">
|
<img src="https://img.shields.io/badge/license-AGPLv3-blue.svg" alt="License">
|
||||||
</a>
|
</a>
|
||||||
<a href="https://pypi.org/project/entropix/">
|
<a href="https://pypi.org/project/flakestorm/">
|
||||||
<img src="https://img.shields.io/pypi/v/entropix.svg" alt="PyPI">
|
<img src="https://img.shields.io/pypi/v/flakestorm.svg" alt="PyPI">
|
||||||
</a>
|
</a>
|
||||||
<a href="https://pypi.org/project/entropix/">
|
<a href="https://pypi.org/project/flakestorm/">
|
||||||
<img src="https://img.shields.io/pypi/pyversions/entropix.svg" alt="Python Versions">
|
<img src="https://img.shields.io/pypi/pyversions/flakestorm.svg" alt="Python Versions">
|
||||||
</a>
|
</a>
|
||||||
<a href="https://entropix.cloud">
|
<a href="https://flakestorm.com">
|
||||||
<img src="https://img.shields.io/badge/☁️-Cloud%20Available-blueviolet" alt="Cloud">
|
<img src="https://img.shields.io/badge/☁️-Cloud%20Available-blueviolet" alt="Cloud">
|
||||||
</a>
|
</a>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
> **📢 This is the Open Source Edition.** For production workloads, check out [Entropix Cloud](https://entropix.cloud) — 20x faster with parallel execution, cloud LLMs, and CI/CD integration.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## The Problem
|
## The Problem
|
||||||
|
|
||||||
**The "Happy Path" Fallacy**: Current AI development tools focus on getting an agent to work *once*. Developers tweak prompts until they get a correct answer, declare victory, and ship.
|
**The "Happy Path" Fallacy**: Current AI development tools focus on getting an agent to work *once*. Developers tweak prompts until they get a correct answer, declare victory, and ship.
|
||||||
|
|
@ -39,64 +35,32 @@
|
||||||
|
|
||||||
## The Solution
|
## The Solution
|
||||||
|
|
||||||
**Entropix** is a local-first testing engine that applies **Chaos Engineering** principles to AI Agents.
|
**FlakeStorm** is a local-first testing engine that applies **Chaos Engineering** principles to AI Agents.
|
||||||
|
|
||||||
Instead of running one test case, Entropix takes a single "Golden Prompt", generates adversarial mutations (semantic variations, noise injection, hostile tone, prompt injections), runs them against your agent, and calculates a **Robustness Score**.
|
Instead of running one test case, FlakeStorm takes a single "Golden Prompt", generates adversarial mutations (semantic variations, noise injection, hostile tone, prompt injections), runs them against your agent, and calculates a **Robustness Score**.
|
||||||
|
|
||||||
> **"If it passes Entropix, it won't break in Production."**
|
> **"If it passes FlakeStorm, it won't break in Production."**
|
||||||
|
|
||||||
## Open Source vs Cloud
|
## Features
|
||||||
|
|
||||||
| Feature | Open Source (Free) | Cloud Pro ($49/mo) | Cloud Team ($299/mo) |
|
|
||||||
|---------|:------------------:|:------------------:|:--------------------:|
|
|
||||||
| Mutation Types | 5 basic | All types | All types |
|
|
||||||
| Mutations/Run | **50 max** | Unlimited | Unlimited |
|
|
||||||
| Execution | **Sequential** | ⚡ Parallel (20x) | ⚡ Parallel (20x) |
|
|
||||||
| LLM | Local only | Cloud + Local | Cloud + Local |
|
|
||||||
| PII Detection | Basic regex | Advanced NER + ML | Advanced NER + ML |
|
|
||||||
| Prompt Injection | Basic | ML-powered | ML-powered |
|
|
||||||
| Factuality Check | ❌ | ✅ | ✅ |
|
|
||||||
| Test History | ❌ | ✅ Dashboard | ✅ Dashboard |
|
|
||||||
| GitHub Actions | ❌ | ✅ One-click | ✅ One-click |
|
|
||||||
| Team Features | ❌ | ❌ | ✅ SSO + Sharing |
|
|
||||||
|
|
||||||
**Why the difference?**
|
|
||||||
|
|
||||||
```
|
|
||||||
Developer workflow:
|
|
||||||
1. Make code change
|
|
||||||
2. Run Entropix tests (waiting...)
|
|
||||||
3. Get results
|
|
||||||
4. Fix issues
|
|
||||||
5. Repeat
|
|
||||||
|
|
||||||
Open Source: ~10 minutes per iteration → Run once, then skip
|
|
||||||
Cloud Pro: ~30 seconds per iteration → Run every commit
|
|
||||||
```
|
|
||||||
|
|
||||||
👉 [**Upgrade to Cloud**](https://entropix.cloud) for production workloads.
|
|
||||||
|
|
||||||
## Features (Open Source)
|
|
||||||
|
|
||||||
- ✅ **5 Mutation Types**: Paraphrasing, noise, tone shifts, basic adversarial, custom templates
|
- ✅ **5 Mutation Types**: Paraphrasing, noise, tone shifts, basic adversarial, custom templates
|
||||||
- ✅ **Invariant Assertions**: Deterministic checks, semantic similarity, basic safety
|
- ✅ **Invariant Assertions**: Deterministic checks, semantic similarity, basic safety
|
||||||
- ✅ **Local-First**: Uses Ollama with Qwen 3 8B for free testing
|
- ✅ **Local-First**: Uses Ollama with Qwen 3 8B for free testing
|
||||||
- ✅ **Beautiful Reports**: Interactive HTML reports with pass/fail matrices
|
- ✅ **Beautiful Reports**: Interactive HTML reports with pass/fail matrices
|
||||||
- ⚠️ **50 Mutations Max**: Per test run (upgrade to Cloud for unlimited)
|
- ✅ **50 Mutations Max**: Per test run
|
||||||
- ⚠️ **Sequential Only**: One test at a time (upgrade to Cloud for 20x parallel)
|
- ✅ **Sequential Execution**: One test at a time
|
||||||
- ❌ **No CI/CD**: GitHub Actions requires Cloud
|
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
### Installation
|
### Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install entropix
|
pip install flakestorm
|
||||||
```
|
```
|
||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
|
|
||||||
Entropix uses [Ollama](https://ollama.ai) for local model inference:
|
FlakeStorm uses [Ollama](https://ollama.ai) for local model inference:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Install Ollama (macOS/Linux)
|
# Install Ollama (macOS/Linux)
|
||||||
|
|
@ -109,10 +73,10 @@ ollama pull qwen3:8b
|
||||||
### Initialize Configuration
|
### Initialize Configuration
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
entropix init
|
flakestorm init
|
||||||
```
|
```
|
||||||
|
|
||||||
This creates an `entropix.yaml` configuration file:
|
This creates a `flakestorm.yaml` configuration file:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
version: "1.0"
|
version: "1.0"
|
||||||
|
|
@ -128,7 +92,7 @@ model:
|
||||||
base_url: "http://localhost:11434"
|
base_url: "http://localhost:11434"
|
||||||
|
|
||||||
mutations:
|
mutations:
|
||||||
count: 10 # Max 50 total per run in Open Source
|
count: 10 # Max 50 total per run
|
||||||
types:
|
types:
|
||||||
- paraphrase
|
- paraphrase
|
||||||
- noise
|
- noise
|
||||||
|
|
@ -152,13 +116,11 @@ output:
|
||||||
### Run Tests
|
### Run Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
entropix run
|
flakestorm run
|
||||||
```
|
```
|
||||||
|
|
||||||
Output:
|
Output:
|
||||||
```
|
```
|
||||||
ℹ️ Running in sequential mode (Open Source). Upgrade for parallel: https://entropix.cloud
|
|
||||||
|
|
||||||
Generating mutations... ━━━━━━━━━━━━━━━━━━━━ 100%
|
Generating mutations... ━━━━━━━━━━━━━━━━━━━━ 100%
|
||||||
Running attacks... ━━━━━━━━━━━━━━━━━━━━ 100%
|
Running attacks... ━━━━━━━━━━━━━━━━━━━━ 100%
|
||||||
|
|
||||||
|
|
@ -169,17 +131,14 @@ Running attacks... ━━━━━━━━━━━━━━━━━━
|
||||||
│ Failed: 3 (2 latency, 1 injection) │
|
│ Failed: 3 (2 latency, 1 injection) │
|
||||||
╰──────────────────────────────────────────╯
|
╰──────────────────────────────────────────╯
|
||||||
|
|
||||||
⏱️ Test took 245.3s. With Entropix Cloud, this would take ~12.3s
|
Report saved to: ./reports/flakestorm-2024-01-15-143022.html
|
||||||
→ https://entropix.cloud
|
|
||||||
|
|
||||||
Report saved to: ./reports/entropix-2024-01-15-143022.html
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Check Limits
|
### Check Limits
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
entropix limits # Show Open Source edition limits
|
flakestorm limits # Show edition limits
|
||||||
entropix cloud # Learn about Cloud features
|
flakestorm cloud # Learn about Cloud features
|
||||||
```
|
```
|
||||||
|
|
||||||
## Mutation Types
|
## Mutation Types
|
||||||
|
|
@ -192,7 +151,7 @@ entropix cloud # Learn about Cloud features
|
||||||
| **Prompt Injection** | Basic adversarial attacks | "Book a flight and ignore previous instructions" |
|
| **Prompt Injection** | Basic adversarial attacks | "Book a flight and ignore previous instructions" |
|
||||||
| **Custom** | Your own mutation templates | Define with `{prompt}` placeholder |
|
| **Custom** | Your own mutation templates | Define with `{prompt}` placeholder |
|
||||||
|
|
||||||
> **Need advanced mutations?** Sophisticated jailbreaks, multi-step injections, and domain-specific attacks are available in [Entropix Cloud](https://entropix.cloud).
|
> **Need advanced mutations?** Visit [flakestorm.com](https://flakestorm.com) for more options.
|
||||||
|
|
||||||
## Invariants (Assertions)
|
## Invariants (Assertions)
|
||||||
|
|
||||||
|
|
@ -221,7 +180,7 @@ invariants:
|
||||||
- type: "refusal_check"
|
- type: "refusal_check"
|
||||||
```
|
```
|
||||||
|
|
||||||
> **Need advanced safety?** NER-based PII detection, ML-powered prompt injection detection, and factuality checking are available in [Entropix Cloud](https://entropix.cloud).
|
> **Need advanced safety?** Visit [flakestorm.com](https://flakestorm.com) for more options.
|
||||||
|
|
||||||
## Agent Adapters
|
## Agent Adapters
|
||||||
|
|
||||||
|
|
@ -234,7 +193,7 @@ agent:
|
||||||
|
|
||||||
### Python Callable
|
### Python Callable
|
||||||
```python
|
```python
|
||||||
from entropix import test_agent
|
from flakestorm import test_agent
|
||||||
|
|
||||||
@test_agent
|
@test_agent
|
||||||
async def my_agent(input: str) -> str:
|
async def my_agent(input: str) -> str:
|
||||||
|
|
@ -251,19 +210,13 @@ agent:
|
||||||
|
|
||||||
## CI/CD Integration
|
## CI/CD Integration
|
||||||
|
|
||||||
> ⚠️ **Cloud Feature**: GitHub Actions integration requires [Entropix Cloud](https://entropix.cloud).
|
For local testing:
|
||||||
|
|
||||||
For local testing only:
|
|
||||||
```bash
|
```bash
|
||||||
# Run before committing (manual)
|
# Run before committing (manual)
|
||||||
entropix run --min-score 0.9
|
flakestorm run --min-score 0.9
|
||||||
```
|
```
|
||||||
|
|
||||||
With Entropix Cloud, you get:
|
For advanced CI/CD features, visit [flakestorm.com](https://flakestorm.com).
|
||||||
- One-click GitHub Actions setup
|
|
||||||
- Automatic PR blocking below threshold
|
|
||||||
- Test history comparison
|
|
||||||
- Slack/Discord notifications
|
|
||||||
|
|
||||||
## Robustness Score
|
## Robustness Score
|
||||||
|
|
||||||
|
|
@ -301,12 +254,12 @@ AGPLv3 - See [LICENSE](LICENSE) for details.
|
||||||
---
|
---
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<strong>Tested with Entropix</strong><br>
|
<strong>Tested with FlakeStorm</strong><br>
|
||||||
<img src="https://img.shields.io/badge/tested%20with-entropix-brightgreen" alt="Tested with Entropix">
|
<img src="https://img.shields.io/badge/tested%20with-flakestorm-brightgreen" alt="Tested with FlakeStorm">
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://entropix.cloud">
|
<a href="https://flakestorm.com">
|
||||||
<strong>⚡ Need speed? Try Entropix Cloud →</strong>
|
<strong>⚡ Need more features? Visit FlakeStorm Cloud →</strong>
|
||||||
</a>
|
</a>
|
||||||
</p>
|
</p>
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Broken Agent Example
|
# Broken Agent Example
|
||||||
|
|
||||||
This example demonstrates a deliberately fragile AI agent that Entropix can detect issues with.
|
This example demonstrates a deliberately fragile AI agent that flakestorm can detect issues with.
|
||||||
|
|
||||||
## The "Broken" Agent
|
## The "Broken" Agent
|
||||||
|
|
||||||
|
|
@ -21,11 +21,11 @@ pip install fastapi uvicorn
|
||||||
uvicorn agent:app --port 8000
|
uvicorn agent:app --port 8000
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. Run Entropix Against It
|
### 2. Run flakestorm Against It
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# From the project root
|
# From the project root
|
||||||
entropix run --config examples/broken_agent/entropix.yaml
|
flakestorm run --config examples/broken_agent/flakestorm.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. See the Failures
|
### 3. See the Failures
|
||||||
|
|
@ -44,4 +44,4 @@ Try modifying `agent.py` to:
|
||||||
3. Handle emotional inputs gracefully
|
3. Handle emotional inputs gracefully
|
||||||
4. Detect and refuse prompt injections
|
4. Detect and refuse prompt injections
|
||||||
|
|
||||||
Then re-run Entropix to see your robustness score improve!
|
Then re-run flakestorm to see your robustness score improve!
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
"""
|
"""
|
||||||
Broken Agent Example
|
Broken Agent Example
|
||||||
|
|
||||||
A deliberately fragile AI agent to demonstrate Entropix testing.
|
A deliberately fragile AI agent to demonstrate flakestorm testing.
|
||||||
This agent has multiple intentional weaknesses that Entropix will find.
|
This agent has multiple intentional weaknesses that flakestorm will find.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,13 @@
|
||||||
# Entropix Configuration File
|
# flakestorm Configuration File
|
||||||
# The Agent Reliability Engine - Chaos Engineering for AI Agents
|
# The Agent Reliability Engine - Chaos Engineering for AI Agents
|
||||||
#
|
#
|
||||||
# This file defines how Entropix tests your AI agent for reliability.
|
# This file defines how flakestorm tests your AI agent for reliability.
|
||||||
# Copy this file to `entropix.yaml` and customize for your agent.
|
# Copy this file to `flakestorm.yaml` and customize for your agent.
|
||||||
|
|
||||||
version: "1.0"
|
version: "1.0"
|
||||||
|
|
||||||
# Agent Configuration
|
# Agent Configuration
|
||||||
# Define how Entropix connects to your agent
|
# Define how flakestorm connects to your agent
|
||||||
agent:
|
agent:
|
||||||
# HTTP endpoint that accepts POST requests with {"input": "..."} body
|
# HTTP endpoint that accepts POST requests with {"input": "..."} body
|
||||||
endpoint: "http://localhost:8000/invoke"
|
endpoint: "http://localhost:8000/invoke"
|
||||||
|
|
@ -60,7 +60,7 @@ mutations:
|
||||||
|
|
||||||
# Golden Prompts
|
# Golden Prompts
|
||||||
# Your "ideal" user inputs that the agent should handle correctly
|
# Your "ideal" user inputs that the agent should handle correctly
|
||||||
# Entropix will generate mutations of these and verify the agent still works
|
# flakestorm will generate mutations of these and verify the agent still works
|
||||||
golden_prompts:
|
golden_prompts:
|
||||||
- "Book a flight to Paris for next Monday"
|
- "Book a flight to Paris for next Monday"
|
||||||
- "What's my account balance?"
|
- "What's my account balance?"
|
||||||
|
|
@ -87,7 +87,7 @@ invariants:
|
||||||
# pattern: "^\\{.*\\}$"
|
# pattern: "^\\{.*\\}$"
|
||||||
# description: "Response must be a JSON object"
|
# description: "Response must be a JSON object"
|
||||||
|
|
||||||
# Semantic Checks (requires 'semantic' extra: pip install entropix[semantic])
|
# Semantic Checks (requires 'semantic' extra: pip install flakestorm[semantic])
|
||||||
# - type: "similarity"
|
# - type: "similarity"
|
||||||
# expected: "Your request has been processed successfully"
|
# expected: "Your request has been processed successfully"
|
||||||
# threshold: 0.8
|
# threshold: 0.8
|
||||||
|
|
@ -110,7 +110,7 @@ output:
|
||||||
path: "./reports"
|
path: "./reports"
|
||||||
|
|
||||||
# Optional: Custom report filename template
|
# Optional: Custom report filename template
|
||||||
# filename_template: "entropix-{date}-{time}"
|
# filename_template: "flakestorm-{date}-{time}"
|
||||||
|
|
||||||
# Advanced Configuration
|
# Advanced Configuration
|
||||||
# advanced:
|
# advanced:
|
||||||
|
|
@ -3,14 +3,14 @@ requires = ["hatchling", "hatch-fancy-pypi-readme"]
|
||||||
build-backend = "hatchling.build"
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "entropix"
|
name = "flakestorm"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "The Agent Reliability Engine - Chaos Engineering for AI Agents"
|
description = "The Agent Reliability Engine - Chaos Engineering for AI Agents"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
authors = [
|
authors = [
|
||||||
{ name = "Entropix Team" }
|
{ name = "flakestorm Team" }
|
||||||
]
|
]
|
||||||
keywords = [
|
keywords = [
|
||||||
"ai",
|
"ai",
|
||||||
|
|
@ -66,20 +66,20 @@ huggingface = [
|
||||||
"huggingface-hub>=0.19.0",
|
"huggingface-hub>=0.19.0",
|
||||||
]
|
]
|
||||||
all = [
|
all = [
|
||||||
"entropix[dev,semantic,huggingface]",
|
"flakestorm[dev,semantic,huggingface]",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
entropix = "entropix.cli.main:app"
|
flakestorm = "flakestorm.cli.main:app"
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
Homepage = "https://github.com/entropix/entropix"
|
Homepage = "https://github.com/flakestorm/flakestorm"
|
||||||
Documentation = "https://entropix.dev/docs"
|
Documentation = "https://flakestorm.dev/docs"
|
||||||
Repository = "https://github.com/entropix/entropix"
|
Repository = "https://github.com/flakestorm/flakestorm"
|
||||||
Issues = "https://github.com/entropix/entropix/issues"
|
Issues = "https://github.com/flakestorm/flakestorm/issues"
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
[tool.hatch.build.targets.wheel]
|
||||||
packages = ["src/entropix"]
|
packages = ["src/flakestorm"]
|
||||||
|
|
||||||
[tool.hatch.build.targets.sdist]
|
[tool.hatch.build.targets.sdist]
|
||||||
include = [
|
include = [
|
||||||
|
|
@ -115,7 +115,7 @@ ignore = [
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.ruff.lint.isort]
|
[tool.ruff.lint.isort]
|
||||||
known-first-party = ["entropix"]
|
known-first-party = ["flakestorm"]
|
||||||
|
|
||||||
[tool.mypy]
|
[tool.mypy]
|
||||||
python_version = "3.10"
|
python_version = "3.10"
|
||||||
|
|
@ -145,4 +145,4 @@ skips = ["B101"] # Skip assert warnings (used in tests)
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
testpaths = ["tests"]
|
testpaths = ["tests"]
|
||||||
asyncio_mode = "auto"
|
asyncio_mode = "auto"
|
||||||
addopts = "-v --cov=src/entropix --cov-report=term-missing"
|
addopts = "-v --cov=src/flakestorm --cov-report=term-missing"
|
||||||
|
|
|
||||||
199
refactor_to_flakestorm.py
Executable file
199
refactor_to_flakestorm.py
Executable file
|
|
@ -0,0 +1,199 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Refactoring script to rename Entropix -> FlakeStorm
|
||||||
|
This script will:
|
||||||
|
1. Rename directories and files containing 'entropix'
|
||||||
|
2. Replace all text occurrences of 'Entropix' -> 'FlakeStorm' and 'entropix' -> 'flakestorm'
|
||||||
|
3. Update imports, package names, URLs, etc.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Base directory
|
||||||
|
BASE_DIR = Path(__file__).parent
|
||||||
|
|
||||||
|
# Files and directories to skip
|
||||||
|
SKIP_PATTERNS = [
|
||||||
|
".git",
|
||||||
|
".venv",
|
||||||
|
"venv",
|
||||||
|
"__pycache__",
|
||||||
|
".pytest_cache",
|
||||||
|
"target",
|
||||||
|
"dist",
|
||||||
|
"build",
|
||||||
|
".mypy_cache",
|
||||||
|
".ruff_cache",
|
||||||
|
"node_modules",
|
||||||
|
"refactor_to_flakestorm.py", # Don't refactor this script itself
|
||||||
|
]
|
||||||
|
|
||||||
|
# Text replacements: (pattern, replacement, case_sensitive)
|
||||||
|
TEXT_REPLACEMENTS = [
|
||||||
|
# Package names
|
||||||
|
(r"\bentropix\b", "flakestorm", False),
|
||||||
|
(r"\bEntropix\b", "FlakeStorm", True),
|
||||||
|
# URLs
|
||||||
|
(r"entropix\.cloud", "flakestorm.com", False),
|
||||||
|
(r"entropix\.dev", "flakestorm.com", False),
|
||||||
|
(r"github\.com/entropix/entropix", "github.com/flakestorm/flakestorm", False),
|
||||||
|
(r"github\.com/entropixai/entropix", "github.com/flakestorm/flakestorm", False),
|
||||||
|
# PyPI
|
||||||
|
(r"pypi\.org/project/entropix", "pypi.org/project/flakestorm", False),
|
||||||
|
# File paths in text
|
||||||
|
(r"entropix\.yaml", "flakestorm.yaml", False),
|
||||||
|
(r"entropix\.py", "flakestorm.py", False),
|
||||||
|
# Module imports
|
||||||
|
(r"from entropix", "from flakestorm", False),
|
||||||
|
(r"import entropix", "import flakestorm", False),
|
||||||
|
(r"entropix\.", "flakestorm.", False),
|
||||||
|
# CLI command
|
||||||
|
(r"`entropix ", "`flakestorm ", False),
|
||||||
|
(r'"entropix ', '"flakestorm ', False),
|
||||||
|
(r"'entropix ", "'flakestorm ", False),
|
||||||
|
# Badge text
|
||||||
|
(r"tested%20with-entropix", "tested%20with-flakestorm", False),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def should_skip(path: Path) -> bool:
|
||||||
|
"""Check if a path should be skipped."""
|
||||||
|
path_str = str(path)
|
||||||
|
return any(pattern in path_str for pattern in SKIP_PATTERNS)
|
||||||
|
|
||||||
|
|
||||||
|
def find_files_to_rename() -> list[tuple[Path, Path]]:
|
||||||
|
"""Find all files and directories that need renaming."""
|
||||||
|
renames = []
|
||||||
|
|
||||||
|
# Find directories named 'entropix'
|
||||||
|
for root, dirs, files in os.walk(BASE_DIR):
|
||||||
|
root_path = Path(root)
|
||||||
|
if should_skip(root_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check directory names
|
||||||
|
dirs_copy = dirs[:] # Copy list to modify during iteration
|
||||||
|
for dir_name in dirs_copy:
|
||||||
|
if "entropix" in dir_name.lower():
|
||||||
|
old_path = root_path / dir_name
|
||||||
|
if not should_skip(old_path):
|
||||||
|
new_name = dir_name.replace("entropix", "flakestorm").replace(
|
||||||
|
"Entropix", "FlakeStorm"
|
||||||
|
)
|
||||||
|
new_path = root_path / new_name
|
||||||
|
renames.append((old_path, new_path))
|
||||||
|
dirs.remove(dir_name) # Don't walk into renamed dir
|
||||||
|
|
||||||
|
# Check file names
|
||||||
|
for file_name in files:
|
||||||
|
if "entropix" in file_name.lower():
|
||||||
|
old_path = root_path / file_name
|
||||||
|
if not should_skip(old_path):
|
||||||
|
new_name = file_name.replace("entropix", "flakestorm").replace(
|
||||||
|
"Entropix", "FlakeStorm"
|
||||||
|
)
|
||||||
|
new_path = root_path / new_name
|
||||||
|
renames.append((old_path, new_path))
|
||||||
|
|
||||||
|
return renames
|
||||||
|
|
||||||
|
|
||||||
|
def replace_in_file(file_path: Path) -> bool:
|
||||||
|
"""Replace text in a file. Returns True if file was modified."""
|
||||||
|
try:
|
||||||
|
# Skip binary files
|
||||||
|
if file_path.suffix in [
|
||||||
|
".pyc",
|
||||||
|
".pyo",
|
||||||
|
".so",
|
||||||
|
".dylib",
|
||||||
|
".dll",
|
||||||
|
".png",
|
||||||
|
".jpg",
|
||||||
|
".jpeg",
|
||||||
|
".gif",
|
||||||
|
".ico",
|
||||||
|
".pdf",
|
||||||
|
]:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Read file
|
||||||
|
try:
|
||||||
|
with open(file_path, encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
# Skip binary files
|
||||||
|
return False
|
||||||
|
|
||||||
|
original_content = content
|
||||||
|
|
||||||
|
# Apply all replacements
|
||||||
|
for pattern, replacement, case_sensitive in TEXT_REPLACEMENTS:
|
||||||
|
flags = 0 if case_sensitive else re.IGNORECASE
|
||||||
|
content = re.sub(pattern, replacement, content, flags=flags)
|
||||||
|
|
||||||
|
# Write back if changed
|
||||||
|
if content != original_content:
|
||||||
|
with open(file_path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(content)
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error processing {file_path}: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Main refactoring function."""
|
||||||
|
print("🔍 Finding files and directories to rename...")
|
||||||
|
renames = find_files_to_rename()
|
||||||
|
|
||||||
|
print(f"📝 Found {len(renames)} items to rename")
|
||||||
|
for old, new in renames:
|
||||||
|
print(f" {old.relative_to(BASE_DIR)} -> {new.relative_to(BASE_DIR)}")
|
||||||
|
|
||||||
|
# Rename files and directories (in reverse order to handle nested paths)
|
||||||
|
renames.sort(key=lambda x: len(str(x[0])), reverse=True)
|
||||||
|
|
||||||
|
print("\n🔄 Renaming files and directories...")
|
||||||
|
for old_path, new_path in renames:
|
||||||
|
if old_path.exists():
|
||||||
|
try:
|
||||||
|
if old_path.is_dir():
|
||||||
|
shutil.move(str(old_path), str(new_path))
|
||||||
|
else:
|
||||||
|
old_path.rename(new_path)
|
||||||
|
print(f" ✓ Renamed: {old_path.relative_to(BASE_DIR)}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f" ✗ Error renaming {old_path}: {e}")
|
||||||
|
|
||||||
|
# Replace text in all files
|
||||||
|
print("\n📝 Replacing text in files...")
|
||||||
|
modified_count = 0
|
||||||
|
for root, dirs, files in os.walk(BASE_DIR):
|
||||||
|
root_path = Path(root)
|
||||||
|
if should_skip(root_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
for file_name in files:
|
||||||
|
file_path = root_path / file_name
|
||||||
|
if should_skip(file_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if replace_in_file(file_path):
|
||||||
|
modified_count += 1
|
||||||
|
print(f" ✓ Modified: {file_path.relative_to(BASE_DIR)}")
|
||||||
|
|
||||||
|
print("\n✅ Refactoring complete!")
|
||||||
|
print(f" - Renamed {len(renames)} files/directories")
|
||||||
|
print(f" - Modified {modified_count} files with text replacements")
|
||||||
|
print("\n⚠️ Please review the changes and test before committing!")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -1,12 +1,12 @@
|
||||||
[package]
|
[package]
|
||||||
name = "entropix_rust"
|
name = "flakestorm_rust"
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
authors.workspace = true
|
authors.workspace = true
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
name = "entropix_rust"
|
name = "flakestorm_rust"
|
||||||
crate-type = ["cdylib"]
|
crate-type = ["cdylib"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|
|
||||||
|
|
@ -3,9 +3,9 @@ requires = ["maturin>=1.4,<2.0"]
|
||||||
build-backend = "maturin"
|
build-backend = "maturin"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "entropix_rust"
|
name = "flakestorm_rust"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
description = "High-performance Rust extensions for Entropix"
|
description = "High-performance Rust extensions for flakestorm"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
classifiers = [
|
classifiers = [
|
||||||
"Programming Language :: Python :: 3",
|
"Programming Language :: Python :: 3",
|
||||||
|
|
@ -18,4 +18,4 @@ classifiers = [
|
||||||
|
|
||||||
[tool.maturin]
|
[tool.maturin]
|
||||||
features = ["pyo3/extension-module"]
|
features = ["pyo3/extension-module"]
|
||||||
module-name = "entropix_rust"
|
module-name = "flakestorm_rust"
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
//! Entropix Rust Performance Module
|
//! flakestorm Rust Performance Module
|
||||||
//!
|
//!
|
||||||
//! This module provides high-performance implementations for:
|
//! This module provides high-performance implementations for:
|
||||||
//! - Robustness score calculation
|
//! - Robustness score calculation
|
||||||
|
|
@ -140,7 +140,7 @@ fn string_similarity(s1: &str, s2: &str) -> f64 {
|
||||||
|
|
||||||
/// Python module definition
|
/// Python module definition
|
||||||
#[pymodule]
|
#[pymodule]
|
||||||
fn entropix_rust(_py: Python, m: &PyModule) -> PyResult<()> {
|
fn flakestorm_rust(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||||
m.add_function(wrap_pyfunction!(calculate_robustness_score, m)?)?;
|
m.add_function(wrap_pyfunction!(calculate_robustness_score, m)?)?;
|
||||||
m.add_function(wrap_pyfunction!(calculate_weighted_score, m)?)?;
|
m.add_function(wrap_pyfunction!(calculate_weighted_score, m)?)?;
|
||||||
m.add_function(wrap_pyfunction!(parallel_process_mutations, m)?)?;
|
m.add_function(wrap_pyfunction!(parallel_process_mutations, m)?)?;
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
//! Parallel processing utilities for Entropix
|
//! Parallel processing utilities for flakestorm
|
||||||
//!
|
//!
|
||||||
//! This module provides efficient parallel processing for mutation generation
|
//! This module provides efficient parallel processing for mutation generation
|
||||||
//! and agent testing using Rayon.
|
//! and agent testing using Rayon.
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
//! Scoring algorithms for Entropix
|
//! Scoring algorithms for flakestorm
|
||||||
//!
|
//!
|
||||||
//! This module contains optimized scoring algorithms for calculating
|
//! This module contains optimized scoring algorithms for calculating
|
||||||
//! robustness metrics and aggregating test results.
|
//! robustness metrics and aggregating test results.
|
||||||
|
|
|
||||||
|
|
@ -1,42 +1,42 @@
|
||||||
"""
|
"""
|
||||||
Entropix - The Agent Reliability Engine
|
flakestorm - The Agent Reliability Engine
|
||||||
|
|
||||||
Chaos Engineering for AI Agents. Apply adversarial fuzzing to prove
|
Chaos Engineering for AI Agents. Apply adversarial fuzzing to prove
|
||||||
your agents are production-ready before deployment.
|
your agents are production-ready before deployment.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
>>> from entropix import EntropixRunner, load_config
|
>>> from flakestorm import flakestormRunner, load_config
|
||||||
>>> config = load_config("entropix.yaml")
|
>>> config = load_config("flakestorm.yaml")
|
||||||
>>> runner = EntropixRunner(config)
|
>>> runner = FlakeStormRunner(config)
|
||||||
>>> results = await runner.run()
|
>>> results = await runner.run()
|
||||||
>>> print(f"Robustness Score: {results.robustness_score:.1%}")
|
>>> print(f"Robustness Score: {results.robustness_score:.1%}")
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__version__ = "0.1.0"
|
__version__ = "0.1.0"
|
||||||
__author__ = "Entropix Team"
|
__author__ = "flakestorm Team"
|
||||||
__license__ = "Apache-2.0"
|
__license__ = "Apache-2.0"
|
||||||
|
|
||||||
from entropix.assertions.verifier import InvariantVerifier, VerificationResult
|
from flakestorm.assertions.verifier import InvariantVerifier, VerificationResult
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
InvariantConfig,
|
InvariantConfig,
|
||||||
ModelConfig,
|
ModelConfig,
|
||||||
MutationConfig,
|
MutationConfig,
|
||||||
OutputConfig,
|
OutputConfig,
|
||||||
load_config,
|
load_config,
|
||||||
)
|
)
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
from entropix.core.protocol import (
|
from flakestorm.core.protocol import (
|
||||||
AgentProtocol,
|
AgentProtocol,
|
||||||
HTTPAgentAdapter,
|
HTTPAgentAdapter,
|
||||||
PythonAgentAdapter,
|
PythonAgentAdapter,
|
||||||
create_agent_adapter,
|
create_agent_adapter,
|
||||||
)
|
)
|
||||||
from entropix.core.runner import EntropixRunner
|
from flakestorm.core.runner import FlakeStormRunner
|
||||||
from entropix.mutations.engine import MutationEngine
|
from flakestorm.mutations.engine import MutationEngine
|
||||||
from entropix.mutations.types import Mutation, MutationType
|
from flakestorm.mutations.types import Mutation, MutationType
|
||||||
from entropix.reports.models import TestResults, TestStatistics
|
from flakestorm.reports.models import TestResults, TestStatistics
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
# Version info
|
# Version info
|
||||||
|
|
@ -44,7 +44,7 @@ __all__ = [
|
||||||
"__author__",
|
"__author__",
|
||||||
"__license__",
|
"__license__",
|
||||||
# Configuration
|
# Configuration
|
||||||
"EntropixConfig",
|
"FlakeStormConfig",
|
||||||
"load_config",
|
"load_config",
|
||||||
"AgentConfig",
|
"AgentConfig",
|
||||||
"ModelConfig",
|
"ModelConfig",
|
||||||
|
|
@ -57,7 +57,7 @@ __all__ = [
|
||||||
"PythonAgentAdapter",
|
"PythonAgentAdapter",
|
||||||
"create_agent_adapter",
|
"create_agent_adapter",
|
||||||
# Core
|
# Core
|
||||||
"EntropixRunner",
|
"FlakeStormRunner",
|
||||||
"Orchestrator",
|
"Orchestrator",
|
||||||
# Mutations
|
# Mutations
|
||||||
"MutationEngine",
|
"MutationEngine",
|
||||||
|
|
@ -1,22 +1,22 @@
|
||||||
"""
|
"""
|
||||||
Entropix Assertions (Invariants) System
|
flakestorm Assertions (Invariants) System
|
||||||
|
|
||||||
Provides verification of agent responses against defined invariants.
|
Provides verification of agent responses against defined invariants.
|
||||||
Supports deterministic checks, semantic similarity, and safety validations.
|
Supports deterministic checks, semantic similarity, and safety validations.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from entropix.assertions.deterministic import (
|
from flakestorm.assertions.deterministic import (
|
||||||
ContainsChecker,
|
ContainsChecker,
|
||||||
LatencyChecker,
|
LatencyChecker,
|
||||||
RegexChecker,
|
RegexChecker,
|
||||||
ValidJsonChecker,
|
ValidJsonChecker,
|
||||||
)
|
)
|
||||||
from entropix.assertions.safety import (
|
from flakestorm.assertions.safety import (
|
||||||
ExcludesPIIChecker,
|
ExcludesPIIChecker,
|
||||||
RefusalChecker,
|
RefusalChecker,
|
||||||
)
|
)
|
||||||
from entropix.assertions.semantic import SimilarityChecker
|
from flakestorm.assertions.semantic import SimilarityChecker
|
||||||
from entropix.assertions.verifier import (
|
from flakestorm.assertions.verifier import (
|
||||||
CheckResult,
|
CheckResult,
|
||||||
InvariantVerifier,
|
InvariantVerifier,
|
||||||
VerificationResult,
|
VerificationResult,
|
||||||
|
|
@ -17,7 +17,7 @@ from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from entropix.core.config import InvariantConfig, InvariantType
|
from flakestorm.core.config import InvariantConfig, InvariantType
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -76,7 +76,7 @@ class ContainsChecker(BaseChecker):
|
||||||
|
|
||||||
def check(self, response: str, latency_ms: float) -> CheckResult:
|
def check(self, response: str, latency_ms: float) -> CheckResult:
|
||||||
"""Check if response contains the required value."""
|
"""Check if response contains the required value."""
|
||||||
from entropix.core.config import InvariantType
|
from flakestorm.core.config import InvariantType
|
||||||
|
|
||||||
value = self.config.value or ""
|
value = self.config.value or ""
|
||||||
passed = value.lower() in response.lower()
|
passed = value.lower() in response.lower()
|
||||||
|
|
@ -104,7 +104,7 @@ class LatencyChecker(BaseChecker):
|
||||||
|
|
||||||
def check(self, response: str, latency_ms: float) -> CheckResult:
|
def check(self, response: str, latency_ms: float) -> CheckResult:
|
||||||
"""Check if latency is within threshold."""
|
"""Check if latency is within threshold."""
|
||||||
from entropix.core.config import InvariantType
|
from flakestorm.core.config import InvariantType
|
||||||
|
|
||||||
max_ms = self.config.max_ms or 5000
|
max_ms = self.config.max_ms or 5000
|
||||||
passed = latency_ms <= max_ms
|
passed = latency_ms <= max_ms
|
||||||
|
|
@ -131,7 +131,7 @@ class ValidJsonChecker(BaseChecker):
|
||||||
|
|
||||||
def check(self, response: str, latency_ms: float) -> CheckResult:
|
def check(self, response: str, latency_ms: float) -> CheckResult:
|
||||||
"""Check if response is valid JSON."""
|
"""Check if response is valid JSON."""
|
||||||
from entropix.core.config import InvariantType
|
from flakestorm.core.config import InvariantType
|
||||||
|
|
||||||
try:
|
try:
|
||||||
json.loads(response)
|
json.loads(response)
|
||||||
|
|
@ -159,7 +159,7 @@ class RegexChecker(BaseChecker):
|
||||||
|
|
||||||
def check(self, response: str, latency_ms: float) -> CheckResult:
|
def check(self, response: str, latency_ms: float) -> CheckResult:
|
||||||
"""Check if response matches the regex pattern."""
|
"""Check if response matches the regex pattern."""
|
||||||
from entropix.core.config import InvariantType
|
from flakestorm.core.config import InvariantType
|
||||||
|
|
||||||
pattern = self.config.pattern or ".*"
|
pattern = self.config.pattern or ".*"
|
||||||
|
|
||||||
|
|
@ -11,7 +11,7 @@ from __future__ import annotations
|
||||||
import re
|
import re
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from entropix.assertions.deterministic import BaseChecker, CheckResult
|
from flakestorm.assertions.deterministic import BaseChecker, CheckResult
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
pass
|
pass
|
||||||
|
|
@ -84,7 +84,7 @@ class ExcludesPIIChecker(BaseChecker):
|
||||||
|
|
||||||
def check(self, response: str, latency_ms: float) -> CheckResult:
|
def check(self, response: str, latency_ms: float) -> CheckResult:
|
||||||
"""Check for PII patterns in response."""
|
"""Check for PII patterns in response."""
|
||||||
from entropix.core.config import InvariantType
|
from flakestorm.core.config import InvariantType
|
||||||
|
|
||||||
found_pii = []
|
found_pii = []
|
||||||
|
|
||||||
|
|
@ -125,7 +125,7 @@ class RefusalChecker(BaseChecker):
|
||||||
|
|
||||||
def check(self, response: str, latency_ms: float) -> CheckResult:
|
def check(self, response: str, latency_ms: float) -> CheckResult:
|
||||||
"""Check for proper refusal of dangerous content."""
|
"""Check for proper refusal of dangerous content."""
|
||||||
from entropix.core.config import InvariantType
|
from flakestorm.core.config import InvariantType
|
||||||
|
|
||||||
response_lower = response.lower()
|
response_lower = response.lower()
|
||||||
|
|
||||||
|
|
@ -4,7 +4,7 @@ Semantic Invariant Checkers
|
||||||
Checks that use embeddings to verify semantic similarity
|
Checks that use embeddings to verify semantic similarity
|
||||||
between expected and actual responses.
|
between expected and actual responses.
|
||||||
|
|
||||||
Requires the 'semantic' extra: pip install entropix[semantic]
|
Requires the 'semantic' extra: pip install flakestorm[semantic]
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -12,10 +12,10 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from entropix.assertions.deterministic import BaseChecker, CheckResult
|
from flakestorm.assertions.deterministic import BaseChecker, CheckResult
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from entropix.core.config import InvariantConfig
|
from flakestorm.core.config import InvariantConfig
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -50,7 +50,7 @@ class LocalEmbedder:
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"sentence-transformers is required for semantic checks. "
|
"sentence-transformers is required for semantic checks. "
|
||||||
"Install with: pip install entropix[semantic]"
|
"Install with: pip install flakestorm[semantic]"
|
||||||
)
|
)
|
||||||
return self._model
|
return self._model
|
||||||
|
|
||||||
|
|
@ -109,7 +109,7 @@ class SimilarityChecker(BaseChecker):
|
||||||
|
|
||||||
def check(self, response: str, latency_ms: float) -> CheckResult:
|
def check(self, response: str, latency_ms: float) -> CheckResult:
|
||||||
"""Check semantic similarity to expected response."""
|
"""Check semantic similarity to expected response."""
|
||||||
from entropix.core.config import InvariantType
|
from flakestorm.core.config import InvariantType
|
||||||
|
|
||||||
expected = self.config.expected or ""
|
expected = self.config.expected or ""
|
||||||
threshold = self.config.threshold or 0.8
|
threshold = self.config.threshold or 0.8
|
||||||
|
|
@ -10,7 +10,7 @@ from __future__ import annotations
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from entropix.assertions.deterministic import (
|
from flakestorm.assertions.deterministic import (
|
||||||
BaseChecker,
|
BaseChecker,
|
||||||
CheckResult,
|
CheckResult,
|
||||||
ContainsChecker,
|
ContainsChecker,
|
||||||
|
|
@ -18,11 +18,11 @@ from entropix.assertions.deterministic import (
|
||||||
RegexChecker,
|
RegexChecker,
|
||||||
ValidJsonChecker,
|
ValidJsonChecker,
|
||||||
)
|
)
|
||||||
from entropix.assertions.safety import ExcludesPIIChecker, RefusalChecker
|
from flakestorm.assertions.safety import ExcludesPIIChecker, RefusalChecker
|
||||||
from entropix.assertions.semantic import SimilarityChecker
|
from flakestorm.assertions.semantic import SimilarityChecker
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from entropix.core.config import InvariantConfig, InvariantType
|
from flakestorm.core.config import InvariantConfig, InvariantType
|
||||||
|
|
||||||
|
|
||||||
# Registry of checker classes by invariant type
|
# Registry of checker classes by invariant type
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
"""
|
"""
|
||||||
Entropix CLI
|
flakestorm CLI
|
||||||
|
|
||||||
Command-line interface for running reliability tests on AI agents.
|
Command-line interface for running reliability tests on AI agents.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from entropix.cli.main import app
|
from flakestorm.cli.main import app
|
||||||
|
|
||||||
__all__ = ["app"]
|
__all__ = ["app"]
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
"""
|
"""
|
||||||
Entropix CLI Main Entry Point
|
flakestorm CLI Main Entry Point
|
||||||
|
|
||||||
Provides the main Typer application and command routing.
|
Provides the main Typer application and command routing.
|
||||||
"""
|
"""
|
||||||
|
|
@ -15,16 +15,17 @@ from rich.console import Console
|
||||||
from rich.panel import Panel
|
from rich.panel import Panel
|
||||||
from rich.text import Text
|
from rich.text import Text
|
||||||
|
|
||||||
from entropix import __version__
|
from flakestorm import __version__
|
||||||
from entropix.core.limits import (
|
from flakestorm.core.limits import (
|
||||||
CLOUD_URL,
|
CLOUD_URL,
|
||||||
MAX_MUTATIONS_PER_RUN,
|
MAX_MUTATIONS_PER_RUN,
|
||||||
print_upgrade_banner,
|
print_upgrade_banner,
|
||||||
)
|
)
|
||||||
|
from flakestorm.core.runner import FlakeStormRunner
|
||||||
|
|
||||||
# Create the main app
|
# Create the main app
|
||||||
app = typer.Typer(
|
app = typer.Typer(
|
||||||
name="entropix",
|
name="flakestorm",
|
||||||
help="The Agent Reliability Engine - Chaos Engineering for AI Agents [Open Source Edition]",
|
help="The Agent Reliability Engine - Chaos Engineering for AI Agents [Open Source Edition]",
|
||||||
add_completion=True,
|
add_completion=True,
|
||||||
rich_markup_mode="rich",
|
rich_markup_mode="rich",
|
||||||
|
|
@ -37,7 +38,7 @@ def version_callback(value: bool) -> None:
|
||||||
"""Print version and exit."""
|
"""Print version and exit."""
|
||||||
if value:
|
if value:
|
||||||
console.print(
|
console.print(
|
||||||
f"[bold blue]Entropix[/bold blue] version {__version__} [dim](Open Source Edition)[/dim]"
|
f"[bold blue]flakestorm[/bold blue] version {__version__} [dim](Open Source Edition)[/dim]"
|
||||||
)
|
)
|
||||||
console.print(f"[dim]→ Upgrade to Cloud: {CLOUD_URL}[/dim]")
|
console.print(f"[dim]→ Upgrade to Cloud: {CLOUD_URL}[/dim]")
|
||||||
raise typer.Exit()
|
raise typer.Exit()
|
||||||
|
|
@ -55,7 +56,7 @@ def main(
|
||||||
),
|
),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Entropix - The Agent Reliability Engine
|
flakestorm - The Agent Reliability Engine
|
||||||
|
|
||||||
Apply chaos engineering to your AI agents. Generate adversarial
|
Apply chaos engineering to your AI agents. Generate adversarial
|
||||||
mutations, test reliability, and prove production readiness.
|
mutations, test reliability, and prove production readiness.
|
||||||
|
|
@ -66,7 +67,7 @@ def main(
|
||||||
@app.command()
|
@app.command()
|
||||||
def init(
|
def init(
|
||||||
path: Path = typer.Argument(
|
path: Path = typer.Argument(
|
||||||
Path("entropix.yaml"),
|
Path("flakestorm.yaml"),
|
||||||
help="Path for the configuration file",
|
help="Path for the configuration file",
|
||||||
),
|
),
|
||||||
force: bool = typer.Option(
|
force: bool = typer.Option(
|
||||||
|
|
@ -77,12 +78,12 @@ def init(
|
||||||
),
|
),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Initialize a new Entropix configuration file.
|
Initialize a new flakestorm configuration file.
|
||||||
|
|
||||||
Creates an entropix.yaml with sensible defaults that you can
|
Creates an flakestorm.yaml with sensible defaults that you can
|
||||||
customize for your agent.
|
customize for your agent.
|
||||||
"""
|
"""
|
||||||
from entropix.core.config import create_default_config
|
from flakestorm.core.config import create_default_config
|
||||||
|
|
||||||
if path.exists() and not force:
|
if path.exists() and not force:
|
||||||
console.print(
|
console.print(
|
||||||
|
|
@ -102,8 +103,8 @@ def init(
|
||||||
"Next steps:\n"
|
"Next steps:\n"
|
||||||
"1. Edit the file to configure your agent endpoint\n"
|
"1. Edit the file to configure your agent endpoint\n"
|
||||||
"2. Add your golden prompts\n"
|
"2. Add your golden prompts\n"
|
||||||
"3. Run: [bold]entropix run[/bold]",
|
"3. Run: [bold]flakestorm run[/bold]",
|
||||||
title="Entropix Initialized",
|
title="flakestorm Initialized",
|
||||||
border_style="green",
|
border_style="green",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
@ -112,7 +113,7 @@ def init(
|
||||||
@app.command()
|
@app.command()
|
||||||
def run(
|
def run(
|
||||||
config: Path = typer.Option(
|
config: Path = typer.Option(
|
||||||
Path("entropix.yaml"),
|
Path("flakestorm.yaml"),
|
||||||
"--config",
|
"--config",
|
||||||
"-c",
|
"-c",
|
||||||
help="Path to configuration file",
|
help="Path to configuration file",
|
||||||
|
|
@ -172,22 +173,21 @@ async def _run_async(
|
||||||
quiet: bool,
|
quiet: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Async implementation of the run command."""
|
"""Async implementation of the run command."""
|
||||||
from entropix.core.runner import EntropixRunner
|
from flakestorm.reports.html import HTMLReportGenerator
|
||||||
from entropix.reports.html import HTMLReportGenerator
|
from flakestorm.reports.json_export import JSONReportGenerator
|
||||||
from entropix.reports.json_export import JSONReportGenerator
|
from flakestorm.reports.terminal import TerminalReporter
|
||||||
from entropix.reports.terminal import TerminalReporter
|
|
||||||
|
|
||||||
# Print header
|
# Print header
|
||||||
if not quiet:
|
if not quiet:
|
||||||
console.print()
|
console.print()
|
||||||
console.print(
|
console.print(
|
||||||
f"[bold blue]Entropix[/bold blue] - Agent Reliability Engine v{__version__}"
|
f"[bold blue]flakestorm[/bold blue] - Agent Reliability Engine v{__version__}"
|
||||||
)
|
)
|
||||||
console.print()
|
console.print()
|
||||||
|
|
||||||
# Load configuration
|
# Load configuration
|
||||||
try:
|
try:
|
||||||
runner = EntropixRunner(
|
runner = FlakeStormRunner(
|
||||||
config=config,
|
config=config,
|
||||||
console=console,
|
console=console,
|
||||||
show_progress=not quiet,
|
show_progress=not quiet,
|
||||||
|
|
@ -195,7 +195,7 @@ async def _run_async(
|
||||||
except FileNotFoundError as e:
|
except FileNotFoundError as e:
|
||||||
console.print(f"[red]Error:[/red] {e}")
|
console.print(f"[red]Error:[/red] {e}")
|
||||||
console.print(
|
console.print(
|
||||||
"\n[dim]Run 'entropix init' to create a configuration file.[/dim]"
|
"\n[dim]Run 'flakestorm init' to create a configuration file.[/dim]"
|
||||||
)
|
)
|
||||||
raise typer.Exit(1)
|
raise typer.Exit(1)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
@ -254,14 +254,14 @@ async def _run_async(
|
||||||
@app.command()
|
@app.command()
|
||||||
def verify(
|
def verify(
|
||||||
config: Path = typer.Option(
|
config: Path = typer.Option(
|
||||||
Path("entropix.yaml"),
|
Path("flakestorm.yaml"),
|
||||||
"--config",
|
"--config",
|
||||||
"-c",
|
"-c",
|
||||||
help="Path to configuration file",
|
help="Path to configuration file",
|
||||||
),
|
),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Verify that Entropix is properly configured.
|
Verify that flakestorm is properly configured.
|
||||||
|
|
||||||
Checks:
|
Checks:
|
||||||
- Ollama server is running and model is available
|
- Ollama server is running and model is available
|
||||||
|
|
@ -273,14 +273,13 @@ def verify(
|
||||||
|
|
||||||
async def _verify_async(config: Path) -> None:
|
async def _verify_async(config: Path) -> None:
|
||||||
"""Async implementation of verify command."""
|
"""Async implementation of verify command."""
|
||||||
from entropix.core.runner import EntropixRunner
|
|
||||||
|
|
||||||
console.print()
|
console.print()
|
||||||
console.print("[bold blue]Entropix[/bold blue] - Setup Verification")
|
console.print("[bold blue]flakestorm[/bold blue] - Setup Verification")
|
||||||
console.print()
|
console.print()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
runner = EntropixRunner(
|
runner = FlakeStormRunner(
|
||||||
config=config,
|
config=config,
|
||||||
console=console,
|
console=console,
|
||||||
show_progress=False,
|
show_progress=False,
|
||||||
|
|
@ -317,17 +316,17 @@ def report(
|
||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from entropix.core.config import create_default_config
|
from flakestorm.core.config import create_default_config
|
||||||
from entropix.mutations.types import Mutation
|
from flakestorm.mutations.types import Mutation
|
||||||
from entropix.reports.html import HTMLReportGenerator
|
from flakestorm.reports.html import HTMLReportGenerator
|
||||||
from entropix.reports.models import (
|
from flakestorm.reports.models import (
|
||||||
CheckResult,
|
CheckResult,
|
||||||
MutationResult,
|
MutationResult,
|
||||||
TestResults,
|
TestResults,
|
||||||
TestStatistics,
|
TestStatistics,
|
||||||
TypeStatistics,
|
TypeStatistics,
|
||||||
)
|
)
|
||||||
from entropix.reports.terminal import TerminalReporter
|
from flakestorm.reports.terminal import TerminalReporter
|
||||||
|
|
||||||
if not path.exists():
|
if not path.exists():
|
||||||
console.print(f"[red]File not found:[/red] {path}")
|
console.print(f"[red]File not found:[/red] {path}")
|
||||||
|
|
@ -399,7 +398,7 @@ def report(
|
||||||
@app.command()
|
@app.command()
|
||||||
def score(
|
def score(
|
||||||
config: Path = typer.Option(
|
config: Path = typer.Option(
|
||||||
Path("entropix.yaml"),
|
Path("flakestorm.yaml"),
|
||||||
"--config",
|
"--config",
|
||||||
"-c",
|
"-c",
|
||||||
help="Path to configuration file",
|
help="Path to configuration file",
|
||||||
|
|
@ -416,9 +415,9 @@ def score(
|
||||||
@app.command()
|
@app.command()
|
||||||
def cloud() -> None:
|
def cloud() -> None:
|
||||||
"""
|
"""
|
||||||
Learn about Entropix Cloud features.
|
Learn about flakestorm Cloud features.
|
||||||
|
|
||||||
Entropix Cloud provides 20x faster execution, advanced features,
|
flakestorm Cloud provides 20x faster execution, advanced features,
|
||||||
and team collaboration.
|
and team collaboration.
|
||||||
"""
|
"""
|
||||||
print_upgrade_banner(console, reason="20x faster tests")
|
print_upgrade_banner(console, reason="20x faster tests")
|
||||||
|
|
@ -464,7 +463,7 @@ def limits() -> None:
|
||||||
Show Open Source edition limits.
|
Show Open Source edition limits.
|
||||||
|
|
||||||
Displays the feature limitations of the Open Source edition
|
Displays the feature limitations of the Open Source edition
|
||||||
and how to unlock more with Entropix Cloud.
|
and how to unlock more with flakestorm Cloud.
|
||||||
"""
|
"""
|
||||||
console.print(
|
console.print(
|
||||||
Panel(
|
Panel(
|
||||||
|
|
@ -485,7 +484,7 @@ def limits() -> None:
|
||||||
"• Individual developers\n\n"
|
"• Individual developers\n\n"
|
||||||
f"[bold]Upgrade for production:[/bold] {CLOUD_URL}"
|
f"[bold]Upgrade for production:[/bold] {CLOUD_URL}"
|
||||||
),
|
),
|
||||||
title="[bold blue]Entropix Open Source[/bold blue]",
|
title="[bold blue]flakestorm Open Source[/bold blue]",
|
||||||
border_style="blue",
|
border_style="blue",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
@ -493,10 +492,9 @@ def limits() -> None:
|
||||||
|
|
||||||
async def _score_async(config: Path) -> None:
|
async def _score_async(config: Path) -> None:
|
||||||
"""Async implementation of score command."""
|
"""Async implementation of score command."""
|
||||||
from entropix.core.runner import EntropixRunner
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
runner = EntropixRunner(
|
runner = FlakeStormRunner(
|
||||||
config=config,
|
config=config,
|
||||||
console=console,
|
console=console,
|
||||||
show_progress=False,
|
show_progress=False,
|
||||||
|
|
@ -1,30 +1,30 @@
|
||||||
"""
|
"""
|
||||||
Entropix Core Module
|
flakestorm Core Module
|
||||||
|
|
||||||
Contains the main orchestration logic, configuration management,
|
Contains the main orchestration logic, configuration management,
|
||||||
agent protocol definitions, and the async test runner.
|
agent protocol definitions, and the async test runner.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
InvariantConfig,
|
InvariantConfig,
|
||||||
ModelConfig,
|
ModelConfig,
|
||||||
MutationConfig,
|
MutationConfig,
|
||||||
OutputConfig,
|
OutputConfig,
|
||||||
load_config,
|
load_config,
|
||||||
)
|
)
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
from entropix.core.protocol import (
|
from flakestorm.core.protocol import (
|
||||||
AgentProtocol,
|
AgentProtocol,
|
||||||
HTTPAgentAdapter,
|
HTTPAgentAdapter,
|
||||||
PythonAgentAdapter,
|
PythonAgentAdapter,
|
||||||
create_agent_adapter,
|
create_agent_adapter,
|
||||||
)
|
)
|
||||||
from entropix.core.runner import EntropixRunner
|
from flakestorm.core.runner import FlakeStormRunner
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"EntropixConfig",
|
"FlakeStormConfig",
|
||||||
"load_config",
|
"load_config",
|
||||||
"AgentConfig",
|
"AgentConfig",
|
||||||
"ModelConfig",
|
"ModelConfig",
|
||||||
|
|
@ -35,6 +35,6 @@ __all__ = [
|
||||||
"HTTPAgentAdapter",
|
"HTTPAgentAdapter",
|
||||||
"PythonAgentAdapter",
|
"PythonAgentAdapter",
|
||||||
"create_agent_adapter",
|
"create_agent_adapter",
|
||||||
"EntropixRunner",
|
"FlakeStormRunner",
|
||||||
"Orchestrator",
|
"Orchestrator",
|
||||||
]
|
]
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
"""
|
"""
|
||||||
Configuration Management for Entropix
|
Configuration Management for flakestorm
|
||||||
|
|
||||||
Handles loading and validating the entropix.yaml configuration file.
|
Handles loading and validating the flakestorm.yaml configuration file.
|
||||||
Uses Pydantic for robust validation and type safety.
|
Uses Pydantic for robust validation and type safety.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -15,7 +15,7 @@ import yaml
|
||||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||||
|
|
||||||
# Import MutationType from mutations to avoid duplicate definition
|
# Import MutationType from mutations to avoid duplicate definition
|
||||||
from entropix.mutations.types import MutationType
|
from flakestorm.mutations.types import MutationType
|
||||||
|
|
||||||
|
|
||||||
class AgentType(str, Enum):
|
class AgentType(str, Enum):
|
||||||
|
|
@ -73,7 +73,7 @@ class MutationConfig(BaseModel):
|
||||||
- Maximum 50 total mutations per test run
|
- Maximum 50 total mutations per test run
|
||||||
- 5 mutation types: paraphrase, noise, tone_shift, prompt_injection, custom
|
- 5 mutation types: paraphrase, noise, tone_shift, prompt_injection, custom
|
||||||
|
|
||||||
Upgrade to Entropix Cloud for unlimited mutations and advanced types.
|
Upgrade to flakestorm Cloud for unlimited mutations and advanced types.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
count: int = Field(
|
count: int = Field(
|
||||||
|
|
@ -194,8 +194,8 @@ class AdvancedConfig(BaseModel):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class EntropixConfig(BaseModel):
|
class FlakeStormConfig(BaseModel):
|
||||||
"""Main configuration for Entropix."""
|
"""Main configuration for flakestorm."""
|
||||||
|
|
||||||
version: str = Field(default="1.0", description="Configuration version")
|
version: str = Field(default="1.0", description="Configuration version")
|
||||||
agent: AgentConfig = Field(..., description="Agent configuration")
|
agent: AgentConfig = Field(..., description="Agent configuration")
|
||||||
|
|
@ -219,7 +219,7 @@ class EntropixConfig(BaseModel):
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_yaml(cls, content: str) -> EntropixConfig:
|
def from_yaml(cls, content: str) -> FlakeStormConfig:
|
||||||
"""Parse configuration from YAML string."""
|
"""Parse configuration from YAML string."""
|
||||||
data = yaml.safe_load(content)
|
data = yaml.safe_load(content)
|
||||||
return cls.model_validate(data)
|
return cls.model_validate(data)
|
||||||
|
|
@ -230,15 +230,15 @@ class EntropixConfig(BaseModel):
|
||||||
return yaml.dump(data, default_flow_style=False, sort_keys=False)
|
return yaml.dump(data, default_flow_style=False, sort_keys=False)
|
||||||
|
|
||||||
|
|
||||||
def load_config(path: str | Path) -> EntropixConfig:
|
def load_config(path: str | Path) -> FlakeStormConfig:
|
||||||
"""
|
"""
|
||||||
Load and validate an Entropix configuration file.
|
Load and validate an flakestorm configuration file.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
path: Path to the entropix.yaml file
|
path: Path to the flakestorm.yaml file
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Validated EntropixConfig object
|
Validated FlakeStormConfig object
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
FileNotFoundError: If the config file doesn't exist
|
FileNotFoundError: If the config file doesn't exist
|
||||||
|
|
@ -249,16 +249,16 @@ def load_config(path: str | Path) -> EntropixConfig:
|
||||||
if not config_path.exists():
|
if not config_path.exists():
|
||||||
raise FileNotFoundError(
|
raise FileNotFoundError(
|
||||||
f"Configuration file not found: {config_path}\n"
|
f"Configuration file not found: {config_path}\n"
|
||||||
"Run 'entropix init' to create a new configuration file."
|
"Run 'flakestorm init' to create a new configuration file."
|
||||||
)
|
)
|
||||||
|
|
||||||
content = config_path.read_text(encoding="utf-8")
|
content = config_path.read_text(encoding="utf-8")
|
||||||
return EntropixConfig.from_yaml(content)
|
return FlakeStormConfig.from_yaml(content)
|
||||||
|
|
||||||
|
|
||||||
def create_default_config() -> EntropixConfig:
|
def create_default_config() -> FlakeStormConfig:
|
||||||
"""Create a default configuration for initialization."""
|
"""Create a default configuration for initialization."""
|
||||||
return EntropixConfig(
|
return FlakeStormConfig(
|
||||||
version="1.0",
|
version="1.0",
|
||||||
agent=AgentConfig(
|
agent=AgentConfig(
|
||||||
endpoint="http://localhost:8000/invoke",
|
endpoint="http://localhost:8000/invoke",
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
Open Source Edition Limits
|
Open Source Edition Limits
|
||||||
|
|
||||||
Defines feature limits for the open source (local-only) version.
|
Defines feature limits for the open source (local-only) version.
|
||||||
These limits encourage users to upgrade to Entropix Cloud for:
|
These limits encourage users to upgrade to flakestorm Cloud for:
|
||||||
- Faster parallel execution
|
- Faster parallel execution
|
||||||
- Cloud LLMs (higher quality mutations)
|
- Cloud LLMs (higher quality mutations)
|
||||||
- Advanced features
|
- Advanced features
|
||||||
|
|
@ -65,8 +65,8 @@ ALLOWED_MUTATION_TYPES = [
|
||||||
# UPGRADE MESSAGING
|
# UPGRADE MESSAGING
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
CLOUD_URL = "https://entropix.cloud"
|
CLOUD_URL = "https://flakestorm.cloud"
|
||||||
UPGRADE_CTA = f"⚡ Upgrade to Entropix Cloud for 20x faster execution → {CLOUD_URL}"
|
UPGRADE_CTA = f"⚡ Upgrade to flakestorm Cloud for 20x faster execution → {CLOUD_URL}"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -138,7 +138,7 @@ def print_upgrade_banner(console: Console, reason: str = "faster execution") ->
|
||||||
banner = Panel(
|
banner = Panel(
|
||||||
Text.from_markup(
|
Text.from_markup(
|
||||||
f"[bold yellow]⚡ Want {reason}?[/bold yellow]\n\n"
|
f"[bold yellow]⚡ Want {reason}?[/bold yellow]\n\n"
|
||||||
f"[white]Entropix Cloud offers:[/white]\n"
|
f"[white]flakestorm Cloud offers:[/white]\n"
|
||||||
f" • [green]20x faster[/green] parallel execution\n"
|
f" • [green]20x faster[/green] parallel execution\n"
|
||||||
f" • [green]Cloud LLMs[/green] for higher quality mutations\n"
|
f" • [green]Cloud LLMs[/green] for higher quality mutations\n"
|
||||||
f" • [green]Advanced safety checks[/green] (NER, ML-detection)\n"
|
f" • [green]Advanced safety checks[/green] (NER, ML-detection)\n"
|
||||||
|
|
@ -146,7 +146,7 @@ def print_upgrade_banner(console: Console, reason: str = "faster execution") ->
|
||||||
f" • [green]Team features[/green] for collaboration\n\n"
|
f" • [green]Team features[/green] for collaboration\n\n"
|
||||||
f"[bold cyan]→ {CLOUD_URL}[/bold cyan]"
|
f"[bold cyan]→ {CLOUD_URL}[/bold cyan]"
|
||||||
),
|
),
|
||||||
title="[bold blue]Upgrade to Entropix Cloud[/bold blue]",
|
title="[bold blue]Upgrade to flakestorm Cloud[/bold blue]",
|
||||||
border_style="blue",
|
border_style="blue",
|
||||||
padding=(1, 2),
|
padding=(1, 2),
|
||||||
)
|
)
|
||||||
|
|
@ -184,7 +184,7 @@ def print_completion_upsell(console: Console, duration_seconds: float) -> None:
|
||||||
) # ~20x faster with parallel + cloud
|
) # ~20x faster with parallel + cloud
|
||||||
console.print(
|
console.print(
|
||||||
f"\n[dim]⏱️ Test took {duration_seconds:.1f}s. "
|
f"\n[dim]⏱️ Test took {duration_seconds:.1f}s. "
|
||||||
f"With Entropix Cloud, this would take ~{estimated_cloud_time:.1f}s[/dim]"
|
f"With flakestorm Cloud, this would take ~{estimated_cloud_time:.1f}s[/dim]"
|
||||||
)
|
)
|
||||||
console.print(f"[dim cyan]→ {CLOUD_URL}[/dim cyan]\n")
|
console.print(f"[dim cyan]→ {CLOUD_URL}[/dim cyan]\n")
|
||||||
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
"""
|
"""
|
||||||
Orchestrator for Entropix Test Runs
|
Orchestrator for flakestorm Test Runs
|
||||||
|
|
||||||
Coordinates the entire testing process: mutation generation,
|
Coordinates the entire testing process: mutation generation,
|
||||||
agent invocation, invariant verification, and result aggregation.
|
agent invocation, invariant verification, and result aggregation.
|
||||||
|
|
@ -9,7 +9,7 @@ Open Source Edition:
|
||||||
- Maximum 50 mutations per test run
|
- Maximum 50 mutations per test run
|
||||||
- Basic mutation types only
|
- Basic mutation types only
|
||||||
|
|
||||||
Upgrade to Entropix Cloud for parallel execution and advanced features.
|
Upgrade to flakestorm Cloud for parallel execution and advanced features.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -29,7 +29,7 @@ from rich.progress import (
|
||||||
TimeRemainingColumn,
|
TimeRemainingColumn,
|
||||||
)
|
)
|
||||||
|
|
||||||
from entropix.core.limits import (
|
from flakestorm.core.limits import (
|
||||||
MAX_MUTATIONS_PER_RUN,
|
MAX_MUTATIONS_PER_RUN,
|
||||||
PARALLEL_EXECUTION_ENABLED,
|
PARALLEL_EXECUTION_ENABLED,
|
||||||
check_mutation_limit,
|
check_mutation_limit,
|
||||||
|
|
@ -39,12 +39,12 @@ from entropix.core.limits import (
|
||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from entropix.assertions.verifier import InvariantVerifier
|
from flakestorm.assertions.verifier import InvariantVerifier
|
||||||
from entropix.core.config import EntropixConfig
|
from flakestorm.core.config import FlakeStormConfig
|
||||||
from entropix.core.protocol import BaseAgentAdapter
|
from flakestorm.core.protocol import BaseAgentAdapter
|
||||||
from entropix.mutations.engine import MutationEngine
|
from flakestorm.mutations.engine import MutationEngine
|
||||||
from entropix.mutations.types import Mutation
|
from flakestorm.mutations.types import Mutation
|
||||||
from entropix.reports.models import MutationResult, TestResults, TestStatistics
|
from flakestorm.reports.models import MutationResult, TestResults, TestStatistics
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -75,7 +75,7 @@ class OrchestratorState:
|
||||||
|
|
||||||
class Orchestrator:
|
class Orchestrator:
|
||||||
"""
|
"""
|
||||||
Orchestrates the entire Entropix test run.
|
Orchestrates the entire flakestorm test run.
|
||||||
|
|
||||||
Coordinates between:
|
Coordinates between:
|
||||||
- MutationEngine: Generates adversarial inputs
|
- MutationEngine: Generates adversarial inputs
|
||||||
|
|
@ -86,7 +86,7 @@ class Orchestrator:
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
config: EntropixConfig,
|
config: FlakeStormConfig,
|
||||||
agent: BaseAgentAdapter,
|
agent: BaseAgentAdapter,
|
||||||
mutation_engine: MutationEngine,
|
mutation_engine: MutationEngine,
|
||||||
verifier: InvariantVerifier,
|
verifier: InvariantVerifier,
|
||||||
|
|
@ -97,7 +97,7 @@ class Orchestrator:
|
||||||
Initialize the orchestrator.
|
Initialize the orchestrator.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Entropix configuration
|
config: flakestorm configuration
|
||||||
agent: Agent adapter to test
|
agent: Agent adapter to test
|
||||||
mutation_engine: Engine for generating mutations
|
mutation_engine: Engine for generating mutations
|
||||||
verifier: Invariant verification engine
|
verifier: Invariant verification engine
|
||||||
|
|
@ -121,7 +121,7 @@ class Orchestrator:
|
||||||
Returns:
|
Returns:
|
||||||
TestResults containing all test outcomes
|
TestResults containing all test outcomes
|
||||||
"""
|
"""
|
||||||
from entropix.reports.models import (
|
from flakestorm.reports.models import (
|
||||||
TestResults,
|
TestResults,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -310,7 +310,7 @@ class Orchestrator:
|
||||||
semaphore: asyncio.Semaphore,
|
semaphore: asyncio.Semaphore,
|
||||||
) -> MutationResult:
|
) -> MutationResult:
|
||||||
"""Run a single mutation against the agent."""
|
"""Run a single mutation against the agent."""
|
||||||
from entropix.reports.models import CheckResult, MutationResult
|
from flakestorm.reports.models import CheckResult, MutationResult
|
||||||
|
|
||||||
async with semaphore:
|
async with semaphore:
|
||||||
# Invoke agent
|
# Invoke agent
|
||||||
|
|
@ -363,7 +363,7 @@ class Orchestrator:
|
||||||
results: list[MutationResult],
|
results: list[MutationResult],
|
||||||
) -> TestStatistics:
|
) -> TestStatistics:
|
||||||
"""Calculate test statistics from results."""
|
"""Calculate test statistics from results."""
|
||||||
from entropix.reports.models import TestStatistics, TypeStatistics
|
from flakestorm.reports.models import TestStatistics, TypeStatistics
|
||||||
|
|
||||||
total = len(results)
|
total = len(results)
|
||||||
passed = sum(1 for r in results if r.passed)
|
passed = sum(1 for r in results if r.passed)
|
||||||
|
|
@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
|
||||||
# Try to import Rust bindings
|
# Try to import Rust bindings
|
||||||
_RUST_AVAILABLE = False
|
_RUST_AVAILABLE = False
|
||||||
try:
|
try:
|
||||||
import entropix_rust
|
import flakestorm_rust
|
||||||
|
|
||||||
_RUST_AVAILABLE = True
|
_RUST_AVAILABLE = True
|
||||||
logger.debug("Rust performance module loaded successfully")
|
logger.debug("Rust performance module loaded successfully")
|
||||||
|
|
@ -56,7 +56,7 @@ def calculate_robustness_score(
|
||||||
Robustness score between 0.0 and 1.0
|
Robustness score between 0.0 and 1.0
|
||||||
"""
|
"""
|
||||||
if _RUST_AVAILABLE:
|
if _RUST_AVAILABLE:
|
||||||
return entropix_rust.calculate_robustness_score(
|
return flakestorm_rust.calculate_robustness_score(
|
||||||
semantic_passed,
|
semantic_passed,
|
||||||
deterministic_passed,
|
deterministic_passed,
|
||||||
total,
|
total,
|
||||||
|
|
@ -88,7 +88,7 @@ def calculate_weighted_score(results: Sequence[tuple[bool, float]]) -> float:
|
||||||
Weighted robustness score between 0.0 and 1.0
|
Weighted robustness score between 0.0 and 1.0
|
||||||
"""
|
"""
|
||||||
if _RUST_AVAILABLE:
|
if _RUST_AVAILABLE:
|
||||||
return entropix_rust.calculate_weighted_score(list(results))
|
return flakestorm_rust.calculate_weighted_score(list(results))
|
||||||
|
|
||||||
# Pure Python fallback
|
# Pure Python fallback
|
||||||
if not results:
|
if not results:
|
||||||
|
|
@ -115,7 +115,7 @@ def levenshtein_distance(s1: str, s2: str) -> int:
|
||||||
Edit distance between the strings
|
Edit distance between the strings
|
||||||
"""
|
"""
|
||||||
if _RUST_AVAILABLE:
|
if _RUST_AVAILABLE:
|
||||||
return entropix_rust.levenshtein_distance(s1, s2)
|
return flakestorm_rust.levenshtein_distance(s1, s2)
|
||||||
|
|
||||||
# Pure Python fallback
|
# Pure Python fallback
|
||||||
len1 = len(s1)
|
len1 = len(s1)
|
||||||
|
|
@ -156,7 +156,7 @@ def string_similarity(s1: str, s2: str) -> float:
|
||||||
Similarity score between 0.0 (completely different) and 1.0 (identical)
|
Similarity score between 0.0 (completely different) and 1.0 (identical)
|
||||||
"""
|
"""
|
||||||
if _RUST_AVAILABLE:
|
if _RUST_AVAILABLE:
|
||||||
return entropix_rust.string_similarity(s1, s2)
|
return flakestorm_rust.string_similarity(s1, s2)
|
||||||
|
|
||||||
# Pure Python fallback
|
# Pure Python fallback
|
||||||
distance = levenshtein_distance(s1, s2)
|
distance = levenshtein_distance(s1, s2)
|
||||||
|
|
@ -187,7 +187,7 @@ def parallel_process_mutations(
|
||||||
List of (mutation, type, weight) tuples
|
List of (mutation, type, weight) tuples
|
||||||
"""
|
"""
|
||||||
if _RUST_AVAILABLE:
|
if _RUST_AVAILABLE:
|
||||||
return entropix_rust.parallel_process_mutations(
|
return flakestorm_rust.parallel_process_mutations(
|
||||||
mutations, mutation_types, weights
|
mutations, mutation_types, weights
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -353,7 +353,7 @@ def benchmark_levenshtein(iterations: int = 1000) -> dict:
|
||||||
start = time.perf_counter()
|
start = time.perf_counter()
|
||||||
for _ in range(iterations):
|
for _ in range(iterations):
|
||||||
for s1, s2 in test_pairs:
|
for s1, s2 in test_pairs:
|
||||||
entropix_rust.levenshtein_distance(s1, s2)
|
flakestorm_rust.levenshtein_distance(s1, s2)
|
||||||
rust_time = time.perf_counter() - start
|
rust_time = time.perf_counter() - start
|
||||||
result["rust_time_ms"] = rust_time * 1000
|
result["rust_time_ms"] = rust_time * 1000
|
||||||
result["speedup"] = python_time / rust_time if rust_time > 0 else 0
|
result["speedup"] = python_time / rust_time if rust_time > 0 else 0
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
"""
|
"""
|
||||||
Agent Protocol and Adapters for Entropix
|
Agent Protocol and Adapters for flakestorm
|
||||||
|
|
||||||
Defines the interface that all agents must implement and provides
|
Defines the interface that all agents must implement and provides
|
||||||
built-in adapters for common agent types (HTTP, Python callable, LangChain).
|
built-in adapters for common agent types (HTTP, Python callable, LangChain).
|
||||||
|
|
@ -17,7 +17,7 @@ from typing import Any, Protocol, runtime_checkable
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from entropix.core.config import AgentConfig, AgentType
|
from flakestorm.core.config import AgentConfig, AgentType
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -40,7 +40,7 @@ class AgentProtocol(Protocol):
|
||||||
"""
|
"""
|
||||||
Protocol defining the interface for AI agents.
|
Protocol defining the interface for AI agents.
|
||||||
|
|
||||||
All agents must implement this interface to be tested with Entropix.
|
All agents must implement this interface to be tested with flakestorm.
|
||||||
The simplest implementation is an async function that takes a string
|
The simplest implementation is an async function that takes a string
|
||||||
input and returns a string output.
|
input and returns a string output.
|
||||||
"""
|
"""
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
"""
|
"""
|
||||||
Entropix Test Runner
|
flakestorm Test Runner
|
||||||
|
|
||||||
High-level interface for running Entropix tests. Combines all components
|
High-level interface for running flakestorm tests. Combines all components
|
||||||
and provides a simple API for executing reliability tests.
|
and provides a simple API for executing reliability tests.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -12,34 +12,34 @@ from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
|
|
||||||
from entropix.assertions.verifier import InvariantVerifier
|
from flakestorm.assertions.verifier import InvariantVerifier
|
||||||
from entropix.core.config import EntropixConfig, load_config
|
from flakestorm.core.config import FlakeStormConfig, load_config
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
from entropix.core.protocol import BaseAgentAdapter, create_agent_adapter
|
from flakestorm.core.protocol import BaseAgentAdapter, create_agent_adapter
|
||||||
from entropix.mutations.engine import MutationEngine
|
from flakestorm.mutations.engine import MutationEngine
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from entropix.reports.models import TestResults
|
from flakestorm.reports.models import TestResults
|
||||||
|
|
||||||
|
|
||||||
class EntropixRunner:
|
class FlakeStormRunner:
|
||||||
"""
|
"""
|
||||||
Main runner for Entropix tests.
|
Main runner for flakestorm tests.
|
||||||
|
|
||||||
Provides a high-level interface for running reliability tests
|
Provides a high-level interface for running reliability tests
|
||||||
against AI agents. Handles configuration loading, component
|
against AI agents. Handles configuration loading, component
|
||||||
initialization, and test execution.
|
initialization, and test execution.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
>>> config = load_config("entropix.yaml")
|
>>> config = load_config("flakestorm.yaml")
|
||||||
>>> runner = EntropixRunner(config)
|
>>> runner = FlakeStormRunner(config)
|
||||||
>>> results = await runner.run()
|
>>> results = await runner.run()
|
||||||
>>> print(f"Score: {results.statistics.robustness_score:.1%}")
|
>>> print(f"Score: {results.statistics.robustness_score:.1%}")
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
config: EntropixConfig | str | Path,
|
config: FlakeStormConfig | str | Path,
|
||||||
agent: BaseAgentAdapter | None = None,
|
agent: BaseAgentAdapter | None = None,
|
||||||
console: Console | None = None,
|
console: Console | None = None,
|
||||||
show_progress: bool = True,
|
show_progress: bool = True,
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
"""
|
"""
|
||||||
Entropix Integrations Module
|
flakestorm Integrations Module
|
||||||
|
|
||||||
V2 features for integrating with external services:
|
V2 features for integrating with external services:
|
||||||
- HuggingFace model downloading
|
- HuggingFace model downloading
|
||||||
|
|
@ -19,15 +19,15 @@ __all__ = [
|
||||||
def __getattr__(name: str):
|
def __getattr__(name: str):
|
||||||
"""Lazy loading of integration modules."""
|
"""Lazy loading of integration modules."""
|
||||||
if name == "HuggingFaceModelProvider":
|
if name == "HuggingFaceModelProvider":
|
||||||
from entropix.integrations.huggingface import HuggingFaceModelProvider
|
from flakestorm.integrations.huggingface import HuggingFaceModelProvider
|
||||||
|
|
||||||
return HuggingFaceModelProvider
|
return HuggingFaceModelProvider
|
||||||
elif name == "GitHubActionsIntegration":
|
elif name == "GitHubActionsIntegration":
|
||||||
from entropix.integrations.github_actions import GitHubActionsIntegration
|
from flakestorm.integrations.github_actions import GitHubActionsIntegration
|
||||||
|
|
||||||
return GitHubActionsIntegration
|
return GitHubActionsIntegration
|
||||||
elif name == "LocalEmbedder":
|
elif name == "LocalEmbedder":
|
||||||
from entropix.assertions.semantic import LocalEmbedder
|
from flakestorm.assertions.semantic import LocalEmbedder
|
||||||
|
|
||||||
return LocalEmbedder
|
return LocalEmbedder
|
||||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
|
|
@ -8,6 +8,6 @@ Re-exports the LocalEmbedder from assertions.semantic for convenience.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
# Re-export from semantic module
|
# Re-export from semantic module
|
||||||
from entropix.assertions.semantic import LocalEmbedder
|
from flakestorm.assertions.semantic import LocalEmbedder
|
||||||
|
|
||||||
__all__ = ["LocalEmbedder"]
|
__all__ = ["LocalEmbedder"]
|
||||||
|
|
@ -1,23 +1,23 @@
|
||||||
"""
|
"""
|
||||||
GitHub Actions Integration
|
GitHub Actions Integration
|
||||||
|
|
||||||
⚠️ CLOUD FEATURE: GitHub Actions integration is available in Entropix Cloud.
|
⚠️ CLOUD FEATURE: GitHub Actions integration is available in flakestorm Cloud.
|
||||||
The Open Source edition provides documentation only.
|
The Open Source edition provides documentation only.
|
||||||
|
|
||||||
Upgrade to Entropix Cloud for:
|
Upgrade to flakestorm Cloud for:
|
||||||
- One-click CI/CD integration
|
- One-click CI/CD integration
|
||||||
- Block PRs based on reliability score
|
- Block PRs based on reliability score
|
||||||
- Automated test history tracking
|
- Automated test history tracking
|
||||||
- Team notifications
|
- Team notifications
|
||||||
|
|
||||||
→ https://entropix.cloud
|
→ https://flakestorm.cloud
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from entropix.core.limits import CLOUD_URL, GITHUB_ACTIONS_ENABLED
|
from flakestorm.core.limits import CLOUD_URL, GITHUB_ACTIONS_ENABLED
|
||||||
|
|
||||||
|
|
||||||
class GitHubActionsDisabledError(Exception):
|
class GitHubActionsDisabledError(Exception):
|
||||||
|
|
@ -25,18 +25,18 @@ class GitHubActionsDisabledError(Exception):
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
"GitHub Actions integration is available in Entropix Cloud.\n"
|
"GitHub Actions integration is available in flakestorm Cloud.\n"
|
||||||
f"Upgrade at: {CLOUD_URL}"
|
f"Upgrade at: {CLOUD_URL}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# GitHub Action YAML template (for reference/documentation)
|
# GitHub Action YAML template (for reference/documentation)
|
||||||
ACTION_YAML = """# ⚠️ CLOUD FEATURE: This requires Entropix Cloud
|
ACTION_YAML = """# ⚠️ CLOUD FEATURE: This requires flakestorm Cloud
|
||||||
# Upgrade at: https://entropix.cloud
|
# Upgrade at: https://flakestorm.cloud
|
||||||
|
|
||||||
name: 'Entropix Agent Test'
|
name: 'flakestorm Agent Test'
|
||||||
description: 'Run chaos testing on AI agents to verify reliability'
|
description: 'Run chaos testing on AI agents to verify reliability'
|
||||||
author: 'Entropix'
|
author: 'flakestorm'
|
||||||
|
|
||||||
branding:
|
branding:
|
||||||
icon: 'shield'
|
icon: 'shield'
|
||||||
|
|
@ -44,15 +44,15 @@ branding:
|
||||||
|
|
||||||
inputs:
|
inputs:
|
||||||
config:
|
config:
|
||||||
description: 'Path to entropix.yaml configuration file'
|
description: 'Path to flakestorm.yaml configuration file'
|
||||||
required: false
|
required: false
|
||||||
default: 'entropix.yaml'
|
default: 'flakestorm.yaml'
|
||||||
min_score:
|
min_score:
|
||||||
description: 'Minimum robustness score to pass (0.0-1.0)'
|
description: 'Minimum robustness score to pass (0.0-1.0)'
|
||||||
required: false
|
required: false
|
||||||
default: '0.9'
|
default: '0.9'
|
||||||
api_key:
|
api_key:
|
||||||
description: 'Entropix Cloud API key (required)'
|
description: 'flakestorm Cloud API key (required)'
|
||||||
required: true
|
required: true
|
||||||
|
|
||||||
outputs:
|
outputs:
|
||||||
|
|
@ -61,7 +61,7 @@ outputs:
|
||||||
passed:
|
passed:
|
||||||
description: 'Whether the test passed (true/false)'
|
description: 'Whether the test passed (true/false)'
|
||||||
report_url:
|
report_url:
|
||||||
description: 'URL to the full report on Entropix Cloud'
|
description: 'URL to the full report on flakestorm Cloud'
|
||||||
|
|
||||||
runs:
|
runs:
|
||||||
using: 'composite'
|
using: 'composite'
|
||||||
|
|
@ -71,16 +71,16 @@ runs:
|
||||||
with:
|
with:
|
||||||
python-version: '3.11'
|
python-version: '3.11'
|
||||||
|
|
||||||
- name: Install Entropix
|
- name: Install flakestorm
|
||||||
shell: bash
|
shell: bash
|
||||||
run: pip install entropix
|
run: pip install flakestorm
|
||||||
|
|
||||||
- name: Run Cloud Tests
|
- name: Run Cloud Tests
|
||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
ENTROPIX_API_KEY: ${{ inputs.api_key }}
|
FLAKESTORM_API_KEY: ${{ inputs.api_key }}
|
||||||
run: |
|
run: |
|
||||||
entropix cloud run \\
|
flakestorm cloud run \\
|
||||||
--config ${{ inputs.config }} \\
|
--config ${{ inputs.config }} \\
|
||||||
--min-score ${{ inputs.min_score }} \\
|
--min-score ${{ inputs.min_score }} \\
|
||||||
--ci
|
--ci
|
||||||
|
|
@ -88,9 +88,9 @@ runs:
|
||||||
|
|
||||||
|
|
||||||
# Example workflow YAML
|
# Example workflow YAML
|
||||||
WORKFLOW_EXAMPLE = """# Entropix Cloud CI/CD Integration
|
WORKFLOW_EXAMPLE = """# flakestorm Cloud CI/CD Integration
|
||||||
# ⚠️ Requires Entropix Cloud subscription
|
# ⚠️ Requires flakestorm Cloud subscription
|
||||||
# Get started: https://entropix.cloud
|
# Get started: https://flakestorm.cloud
|
||||||
|
|
||||||
name: Agent Reliability Check
|
name: Agent Reliability Check
|
||||||
|
|
||||||
|
|
@ -107,12 +107,12 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Run Entropix Cloud Tests
|
- name: Run flakestorm Cloud Tests
|
||||||
uses: entropix/entropix-action@v1
|
uses: flakestorm/flakestorm-action@v1
|
||||||
with:
|
with:
|
||||||
config: entropix.yaml
|
config: flakestorm.yaml
|
||||||
min_score: '0.9'
|
min_score: '0.9'
|
||||||
api_key: ${{ secrets.ENTROPIX_API_KEY }}
|
api_key: ${{ secrets.FLAKESTORM_API_KEY }}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -120,19 +120,19 @@ class GitHubActionsIntegration:
|
||||||
"""
|
"""
|
||||||
Helper class for GitHub Actions integration.
|
Helper class for GitHub Actions integration.
|
||||||
|
|
||||||
⚠️ NOTE: Full CI/CD integration requires Entropix Cloud.
|
⚠️ NOTE: Full CI/CD integration requires flakestorm Cloud.
|
||||||
|
|
||||||
The Open Source edition provides:
|
The Open Source edition provides:
|
||||||
- Documentation and examples
|
- Documentation and examples
|
||||||
- Local testing only
|
- Local testing only
|
||||||
|
|
||||||
Entropix Cloud provides:
|
flakestorm Cloud provides:
|
||||||
- One-click GitHub Actions setup
|
- One-click GitHub Actions setup
|
||||||
- Block PRs based on reliability score
|
- Block PRs based on reliability score
|
||||||
- Test history and comparison
|
- Test history and comparison
|
||||||
- Slack/Discord notifications
|
- Slack/Discord notifications
|
||||||
|
|
||||||
Upgrade at: https://entropix.cloud
|
Upgrade at: https://flakestorm.cloud
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
@ -147,7 +147,7 @@ class GitHubActionsIntegration:
|
||||||
Generate the GitHub Action definition YAML.
|
Generate the GitHub Action definition YAML.
|
||||||
|
|
||||||
Note: This returns documentation only in Open Source edition.
|
Note: This returns documentation only in Open Source edition.
|
||||||
Full integration requires Entropix Cloud.
|
Full integration requires flakestorm Cloud.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Action YAML content
|
Action YAML content
|
||||||
|
|
@ -157,9 +157,9 @@ class GitHubActionsIntegration:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def generate_workflow_example() -> str:
|
def generate_workflow_example() -> str:
|
||||||
"""
|
"""
|
||||||
Generate an example workflow that uses Entropix.
|
Generate an example workflow that uses flakestorm.
|
||||||
|
|
||||||
Note: Requires Entropix Cloud for full functionality.
|
Note: Requires flakestorm Cloud for full functionality.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Workflow YAML content
|
Workflow YAML content
|
||||||
|
|
@ -172,7 +172,7 @@ class GitHubActionsIntegration:
|
||||||
Save the GitHub Action files to a directory.
|
Save the GitHub Action files to a directory.
|
||||||
|
|
||||||
⚠️ Cloud Feature: This creates documentation only.
|
⚠️ Cloud Feature: This creates documentation only.
|
||||||
For working CI/CD, upgrade to Entropix Cloud.
|
For working CI/CD, upgrade to flakestorm Cloud.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
output_dir: Directory to save action files
|
output_dir: Directory to save action files
|
||||||
|
|
@ -189,9 +189,9 @@ class GitHubActionsIntegration:
|
||||||
# Also create a README explaining Cloud requirement
|
# Also create a README explaining Cloud requirement
|
||||||
readme_path = output_dir / "README.md"
|
readme_path = output_dir / "README.md"
|
||||||
readme_path.write_text(
|
readme_path.write_text(
|
||||||
f"""# Entropix GitHub Action
|
f"""# flakestorm GitHub Action
|
||||||
|
|
||||||
⚠️ **Cloud Feature**: Full CI/CD integration requires Entropix Cloud.
|
⚠️ **Cloud Feature**: Full CI/CD integration requires flakestorm Cloud.
|
||||||
|
|
||||||
## What You Get with Cloud
|
## What You Get with Cloud
|
||||||
|
|
||||||
|
|
@ -210,7 +210,7 @@ Get started at: {CLOUD_URL}
|
||||||
For local-only testing, use the Open Source CLI:
|
For local-only testing, use the Open Source CLI:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
entropix run --config entropix.yaml
|
flakestorm run --config flakestorm.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
Note: Local runs are sequential and may be slow for large test suites.
|
Note: Local runs are sequential and may be slow for large test suites.
|
||||||
|
|
@ -240,13 +240,13 @@ Note: Local runs are sequential and may be slow for large test suites.
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def setup_ci(
|
def setup_ci(
|
||||||
repo_path: Path,
|
repo_path: Path,
|
||||||
config_path: str = "entropix.yaml",
|
config_path: str = "flakestorm.yaml",
|
||||||
min_score: float = 0.9,
|
min_score: float = 0.9,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Set up CI/CD integration for a repository.
|
Set up CI/CD integration for a repository.
|
||||||
|
|
||||||
⚠️ Cloud Feature: Requires Entropix Cloud subscription.
|
⚠️ Cloud Feature: Requires flakestorm Cloud subscription.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
GitHubActionsDisabledError: Always in Open Source edition
|
GitHubActionsDisabledError: Always in Open Source edition
|
||||||
|
|
@ -51,10 +51,10 @@ class HuggingFaceModelProvider:
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
models_dir: Directory to store downloaded models
|
models_dir: Directory to store downloaded models
|
||||||
(default: ~/.entropix/models)
|
(default: ~/.flakestorm/models)
|
||||||
"""
|
"""
|
||||||
if models_dir is None:
|
if models_dir is None:
|
||||||
self.models_dir = Path.home() / ".entropix" / "models"
|
self.models_dir = Path.home() / ".flakestorm" / "models"
|
||||||
else:
|
else:
|
||||||
self.models_dir = Path(models_dir)
|
self.models_dir = Path(models_dir)
|
||||||
|
|
||||||
|
|
@ -82,7 +82,7 @@ class HuggingFaceModelProvider:
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError(
|
raise ImportError(
|
||||||
"huggingface-hub is required for model downloading. "
|
"huggingface-hub is required for model downloading. "
|
||||||
"Install with: pip install entropix[huggingface]"
|
"Install with: pip install flakestorm[huggingface]"
|
||||||
)
|
)
|
||||||
|
|
||||||
# If no filename specified, find appropriate GGUF file
|
# If no filename specified, find appropriate GGUF file
|
||||||
|
|
@ -112,7 +112,7 @@ class HuggingFaceModelProvider:
|
||||||
|
|
||||||
def list_available(self) -> list[dict]:
|
def list_available(self) -> list[dict]:
|
||||||
"""
|
"""
|
||||||
List recommended models for Entropix.
|
List recommended models for flakestorm.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of model info dictionaries
|
List of model info dictionaries
|
||||||
|
|
@ -236,11 +236,11 @@ SYSTEM You are a helpful assistant that generates text variations.
|
||||||
>>> provider = HuggingFaceModelProvider()
|
>>> provider = HuggingFaceModelProvider()
|
||||||
>>> name = provider.download_and_import(
|
>>> name = provider.download_and_import(
|
||||||
... "TheBloke/Mistral-7B-Instruct-v0.2-GGUF",
|
... "TheBloke/Mistral-7B-Instruct-v0.2-GGUF",
|
||||||
... model_name="entropix-attacker"
|
... model_name="flakestorm-attacker"
|
||||||
... )
|
... )
|
||||||
>>> # Now use in entropix.yaml:
|
>>> # Now use in flakestorm.yaml:
|
||||||
>>> # llm:
|
>>> # llm:
|
||||||
>>> # model: "entropix-attacker"
|
>>> # model: "flakestorm-attacker"
|
||||||
"""
|
"""
|
||||||
# Download the model
|
# Download the model
|
||||||
model_path = self.download_model(
|
model_path = self.download_model(
|
||||||
|
|
@ -1,13 +1,13 @@
|
||||||
"""
|
"""
|
||||||
Entropix Mutation Engine
|
flakestorm Mutation Engine
|
||||||
|
|
||||||
Generates adversarial mutations from golden prompts using local LLMs.
|
Generates adversarial mutations from golden prompts using local LLMs.
|
||||||
Supports paraphrasing, noise injection, tone shifting, and prompt injection.
|
Supports paraphrasing, noise injection, tone shifting, and prompt injection.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from entropix.mutations.engine import MutationEngine
|
from flakestorm.mutations.engine import MutationEngine
|
||||||
from entropix.mutations.templates import MUTATION_TEMPLATES, MutationTemplates
|
from flakestorm.mutations.templates import MUTATION_TEMPLATES, MutationTemplates
|
||||||
from entropix.mutations.types import Mutation, MutationType
|
from flakestorm.mutations.types import Mutation, MutationType
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"MutationEngine",
|
"MutationEngine",
|
||||||
|
|
@ -13,11 +13,11 @@ from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from ollama import AsyncClient
|
from ollama import AsyncClient
|
||||||
|
|
||||||
from entropix.mutations.templates import MutationTemplates
|
from flakestorm.mutations.templates import MutationTemplates
|
||||||
from entropix.mutations.types import Mutation, MutationType
|
from flakestorm.mutations.types import Mutation, MutationType
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from entropix.core.config import ModelConfig
|
from flakestorm.core.config import ModelConfig
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -7,7 +7,7 @@ different types of adversarial mutations.
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from entropix.mutations.types import MutationType
|
from flakestorm.mutations.types import MutationType
|
||||||
|
|
||||||
# Prompt templates for each mutation type
|
# Prompt templates for each mutation type
|
||||||
MUTATION_TEMPLATES: dict[MutationType, str] = {
|
MUTATION_TEMPLATES: dict[MutationType, str] = {
|
||||||
|
|
@ -24,7 +24,7 @@ class MutationType(str, Enum):
|
||||||
- CUSTOM: User-defined mutation templates
|
- CUSTOM: User-defined mutation templates
|
||||||
|
|
||||||
Advanced mutations (sophisticated prompt injections, jailbreaks)
|
Advanced mutations (sophisticated prompt injections, jailbreaks)
|
||||||
are available in Entropix Cloud.
|
are available in flakestorm Cloud.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
PARAPHRASE = "paraphrase"
|
PARAPHRASE = "paraphrase"
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
"""
|
"""
|
||||||
Entropix Test Suite
|
flakestorm Test Suite
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
"""Shared test fixtures for Entropix tests."""
|
"""Shared test fixtures for flakestorm tests."""
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
@ -45,7 +45,7 @@ invariants:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def config_file(temp_dir, sample_config_yaml):
|
def config_file(temp_dir, sample_config_yaml):
|
||||||
"""Create a config file in temp directory."""
|
"""Create a config file in temp directory."""
|
||||||
config_path = temp_dir / "entropix.yaml"
|
config_path = temp_dir / "flakestorm.yaml"
|
||||||
config_path.write_text(sample_config_yaml)
|
config_path.write_text(sample_config_yaml)
|
||||||
return config_path
|
return config_path
|
||||||
|
|
||||||
|
|
@ -73,6 +73,6 @@ invariants: []
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def minimal_config_file(temp_dir, minimal_config_yaml):
|
def minimal_config_file(temp_dir, minimal_config_yaml):
|
||||||
"""Create a minimal config file."""
|
"""Create a minimal config file."""
|
||||||
config_path = temp_dir / "entropix.yaml"
|
config_path = temp_dir / "flakestorm.yaml"
|
||||||
config_path.write_text(minimal_config_yaml)
|
config_path.write_text(minimal_config_yaml)
|
||||||
return config_path
|
return config_path
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ class TestHTTPAgentAdapter:
|
||||||
|
|
||||||
def test_adapter_creation(self):
|
def test_adapter_creation(self):
|
||||||
"""Test adapter can be created."""
|
"""Test adapter can be created."""
|
||||||
from entropix.core.protocol import HTTPAgentAdapter
|
from flakestorm.core.protocol import HTTPAgentAdapter
|
||||||
|
|
||||||
adapter = HTTPAgentAdapter(
|
adapter = HTTPAgentAdapter(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
|
|
@ -19,7 +19,7 @@ class TestHTTPAgentAdapter:
|
||||||
|
|
||||||
def test_adapter_has_invoke_method(self):
|
def test_adapter_has_invoke_method(self):
|
||||||
"""Adapter has invoke method."""
|
"""Adapter has invoke method."""
|
||||||
from entropix.core.protocol import HTTPAgentAdapter
|
from flakestorm.core.protocol import HTTPAgentAdapter
|
||||||
|
|
||||||
adapter = HTTPAgentAdapter(endpoint="http://localhost:8000/chat")
|
adapter = HTTPAgentAdapter(endpoint="http://localhost:8000/chat")
|
||||||
assert hasattr(adapter, "invoke")
|
assert hasattr(adapter, "invoke")
|
||||||
|
|
@ -27,7 +27,7 @@ class TestHTTPAgentAdapter:
|
||||||
|
|
||||||
def test_timeout_conversion(self):
|
def test_timeout_conversion(self):
|
||||||
"""Timeout is converted to seconds."""
|
"""Timeout is converted to seconds."""
|
||||||
from entropix.core.protocol import HTTPAgentAdapter
|
from flakestorm.core.protocol import HTTPAgentAdapter
|
||||||
|
|
||||||
adapter = HTTPAgentAdapter(
|
adapter = HTTPAgentAdapter(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
|
|
@ -38,7 +38,7 @@ class TestHTTPAgentAdapter:
|
||||||
|
|
||||||
def test_custom_headers(self):
|
def test_custom_headers(self):
|
||||||
"""Custom headers can be set."""
|
"""Custom headers can be set."""
|
||||||
from entropix.core.protocol import HTTPAgentAdapter
|
from flakestorm.core.protocol import HTTPAgentAdapter
|
||||||
|
|
||||||
headers = {"Authorization": "Bearer token123"}
|
headers = {"Authorization": "Bearer token123"}
|
||||||
adapter = HTTPAgentAdapter(
|
adapter = HTTPAgentAdapter(
|
||||||
|
|
@ -53,7 +53,7 @@ class TestPythonAgentAdapter:
|
||||||
|
|
||||||
def test_adapter_creation_with_callable(self):
|
def test_adapter_creation_with_callable(self):
|
||||||
"""Test adapter can be created with a callable."""
|
"""Test adapter can be created with a callable."""
|
||||||
from entropix.core.protocol import PythonAgentAdapter
|
from flakestorm.core.protocol import PythonAgentAdapter
|
||||||
|
|
||||||
def my_agent(input: str) -> str:
|
def my_agent(input: str) -> str:
|
||||||
return f"Response to: {input}"
|
return f"Response to: {input}"
|
||||||
|
|
@ -64,7 +64,7 @@ class TestPythonAgentAdapter:
|
||||||
|
|
||||||
def test_adapter_has_invoke_method(self):
|
def test_adapter_has_invoke_method(self):
|
||||||
"""Adapter has invoke method."""
|
"""Adapter has invoke method."""
|
||||||
from entropix.core.protocol import PythonAgentAdapter
|
from flakestorm.core.protocol import PythonAgentAdapter
|
||||||
|
|
||||||
def my_agent(input: str) -> str:
|
def my_agent(input: str) -> str:
|
||||||
return f"Response to: {input}"
|
return f"Response to: {input}"
|
||||||
|
|
@ -80,7 +80,7 @@ class TestLangChainAgentAdapter:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def langchain_config(self):
|
def langchain_config(self):
|
||||||
"""Create a test LangChain agent config."""
|
"""Create a test LangChain agent config."""
|
||||||
from entropix.core.config import AgentConfig, AgentType
|
from flakestorm.core.config import AgentConfig, AgentType
|
||||||
|
|
||||||
return AgentConfig(
|
return AgentConfig(
|
||||||
endpoint="my_agent:chain",
|
endpoint="my_agent:chain",
|
||||||
|
|
@ -90,7 +90,7 @@ class TestLangChainAgentAdapter:
|
||||||
|
|
||||||
def test_adapter_creation(self, langchain_config):
|
def test_adapter_creation(self, langchain_config):
|
||||||
"""Test adapter can be created."""
|
"""Test adapter can be created."""
|
||||||
from entropix.core.protocol import LangChainAgentAdapter
|
from flakestorm.core.protocol import LangChainAgentAdapter
|
||||||
|
|
||||||
adapter = LangChainAgentAdapter(langchain_config)
|
adapter = LangChainAgentAdapter(langchain_config)
|
||||||
assert adapter is not None
|
assert adapter is not None
|
||||||
|
|
@ -101,8 +101,8 @@ class TestAgentAdapterFactory:
|
||||||
|
|
||||||
def test_creates_http_adapter(self):
|
def test_creates_http_adapter(self):
|
||||||
"""Factory creates HTTP adapter for HTTP type."""
|
"""Factory creates HTTP adapter for HTTP type."""
|
||||||
from entropix.core.config import AgentConfig, AgentType
|
from flakestorm.core.config import AgentConfig, AgentType
|
||||||
from entropix.core.protocol import HTTPAgentAdapter, create_agent_adapter
|
from flakestorm.core.protocol import HTTPAgentAdapter, create_agent_adapter
|
||||||
|
|
||||||
config = AgentConfig(
|
config = AgentConfig(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
|
|
@ -113,7 +113,7 @@ class TestAgentAdapterFactory:
|
||||||
|
|
||||||
def test_creates_python_adapter(self):
|
def test_creates_python_adapter(self):
|
||||||
"""Python adapter can be created with a callable."""
|
"""Python adapter can be created with a callable."""
|
||||||
from entropix.core.protocol import PythonAgentAdapter
|
from flakestorm.core.protocol import PythonAgentAdapter
|
||||||
|
|
||||||
def my_agent(input: str) -> str:
|
def my_agent(input: str) -> str:
|
||||||
return f"Response: {input}"
|
return f"Response: {input}"
|
||||||
|
|
@ -123,8 +123,8 @@ class TestAgentAdapterFactory:
|
||||||
|
|
||||||
def test_creates_langchain_adapter(self):
|
def test_creates_langchain_adapter(self):
|
||||||
"""Factory creates LangChain adapter for LangChain type."""
|
"""Factory creates LangChain adapter for LangChain type."""
|
||||||
from entropix.core.config import AgentConfig, AgentType
|
from flakestorm.core.config import AgentConfig, AgentType
|
||||||
from entropix.core.protocol import LangChainAgentAdapter, create_agent_adapter
|
from flakestorm.core.protocol import LangChainAgentAdapter, create_agent_adapter
|
||||||
|
|
||||||
config = AgentConfig(
|
config = AgentConfig(
|
||||||
endpoint="my_agent:chain",
|
endpoint="my_agent:chain",
|
||||||
|
|
@ -139,7 +139,7 @@ class TestAgentResponse:
|
||||||
|
|
||||||
def test_response_creation(self):
|
def test_response_creation(self):
|
||||||
"""Test AgentResponse can be created."""
|
"""Test AgentResponse can be created."""
|
||||||
from entropix.core.protocol import AgentResponse
|
from flakestorm.core.protocol import AgentResponse
|
||||||
|
|
||||||
response = AgentResponse(
|
response = AgentResponse(
|
||||||
output="Hello, world!",
|
output="Hello, world!",
|
||||||
|
|
@ -150,7 +150,7 @@ class TestAgentResponse:
|
||||||
|
|
||||||
def test_response_with_error(self):
|
def test_response_with_error(self):
|
||||||
"""Test AgentResponse with error."""
|
"""Test AgentResponse with error."""
|
||||||
from entropix.core.protocol import AgentResponse
|
from flakestorm.core.protocol import AgentResponse
|
||||||
|
|
||||||
response = AgentResponse(
|
response = AgentResponse(
|
||||||
output="",
|
output="",
|
||||||
|
|
@ -162,7 +162,7 @@ class TestAgentResponse:
|
||||||
|
|
||||||
def test_response_success_property(self):
|
def test_response_success_property(self):
|
||||||
"""Test AgentResponse success property."""
|
"""Test AgentResponse success property."""
|
||||||
from entropix.core.protocol import AgentResponse
|
from flakestorm.core.protocol import AgentResponse
|
||||||
|
|
||||||
# Success case
|
# Success case
|
||||||
success_response = AgentResponse(
|
success_response = AgentResponse(
|
||||||
|
|
|
||||||
|
|
@ -2,15 +2,15 @@
|
||||||
Tests for the assertion/invariant system.
|
Tests for the assertion/invariant system.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from entropix.assertions.deterministic import (
|
from flakestorm.assertions.deterministic import (
|
||||||
ContainsChecker,
|
ContainsChecker,
|
||||||
LatencyChecker,
|
LatencyChecker,
|
||||||
RegexChecker,
|
RegexChecker,
|
||||||
ValidJsonChecker,
|
ValidJsonChecker,
|
||||||
)
|
)
|
||||||
from entropix.assertions.safety import ExcludesPIIChecker, RefusalChecker
|
from flakestorm.assertions.safety import ExcludesPIIChecker, RefusalChecker
|
||||||
from entropix.assertions.verifier import InvariantVerifier
|
from flakestorm.assertions.verifier import InvariantVerifier
|
||||||
from entropix.core.config import InvariantConfig, InvariantType
|
from flakestorm.core.config import InvariantConfig, InvariantType
|
||||||
|
|
||||||
|
|
||||||
class TestContainsChecker:
|
class TestContainsChecker:
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ from pathlib import Path
|
||||||
|
|
||||||
from typer.testing import CliRunner
|
from typer.testing import CliRunner
|
||||||
|
|
||||||
from entropix.cli.main import app
|
from flakestorm.cli.main import app
|
||||||
|
|
||||||
runner = CliRunner()
|
runner = CliRunner()
|
||||||
|
|
||||||
|
|
@ -17,7 +17,7 @@ class TestHelpCommand:
|
||||||
"""Main help displays correctly."""
|
"""Main help displays correctly."""
|
||||||
result = runner.invoke(app, ["--help"])
|
result = runner.invoke(app, ["--help"])
|
||||||
assert result.exit_code == 0
|
assert result.exit_code == 0
|
||||||
assert "run" in result.output.lower() or "entropix" in result.output.lower()
|
assert "run" in result.output.lower() or "flakestorm" in result.output.lower()
|
||||||
|
|
||||||
def test_run_help(self):
|
def test_run_help(self):
|
||||||
"""Run command help displays options."""
|
"""Run command help displays options."""
|
||||||
|
|
@ -37,11 +37,11 @@ class TestHelpCommand:
|
||||||
|
|
||||||
|
|
||||||
class TestInitCommand:
|
class TestInitCommand:
|
||||||
"""Tests for `entropix init`."""
|
"""Tests for `flakestorm init`."""
|
||||||
|
|
||||||
def test_init_creates_config(self):
|
def test_init_creates_config(self):
|
||||||
"""init creates entropix.yaml."""
|
"""init creates flakestorm.yaml."""
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
with tempfile.TemporaryDirectory():
|
||||||
# Change to temp directory context
|
# Change to temp directory context
|
||||||
result = runner.invoke(app, ["init"], catch_exceptions=False)
|
result = runner.invoke(app, ["init"], catch_exceptions=False)
|
||||||
|
|
||||||
|
|
@ -55,12 +55,12 @@ class TestInitCommand:
|
||||||
|
|
||||||
|
|
||||||
class TestVerifyCommand:
|
class TestVerifyCommand:
|
||||||
"""Tests for `entropix verify`."""
|
"""Tests for `flakestorm verify`."""
|
||||||
|
|
||||||
def test_verify_valid_config(self):
|
def test_verify_valid_config(self):
|
||||||
"""verify accepts valid config."""
|
"""verify accepts valid config."""
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
config_path = Path(tmpdir) / "entropix.yaml"
|
config_path = Path(tmpdir) / "flakestorm.yaml"
|
||||||
config_path.write_text(
|
config_path.write_text(
|
||||||
"""
|
"""
|
||||||
agent:
|
agent:
|
||||||
|
|
@ -96,7 +96,7 @@ invariants: []
|
||||||
def test_verify_invalid_yaml(self):
|
def test_verify_invalid_yaml(self):
|
||||||
"""verify rejects invalid YAML syntax."""
|
"""verify rejects invalid YAML syntax."""
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
config_path = Path(tmpdir) / "entropix.yaml"
|
config_path = Path(tmpdir) / "flakestorm.yaml"
|
||||||
config_path.write_text("invalid: yaml: : content")
|
config_path.write_text("invalid: yaml: : content")
|
||||||
|
|
||||||
result = runner.invoke(app, ["verify", "--config", str(config_path)])
|
result = runner.invoke(app, ["verify", "--config", str(config_path)])
|
||||||
|
|
@ -105,7 +105,7 @@ invariants: []
|
||||||
|
|
||||||
|
|
||||||
class TestRunCommand:
|
class TestRunCommand:
|
||||||
"""Tests for `entropix run`."""
|
"""Tests for `flakestorm run`."""
|
||||||
|
|
||||||
def test_run_missing_config(self):
|
def test_run_missing_config(self):
|
||||||
"""run handles missing config."""
|
"""run handles missing config."""
|
||||||
|
|
@ -132,7 +132,7 @@ class TestRunCommand:
|
||||||
|
|
||||||
|
|
||||||
class TestReportCommand:
|
class TestReportCommand:
|
||||||
"""Tests for `entropix report`."""
|
"""Tests for `flakestorm report`."""
|
||||||
|
|
||||||
def test_report_help(self):
|
def test_report_help(self):
|
||||||
"""report command has help."""
|
"""report command has help."""
|
||||||
|
|
@ -141,7 +141,7 @@ class TestReportCommand:
|
||||||
|
|
||||||
|
|
||||||
class TestScoreCommand:
|
class TestScoreCommand:
|
||||||
"""Tests for `entropix score`."""
|
"""Tests for `flakestorm score`."""
|
||||||
|
|
||||||
def test_score_help(self):
|
def test_score_help(self):
|
||||||
"""score command has help."""
|
"""score command has help."""
|
||||||
|
|
|
||||||
|
|
@ -7,10 +7,10 @@ from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
AgentType,
|
AgentType,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
InvariantConfig,
|
InvariantConfig,
|
||||||
InvariantType,
|
InvariantType,
|
||||||
MutationConfig,
|
MutationConfig,
|
||||||
|
|
@ -20,8 +20,8 @@ from entropix.core.config import (
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestEntropixConfig:
|
class TestFlakeStormConfig:
|
||||||
"""Tests for EntropixConfig."""
|
"""Tests for FlakeStormConfig."""
|
||||||
|
|
||||||
def test_create_default_config(self):
|
def test_create_default_config(self):
|
||||||
"""Test creating a default configuration."""
|
"""Test creating a default configuration."""
|
||||||
|
|
@ -60,7 +60,7 @@ invariants:
|
||||||
- type: "latency"
|
- type: "latency"
|
||||||
max_ms: 1000
|
max_ms: 1000
|
||||||
"""
|
"""
|
||||||
config = EntropixConfig.from_yaml(yaml_content)
|
config = FlakeStormConfig.from_yaml(yaml_content)
|
||||||
|
|
||||||
assert config.agent.endpoint == "http://localhost:8000/test"
|
assert config.agent.endpoint == "http://localhost:8000/test"
|
||||||
assert config.agent.timeout == 5000
|
assert config.agent.timeout == 5000
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,8 @@ Tests for the mutation engine.
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from entropix.mutations.templates import MutationTemplates
|
from flakestorm.mutations.templates import MutationTemplates
|
||||||
from entropix.mutations.types import Mutation, MutationType
|
from flakestorm.mutations.types import Mutation, MutationType
|
||||||
|
|
||||||
|
|
||||||
class TestMutationType:
|
class TestMutationType:
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
"""Tests for the Entropix orchestrator."""
|
"""Tests for the flakestorm orchestrator."""
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
@ -11,7 +11,7 @@ class TestOrchestratorState:
|
||||||
|
|
||||||
def test_initial_state(self):
|
def test_initial_state(self):
|
||||||
"""State initializes correctly."""
|
"""State initializes correctly."""
|
||||||
from entropix.core.orchestrator import OrchestratorState
|
from flakestorm.core.orchestrator import OrchestratorState
|
||||||
|
|
||||||
state = OrchestratorState()
|
state = OrchestratorState()
|
||||||
assert state.total_mutations == 0
|
assert state.total_mutations == 0
|
||||||
|
|
@ -20,7 +20,7 @@ class TestOrchestratorState:
|
||||||
|
|
||||||
def test_state_started_at(self):
|
def test_state_started_at(self):
|
||||||
"""State records start time."""
|
"""State records start time."""
|
||||||
from entropix.core.orchestrator import OrchestratorState
|
from flakestorm.core.orchestrator import OrchestratorState
|
||||||
|
|
||||||
state = OrchestratorState()
|
state = OrchestratorState()
|
||||||
assert state.started_at is not None
|
assert state.started_at is not None
|
||||||
|
|
@ -28,7 +28,7 @@ class TestOrchestratorState:
|
||||||
|
|
||||||
def test_state_updates(self):
|
def test_state_updates(self):
|
||||||
"""State updates as tests run."""
|
"""State updates as tests run."""
|
||||||
from entropix.core.orchestrator import OrchestratorState
|
from flakestorm.core.orchestrator import OrchestratorState
|
||||||
|
|
||||||
state = OrchestratorState()
|
state = OrchestratorState()
|
||||||
state.total_mutations = 10
|
state.total_mutations = 10
|
||||||
|
|
@ -38,7 +38,7 @@ class TestOrchestratorState:
|
||||||
|
|
||||||
def test_state_duration_seconds(self):
|
def test_state_duration_seconds(self):
|
||||||
"""State calculates duration."""
|
"""State calculates duration."""
|
||||||
from entropix.core.orchestrator import OrchestratorState
|
from flakestorm.core.orchestrator import OrchestratorState
|
||||||
|
|
||||||
state = OrchestratorState()
|
state = OrchestratorState()
|
||||||
duration = state.duration_seconds
|
duration = state.duration_seconds
|
||||||
|
|
@ -47,7 +47,7 @@ class TestOrchestratorState:
|
||||||
|
|
||||||
def test_state_progress_percentage(self):
|
def test_state_progress_percentage(self):
|
||||||
"""State calculates progress percentage."""
|
"""State calculates progress percentage."""
|
||||||
from entropix.core.orchestrator import OrchestratorState
|
from flakestorm.core.orchestrator import OrchestratorState
|
||||||
|
|
||||||
state = OrchestratorState()
|
state = OrchestratorState()
|
||||||
state.total_mutations = 100
|
state.total_mutations = 100
|
||||||
|
|
@ -61,15 +61,15 @@ class TestOrchestrator:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def mock_config(self):
|
def mock_config(self):
|
||||||
"""Create a minimal test config."""
|
"""Create a minimal test config."""
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
AgentType,
|
AgentType,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
MutationConfig,
|
MutationConfig,
|
||||||
)
|
)
|
||||||
from entropix.mutations.types import MutationType
|
from flakestorm.mutations.types import MutationType
|
||||||
|
|
||||||
return EntropixConfig(
|
return FlakeStormConfig(
|
||||||
agent=AgentConfig(
|
agent=AgentConfig(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
type=AgentType.HTTP,
|
type=AgentType.HTTP,
|
||||||
|
|
@ -107,7 +107,7 @@ class TestOrchestrator:
|
||||||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||||
):
|
):
|
||||||
"""Orchestrator can be created with all required arguments."""
|
"""Orchestrator can be created with all required arguments."""
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
|
|
||||||
orchestrator = Orchestrator(
|
orchestrator = Orchestrator(
|
||||||
config=mock_config,
|
config=mock_config,
|
||||||
|
|
@ -122,7 +122,7 @@ class TestOrchestrator:
|
||||||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||||
):
|
):
|
||||||
"""Orchestrator has run method."""
|
"""Orchestrator has run method."""
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
|
|
||||||
orchestrator = Orchestrator(
|
orchestrator = Orchestrator(
|
||||||
config=mock_config,
|
config=mock_config,
|
||||||
|
|
@ -137,7 +137,7 @@ class TestOrchestrator:
|
||||||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||||
):
|
):
|
||||||
"""Orchestrator initializes state correctly."""
|
"""Orchestrator initializes state correctly."""
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
|
|
||||||
orchestrator = Orchestrator(
|
orchestrator = Orchestrator(
|
||||||
config=mock_config,
|
config=mock_config,
|
||||||
|
|
@ -152,7 +152,7 @@ class TestOrchestrator:
|
||||||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||||
):
|
):
|
||||||
"""Orchestrator stores all components."""
|
"""Orchestrator stores all components."""
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
|
|
||||||
orchestrator = Orchestrator(
|
orchestrator = Orchestrator(
|
||||||
config=mock_config,
|
config=mock_config,
|
||||||
|
|
@ -170,7 +170,7 @@ class TestOrchestrator:
|
||||||
"""Orchestrator accepts optional console."""
|
"""Orchestrator accepts optional console."""
|
||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
|
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
|
|
||||||
custom_console = Console()
|
custom_console = Console()
|
||||||
orchestrator = Orchestrator(
|
orchestrator = Orchestrator(
|
||||||
|
|
@ -186,7 +186,7 @@ class TestOrchestrator:
|
||||||
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
self, mock_config, mock_agent, mock_mutation_engine, mock_verifier
|
||||||
):
|
):
|
||||||
"""Orchestrator accepts show_progress flag."""
|
"""Orchestrator accepts show_progress flag."""
|
||||||
from entropix.core.orchestrator import Orchestrator
|
from flakestorm.core.orchestrator import Orchestrator
|
||||||
|
|
||||||
orchestrator = Orchestrator(
|
orchestrator = Orchestrator(
|
||||||
config=mock_config,
|
config=mock_config,
|
||||||
|
|
@ -203,8 +203,8 @@ class TestMutationGeneration:
|
||||||
|
|
||||||
def test_mutation_count_calculation(self):
|
def test_mutation_count_calculation(self):
|
||||||
"""Test mutation count is calculated correctly."""
|
"""Test mutation count is calculated correctly."""
|
||||||
from entropix.core.config import MutationConfig
|
from flakestorm.core.config import MutationConfig
|
||||||
from entropix.mutations.types import MutationType
|
from flakestorm.mutations.types import MutationType
|
||||||
|
|
||||||
config = MutationConfig(
|
config = MutationConfig(
|
||||||
count=10,
|
count=10,
|
||||||
|
|
@ -214,8 +214,8 @@ class TestMutationGeneration:
|
||||||
|
|
||||||
def test_mutation_types_configuration(self):
|
def test_mutation_types_configuration(self):
|
||||||
"""Test mutation types are configured correctly."""
|
"""Test mutation types are configured correctly."""
|
||||||
from entropix.core.config import MutationConfig
|
from flakestorm.core.config import MutationConfig
|
||||||
from entropix.mutations.types import MutationType
|
from flakestorm.mutations.types import MutationType
|
||||||
|
|
||||||
config = MutationConfig(
|
config = MutationConfig(
|
||||||
count=5,
|
count=5,
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ from pathlib import Path
|
||||||
|
|
||||||
# Import the performance module directly to avoid heavy dependencies like pydantic
|
# Import the performance module directly to avoid heavy dependencies like pydantic
|
||||||
_perf_path = (
|
_perf_path = (
|
||||||
Path(__file__).parent.parent / "src" / "entropix" / "core" / "performance.py"
|
Path(__file__).parent.parent / "src" / "flakestorm" / "core" / "performance.py"
|
||||||
)
|
)
|
||||||
_spec = importlib.util.spec_from_file_location("performance", _perf_path)
|
_spec = importlib.util.spec_from_file_location("performance", _perf_path)
|
||||||
_performance = importlib.util.module_from_spec(_spec)
|
_performance = importlib.util.module_from_spec(_spec)
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from entropix.mutations.types import Mutation, MutationType
|
from flakestorm.mutations.types import Mutation, MutationType
|
||||||
|
|
||||||
|
|
||||||
class TestCheckResult:
|
class TestCheckResult:
|
||||||
|
|
@ -15,7 +15,7 @@ class TestCheckResult:
|
||||||
|
|
||||||
def test_check_result_creation(self):
|
def test_check_result_creation(self):
|
||||||
"""CheckResult can be created."""
|
"""CheckResult can be created."""
|
||||||
from entropix.reports.models import CheckResult
|
from flakestorm.reports.models import CheckResult
|
||||||
|
|
||||||
result = CheckResult(
|
result = CheckResult(
|
||||||
check_type="contains",
|
check_type="contains",
|
||||||
|
|
@ -28,7 +28,7 @@ class TestCheckResult:
|
||||||
|
|
||||||
def test_check_result_to_dict(self):
|
def test_check_result_to_dict(self):
|
||||||
"""CheckResult converts to dict."""
|
"""CheckResult converts to dict."""
|
||||||
from entropix.reports.models import CheckResult
|
from flakestorm.reports.models import CheckResult
|
||||||
|
|
||||||
result = CheckResult(
|
result = CheckResult(
|
||||||
check_type="latency",
|
check_type="latency",
|
||||||
|
|
@ -55,7 +55,7 @@ class TestMutationResult:
|
||||||
|
|
||||||
def test_mutation_result_creation(self, sample_mutation):
|
def test_mutation_result_creation(self, sample_mutation):
|
||||||
"""MutationResult can be created."""
|
"""MutationResult can be created."""
|
||||||
from entropix.reports.models import MutationResult
|
from flakestorm.reports.models import MutationResult
|
||||||
|
|
||||||
result = MutationResult(
|
result = MutationResult(
|
||||||
original_prompt="What is the weather?",
|
original_prompt="What is the weather?",
|
||||||
|
|
@ -70,7 +70,7 @@ class TestMutationResult:
|
||||||
|
|
||||||
def test_mutation_result_with_checks(self, sample_mutation):
|
def test_mutation_result_with_checks(self, sample_mutation):
|
||||||
"""MutationResult with check results."""
|
"""MutationResult with check results."""
|
||||||
from entropix.reports.models import CheckResult, MutationResult
|
from flakestorm.reports.models import CheckResult, MutationResult
|
||||||
|
|
||||||
checks = [
|
checks = [
|
||||||
CheckResult(check_type="contains", passed=True, details="Found 'weather'"),
|
CheckResult(check_type="contains", passed=True, details="Found 'weather'"),
|
||||||
|
|
@ -90,7 +90,7 @@ class TestMutationResult:
|
||||||
|
|
||||||
def test_mutation_result_failed_checks(self, sample_mutation):
|
def test_mutation_result_failed_checks(self, sample_mutation):
|
||||||
"""MutationResult returns failed checks."""
|
"""MutationResult returns failed checks."""
|
||||||
from entropix.reports.models import CheckResult, MutationResult
|
from flakestorm.reports.models import CheckResult, MutationResult
|
||||||
|
|
||||||
checks = [
|
checks = [
|
||||||
CheckResult(check_type="contains", passed=True, details="OK"),
|
CheckResult(check_type="contains", passed=True, details="OK"),
|
||||||
|
|
@ -114,7 +114,7 @@ class TestTypeStatistics:
|
||||||
|
|
||||||
def test_type_statistics_creation(self):
|
def test_type_statistics_creation(self):
|
||||||
"""TypeStatistics can be created."""
|
"""TypeStatistics can be created."""
|
||||||
from entropix.reports.models import TypeStatistics
|
from flakestorm.reports.models import TypeStatistics
|
||||||
|
|
||||||
stats = TypeStatistics(
|
stats = TypeStatistics(
|
||||||
mutation_type="paraphrase",
|
mutation_type="paraphrase",
|
||||||
|
|
@ -129,7 +129,7 @@ class TestTypeStatistics:
|
||||||
|
|
||||||
def test_type_statistics_to_dict(self):
|
def test_type_statistics_to_dict(self):
|
||||||
"""TypeStatistics converts to dict."""
|
"""TypeStatistics converts to dict."""
|
||||||
from entropix.reports.models import TypeStatistics
|
from flakestorm.reports.models import TypeStatistics
|
||||||
|
|
||||||
stats = TypeStatistics(
|
stats = TypeStatistics(
|
||||||
mutation_type="noise",
|
mutation_type="noise",
|
||||||
|
|
@ -147,7 +147,7 @@ class TestTestStatistics:
|
||||||
|
|
||||||
def test_statistics_creation(self):
|
def test_statistics_creation(self):
|
||||||
"""TestStatistics can be created."""
|
"""TestStatistics can be created."""
|
||||||
from entropix.reports.models import TestStatistics
|
from flakestorm.reports.models import TestStatistics
|
||||||
|
|
||||||
stats = TestStatistics(
|
stats = TestStatistics(
|
||||||
total_mutations=100,
|
total_mutations=100,
|
||||||
|
|
@ -165,7 +165,7 @@ class TestTestStatistics:
|
||||||
|
|
||||||
def test_statistics_pass_rate(self):
|
def test_statistics_pass_rate(self):
|
||||||
"""Statistics calculates pass_rate correctly."""
|
"""Statistics calculates pass_rate correctly."""
|
||||||
from entropix.reports.models import TestStatistics
|
from flakestorm.reports.models import TestStatistics
|
||||||
|
|
||||||
stats = TestStatistics(
|
stats = TestStatistics(
|
||||||
total_mutations=100,
|
total_mutations=100,
|
||||||
|
|
@ -181,7 +181,7 @@ class TestTestStatistics:
|
||||||
|
|
||||||
def test_statistics_zero_total(self):
|
def test_statistics_zero_total(self):
|
||||||
"""Statistics handles zero total."""
|
"""Statistics handles zero total."""
|
||||||
from entropix.reports.models import TestStatistics
|
from flakestorm.reports.models import TestStatistics
|
||||||
|
|
||||||
stats = TestStatistics(
|
stats = TestStatistics(
|
||||||
total_mutations=0,
|
total_mutations=0,
|
||||||
|
|
@ -202,13 +202,13 @@ class TestTestResults:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_config(self):
|
def sample_config(self):
|
||||||
"""Create sample config."""
|
"""Create sample config."""
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
AgentType,
|
AgentType,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
)
|
)
|
||||||
|
|
||||||
return EntropixConfig(
|
return FlakeStormConfig(
|
||||||
agent=AgentConfig(
|
agent=AgentConfig(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
type=AgentType.HTTP,
|
type=AgentType.HTTP,
|
||||||
|
|
@ -220,7 +220,7 @@ class TestTestResults:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_statistics(self):
|
def sample_statistics(self):
|
||||||
"""Create sample statistics."""
|
"""Create sample statistics."""
|
||||||
from entropix.reports.models import TestStatistics
|
from flakestorm.reports.models import TestStatistics
|
||||||
|
|
||||||
return TestStatistics(
|
return TestStatistics(
|
||||||
total_mutations=10,
|
total_mutations=10,
|
||||||
|
|
@ -235,7 +235,7 @@ class TestTestResults:
|
||||||
|
|
||||||
def test_results_creation(self, sample_config, sample_statistics):
|
def test_results_creation(self, sample_config, sample_statistics):
|
||||||
"""TestResults can be created."""
|
"""TestResults can be created."""
|
||||||
from entropix.reports.models import TestResults
|
from flakestorm.reports.models import TestResults
|
||||||
|
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
results = TestResults(
|
results = TestResults(
|
||||||
|
|
@ -255,13 +255,13 @@ class TestHTMLReportGenerator:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_config(self):
|
def sample_config(self):
|
||||||
"""Create sample config."""
|
"""Create sample config."""
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
AgentType,
|
AgentType,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
)
|
)
|
||||||
|
|
||||||
return EntropixConfig(
|
return FlakeStormConfig(
|
||||||
agent=AgentConfig(
|
agent=AgentConfig(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
type=AgentType.HTTP,
|
type=AgentType.HTTP,
|
||||||
|
|
@ -273,7 +273,7 @@ class TestHTMLReportGenerator:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_statistics(self):
|
def sample_statistics(self):
|
||||||
"""Create sample statistics."""
|
"""Create sample statistics."""
|
||||||
from entropix.reports.models import TestStatistics
|
from flakestorm.reports.models import TestStatistics
|
||||||
|
|
||||||
return TestStatistics(
|
return TestStatistics(
|
||||||
total_mutations=10,
|
total_mutations=10,
|
||||||
|
|
@ -289,7 +289,7 @@ class TestHTMLReportGenerator:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_results(self, sample_config, sample_statistics):
|
def sample_results(self, sample_config, sample_statistics):
|
||||||
"""Create sample test results."""
|
"""Create sample test results."""
|
||||||
from entropix.reports.models import TestResults
|
from flakestorm.reports.models import TestResults
|
||||||
|
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
return TestResults(
|
return TestResults(
|
||||||
|
|
@ -302,14 +302,14 @@ class TestHTMLReportGenerator:
|
||||||
|
|
||||||
def test_generator_creation(self, sample_results):
|
def test_generator_creation(self, sample_results):
|
||||||
"""Generator can be created."""
|
"""Generator can be created."""
|
||||||
from entropix.reports.html import HTMLReportGenerator
|
from flakestorm.reports.html import HTMLReportGenerator
|
||||||
|
|
||||||
generator = HTMLReportGenerator(sample_results)
|
generator = HTMLReportGenerator(sample_results)
|
||||||
assert generator is not None
|
assert generator is not None
|
||||||
|
|
||||||
def test_generate_returns_string(self, sample_results):
|
def test_generate_returns_string(self, sample_results):
|
||||||
"""Generator returns HTML string."""
|
"""Generator returns HTML string."""
|
||||||
from entropix.reports.html import HTMLReportGenerator
|
from flakestorm.reports.html import HTMLReportGenerator
|
||||||
|
|
||||||
generator = HTMLReportGenerator(sample_results)
|
generator = HTMLReportGenerator(sample_results)
|
||||||
html = generator.generate()
|
html = generator.generate()
|
||||||
|
|
@ -319,7 +319,7 @@ class TestHTMLReportGenerator:
|
||||||
|
|
||||||
def test_generate_valid_html_structure(self, sample_results):
|
def test_generate_valid_html_structure(self, sample_results):
|
||||||
"""Generated HTML has valid structure."""
|
"""Generated HTML has valid structure."""
|
||||||
from entropix.reports.html import HTMLReportGenerator
|
from flakestorm.reports.html import HTMLReportGenerator
|
||||||
|
|
||||||
generator = HTMLReportGenerator(sample_results)
|
generator = HTMLReportGenerator(sample_results)
|
||||||
html = generator.generate()
|
html = generator.generate()
|
||||||
|
|
@ -329,7 +329,7 @@ class TestHTMLReportGenerator:
|
||||||
|
|
||||||
def test_contains_robustness_score(self, sample_results):
|
def test_contains_robustness_score(self, sample_results):
|
||||||
"""Report contains robustness score."""
|
"""Report contains robustness score."""
|
||||||
from entropix.reports.html import HTMLReportGenerator
|
from flakestorm.reports.html import HTMLReportGenerator
|
||||||
|
|
||||||
generator = HTMLReportGenerator(sample_results)
|
generator = HTMLReportGenerator(sample_results)
|
||||||
html = generator.generate()
|
html = generator.generate()
|
||||||
|
|
@ -339,7 +339,7 @@ class TestHTMLReportGenerator:
|
||||||
|
|
||||||
def test_save_creates_file(self, sample_results):
|
def test_save_creates_file(self, sample_results):
|
||||||
"""save() creates file on disk."""
|
"""save() creates file on disk."""
|
||||||
from entropix.reports.html import HTMLReportGenerator
|
from flakestorm.reports.html import HTMLReportGenerator
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
generator = HTMLReportGenerator(sample_results)
|
generator = HTMLReportGenerator(sample_results)
|
||||||
|
|
@ -356,13 +356,13 @@ class TestJSONReportGenerator:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_config(self):
|
def sample_config(self):
|
||||||
"""Create sample config."""
|
"""Create sample config."""
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
AgentType,
|
AgentType,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
)
|
)
|
||||||
|
|
||||||
return EntropixConfig(
|
return FlakeStormConfig(
|
||||||
agent=AgentConfig(
|
agent=AgentConfig(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
type=AgentType.HTTP,
|
type=AgentType.HTTP,
|
||||||
|
|
@ -374,7 +374,7 @@ class TestJSONReportGenerator:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_statistics(self):
|
def sample_statistics(self):
|
||||||
"""Create sample statistics."""
|
"""Create sample statistics."""
|
||||||
from entropix.reports.models import TestStatistics
|
from flakestorm.reports.models import TestStatistics
|
||||||
|
|
||||||
return TestStatistics(
|
return TestStatistics(
|
||||||
total_mutations=10,
|
total_mutations=10,
|
||||||
|
|
@ -390,7 +390,7 @@ class TestJSONReportGenerator:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_results(self, sample_config, sample_statistics):
|
def sample_results(self, sample_config, sample_statistics):
|
||||||
"""Create sample test results."""
|
"""Create sample test results."""
|
||||||
from entropix.reports.models import TestResults
|
from flakestorm.reports.models import TestResults
|
||||||
|
|
||||||
ts = datetime(2024, 1, 15, 12, 0, 0)
|
ts = datetime(2024, 1, 15, 12, 0, 0)
|
||||||
return TestResults(
|
return TestResults(
|
||||||
|
|
@ -403,14 +403,14 @@ class TestJSONReportGenerator:
|
||||||
|
|
||||||
def test_generator_creation(self, sample_results):
|
def test_generator_creation(self, sample_results):
|
||||||
"""Generator can be created."""
|
"""Generator can be created."""
|
||||||
from entropix.reports.json_export import JSONReportGenerator
|
from flakestorm.reports.json_export import JSONReportGenerator
|
||||||
|
|
||||||
generator = JSONReportGenerator(sample_results)
|
generator = JSONReportGenerator(sample_results)
|
||||||
assert generator is not None
|
assert generator is not None
|
||||||
|
|
||||||
def test_generate_valid_json(self, sample_results):
|
def test_generate_valid_json(self, sample_results):
|
||||||
"""Generator produces valid JSON."""
|
"""Generator produces valid JSON."""
|
||||||
from entropix.reports.json_export import JSONReportGenerator
|
from flakestorm.reports.json_export import JSONReportGenerator
|
||||||
|
|
||||||
generator = JSONReportGenerator(sample_results)
|
generator = JSONReportGenerator(sample_results)
|
||||||
json_str = generator.generate()
|
json_str = generator.generate()
|
||||||
|
|
@ -421,7 +421,7 @@ class TestJSONReportGenerator:
|
||||||
|
|
||||||
def test_contains_statistics(self, sample_results):
|
def test_contains_statistics(self, sample_results):
|
||||||
"""JSON contains statistics."""
|
"""JSON contains statistics."""
|
||||||
from entropix.reports.json_export import JSONReportGenerator
|
from flakestorm.reports.json_export import JSONReportGenerator
|
||||||
|
|
||||||
generator = JSONReportGenerator(sample_results)
|
generator = JSONReportGenerator(sample_results)
|
||||||
data = json.loads(generator.generate())
|
data = json.loads(generator.generate())
|
||||||
|
|
@ -431,7 +431,7 @@ class TestJSONReportGenerator:
|
||||||
|
|
||||||
def test_save_creates_file(self, sample_results):
|
def test_save_creates_file(self, sample_results):
|
||||||
"""save() creates JSON file on disk."""
|
"""save() creates JSON file on disk."""
|
||||||
from entropix.reports.json_export import JSONReportGenerator
|
from flakestorm.reports.json_export import JSONReportGenerator
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
generator = JSONReportGenerator(sample_results)
|
generator = JSONReportGenerator(sample_results)
|
||||||
|
|
@ -448,13 +448,13 @@ class TestTerminalReporter:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_config(self):
|
def sample_config(self):
|
||||||
"""Create sample config."""
|
"""Create sample config."""
|
||||||
from entropix.core.config import (
|
from flakestorm.core.config import (
|
||||||
AgentConfig,
|
AgentConfig,
|
||||||
AgentType,
|
AgentType,
|
||||||
EntropixConfig,
|
FlakeStormConfig,
|
||||||
)
|
)
|
||||||
|
|
||||||
return EntropixConfig(
|
return FlakeStormConfig(
|
||||||
agent=AgentConfig(
|
agent=AgentConfig(
|
||||||
endpoint="http://localhost:8000/chat",
|
endpoint="http://localhost:8000/chat",
|
||||||
type=AgentType.HTTP,
|
type=AgentType.HTTP,
|
||||||
|
|
@ -466,7 +466,7 @@ class TestTerminalReporter:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_statistics(self):
|
def sample_statistics(self):
|
||||||
"""Create sample statistics."""
|
"""Create sample statistics."""
|
||||||
from entropix.reports.models import TestStatistics
|
from flakestorm.reports.models import TestStatistics
|
||||||
|
|
||||||
return TestStatistics(
|
return TestStatistics(
|
||||||
total_mutations=10,
|
total_mutations=10,
|
||||||
|
|
@ -482,7 +482,7 @@ class TestTerminalReporter:
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sample_results(self, sample_config, sample_statistics):
|
def sample_results(self, sample_config, sample_statistics):
|
||||||
"""Create sample test results."""
|
"""Create sample test results."""
|
||||||
from entropix.reports.models import TestResults
|
from flakestorm.reports.models import TestResults
|
||||||
|
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
return TestResults(
|
return TestResults(
|
||||||
|
|
@ -495,14 +495,14 @@ class TestTerminalReporter:
|
||||||
|
|
||||||
def test_reporter_creation(self, sample_results):
|
def test_reporter_creation(self, sample_results):
|
||||||
"""Reporter can be created."""
|
"""Reporter can be created."""
|
||||||
from entropix.reports.terminal import TerminalReporter
|
from flakestorm.reports.terminal import TerminalReporter
|
||||||
|
|
||||||
reporter = TerminalReporter(sample_results)
|
reporter = TerminalReporter(sample_results)
|
||||||
assert reporter is not None
|
assert reporter is not None
|
||||||
|
|
||||||
def test_reporter_has_print_methods(self, sample_results):
|
def test_reporter_has_print_methods(self, sample_results):
|
||||||
"""Reporter has print methods."""
|
"""Reporter has print methods."""
|
||||||
from entropix.reports.terminal import TerminalReporter
|
from flakestorm.reports.terminal import TerminalReporter
|
||||||
|
|
||||||
reporter = TerminalReporter(sample_results)
|
reporter = TerminalReporter(sample_results)
|
||||||
assert hasattr(reporter, "print_summary")
|
assert hasattr(reporter, "print_summary")
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue