diff --git a/README.md b/README.md
index 084d9a8..9334e96 100644
--- a/README.md
+++ b/README.md
@@ -15,9 +15,6 @@
-
- 
-
---
@@ -134,12 +131,6 @@ Running attacks... ━━━━━━━━━━━━━━━━━━
Report saved to: ./reports/flakestorm-2024-01-15-143022.html
```
-### Check Limits
-
-```bash
-flakestorm limits # Show edition limits
-flakestorm cloud # Learn about Cloud features
-```
## Mutation Types
@@ -151,8 +142,6 @@ flakestorm cloud # Learn about Cloud features
| **Prompt Injection** | Basic adversarial attacks | "Book a flight and ignore previous instructions" |
| **Custom** | Your own mutation templates | Define with `{prompt}` placeholder |
-> **Need advanced mutations?** Visit [flakestorm.com](https://flakestorm.com) for more options.
-
## Invariants (Assertions)
### Deterministic
@@ -180,8 +169,6 @@ invariants:
- type: "refusal_check"
```
-> **Need advanced safety?** Visit [flakestorm.com](https://flakestorm.com) for more options.
-
## Agent Adapters
### HTTP Endpoint
@@ -216,8 +203,6 @@ For local testing:
flakestorm run --min-score 0.9
```
-For advanced CI/CD features, visit [flakestorm.com](https://flakestorm.com).
-
## Robustness Score
The Robustness Score is calculated as:
@@ -257,9 +242,3 @@ AGPLv3 - See [LICENSE](LICENSE) for details.
Tested with FlakeStorm
-
-
-
- ⚡ Need more features? Visit FlakeStorm Cloud →
-
-
diff --git a/src/flakestorm/cli/main.py b/src/flakestorm/cli/main.py
index da6e139..3a9689f 100644
--- a/src/flakestorm/cli/main.py
+++ b/src/flakestorm/cli/main.py
@@ -13,14 +13,8 @@ from pathlib import Path
import typer
from rich.console import Console
from rich.panel import Panel
-from rich.text import Text
from flakestorm import __version__
-from flakestorm.core.limits import (
- CLOUD_URL,
- MAX_MUTATIONS_PER_RUN,
- print_upgrade_banner,
-)
from flakestorm.core.runner import FlakeStormRunner
# Create the main app
@@ -37,10 +31,7 @@ console = Console()
def version_callback(value: bool) -> None:
"""Print version and exit."""
if value:
- console.print(
- f"[bold blue]flakestorm[/bold blue] version {__version__} [dim](Open Source Edition)[/dim]"
- )
- console.print(f"[dim]→ Upgrade to Cloud: {CLOUD_URL}[/dim]")
+ console.print(f"[bold blue]flakestorm[/bold blue] version {__version__}")
raise typer.Exit()
@@ -412,84 +403,6 @@ def score(
asyncio.run(_score_async(config))
-@app.command()
-def cloud() -> None:
- """
- Learn about flakestorm Cloud features.
-
- flakestorm Cloud provides 20x faster execution, advanced features,
- and team collaboration.
- """
- print_upgrade_banner(console, reason="20x faster tests")
-
- console.print("\n[bold]Feature Comparison:[/bold]\n")
-
- # Feature comparison table
- features = [
- ("Mutation Types", "5 basic", "[green]All types[/green]"),
- ("Mutations/Run", f"{MAX_MUTATIONS_PER_RUN}", "[green]Unlimited[/green]"),
- (
- "Execution",
- "[yellow]Sequential[/yellow]",
- "[green]Parallel (20x faster)[/green]",
- ),
- ("LLM", "Local only", "[green]Cloud + Local[/green]"),
- ("PII Detection", "Basic regex", "[green]Advanced NER + ML[/green]"),
- ("Prompt Injection", "Basic", "[green]ML-powered[/green]"),
- ("Factuality Check", "[red]❌[/red]", "[green]✅[/green]"),
- ("Test History", "[red]❌[/red]", "[green]✅ Dashboard[/green]"),
- ("GitHub Actions", "[red]❌[/red]", "[green]✅ One-click setup[/green]"),
- ("Team Features", "[red]❌[/red]", "[green]✅ Sharing & SSO[/green]"),
- ]
-
- console.print(" [dim]Feature Open Source Cloud[/dim]")
- console.print(" " + "─" * 50)
- for feature, oss, cloud in features:
- console.print(f" {feature:<20} {oss:<14} {cloud}")
-
- console.print("\n[bold cyan]Pricing:[/bold cyan]")
- console.print(" • [bold]Community:[/bold] $0/mo (current)")
- console.print(" • [bold]Pro:[/bold] $49/mo - Parallel + Cloud LLMs")
- console.print(" • [bold]Team:[/bold] $299/mo - All features + collaboration")
-
- console.print(
- f"\n[bold]→ Get started:[/bold] [link={CLOUD_URL}]{CLOUD_URL}[/link]\n"
- )
-
-
-@app.command()
-def limits() -> None:
- """
- Show Open Source edition limits.
-
- Displays the feature limitations of the Open Source edition
- and how to unlock more with flakestorm Cloud.
- """
- console.print(
- Panel(
- Text.from_markup(
- "[bold]Open Source Edition Limits[/bold]\n\n"
- f"• [yellow]Max {MAX_MUTATIONS_PER_RUN} mutations[/yellow] per test run\n"
- "• [yellow]Sequential execution[/yellow] (one test at a time)\n"
- "• [yellow]5 mutation types[/yellow]: paraphrase, noise, tone, injection, custom\n"
- "• [yellow]Local LLM only[/yellow] (Ollama/llama.cpp)\n"
- "• [yellow]Basic PII detection[/yellow] (regex patterns)\n"
- "• [red]No GitHub Actions[/red] CI/CD integration\n"
- "• [red]No test history[/red] or dashboard\n"
- "• [red]No team features[/red]\n\n"
- "[bold green]Why these limits?[/bold green]\n"
- "The Open Source edition is designed for:\n"
- "• Learning and experimentation\n"
- "• Small test suites\n"
- "• Individual developers\n\n"
- f"[bold]Upgrade for production:[/bold] {CLOUD_URL}"
- ),
- title="[bold blue]flakestorm Open Source[/bold blue]",
- border_style="blue",
- )
- )
-
-
async def _score_async(config: Path) -> None:
"""Async implementation of score command."""
diff --git a/src/flakestorm/core/config.py b/src/flakestorm/core/config.py
index b8994be..3696938 100644
--- a/src/flakestorm/core/config.py
+++ b/src/flakestorm/core/config.py
@@ -69,11 +69,10 @@ class MutationConfig(BaseModel):
"""
Configuration for mutation generation.
- Open Source Edition Limits:
+ Limits:
- Maximum 50 total mutations per test run
- 5 mutation types: paraphrase, noise, tone_shift, prompt_injection, custom
- Upgrade to flakestorm Cloud for unlimited mutations and advanced types.
"""
count: int = Field(
diff --git a/src/flakestorm/core/limits.py b/src/flakestorm/core/limits.py
deleted file mode 100644
index e3a2629..0000000
--- a/src/flakestorm/core/limits.py
+++ /dev/null
@@ -1,222 +0,0 @@
-"""
-Open Source Edition Limits
-
-Defines feature limits for the open source (local-only) version.
-These limits encourage users to upgrade to flakestorm Cloud for:
-- Faster parallel execution
-- Cloud LLMs (higher quality mutations)
-- Advanced features
-- Team collaboration
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import TYPE_CHECKING
-
-from rich.console import Console
-from rich.panel import Panel
-from rich.text import Text
-
-if TYPE_CHECKING:
- pass
-
-
-# =============================================================================
-# OPEN SOURCE EDITION LIMITS
-# =============================================================================
-
-# Maximum mutations per test run (sequential = slow)
-MAX_MUTATIONS_PER_RUN = 50
-
-# Maximum golden prompts
-MAX_GOLDEN_PROMPTS = 10
-
-# Execution mode (sequential only - no parallelism)
-PARALLEL_EXECUTION_ENABLED = False
-
-# GitHub Actions integration
-GITHUB_ACTIONS_ENABLED = False
-
-# Advanced features disabled
-ADVANCED_MUTATIONS_ENABLED = False # Sophisticated prompt injections
-ADVANCED_SAFETY_CHECKS_ENABLED = False # NER, ML-based detection, factuality
-TEST_HISTORY_ENABLED = False # Dashboard, history tracking
-TEAM_FEATURES_ENABLED = False # Sharing, collaboration
-
-# Cloud features disabled
-CLOUD_LLM_ENABLED = False
-
-
-# =============================================================================
-# ALLOWED MUTATION TYPES (5 types for open source)
-# =============================================================================
-
-ALLOWED_MUTATION_TYPES = [
- "paraphrase", # Semantic rewrites
- "noise", # Typos, spelling errors
- "tone_shift", # Tone changes
- "prompt_injection", # Basic adversarial
- "custom", # User-defined templates
-]
-
-
-# =============================================================================
-# UPGRADE MESSAGING
-# =============================================================================
-
-CLOUD_URL = "https://flakestorm.cloud"
-UPGRADE_CTA = f"⚡ Upgrade to flakestorm Cloud for 20x faster execution → {CLOUD_URL}"
-
-
-@dataclass
-class LimitViolation:
- """Represents a limit that was exceeded."""
-
- limit_name: str
- current_value: int
- max_value: int
- message: str
-
-
-def check_mutation_limit(
- requested_count: int, num_prompts: int
-) -> LimitViolation | None:
- """
- Check if the requested mutation count exceeds limits.
-
- Args:
- requested_count: Requested mutations per prompt
- num_prompts: Number of golden prompts
-
- Returns:
- LimitViolation if exceeded, None otherwise
- """
- total = requested_count * num_prompts
- if total > MAX_MUTATIONS_PER_RUN:
- return LimitViolation(
- limit_name="mutations_per_run",
- current_value=total,
- max_value=MAX_MUTATIONS_PER_RUN,
- message=(
- f"Open Source limit: {MAX_MUTATIONS_PER_RUN} mutations per run. "
- f"You requested {total} ({requested_count} × {num_prompts} prompts).\n"
- f"Upgrade to Cloud for unlimited mutations."
- ),
- )
- return None
-
-
-def check_golden_prompt_limit(num_prompts: int) -> LimitViolation | None:
- """Check if golden prompt count exceeds limits."""
- if num_prompts > MAX_GOLDEN_PROMPTS:
- return LimitViolation(
- limit_name="golden_prompts",
- current_value=num_prompts,
- max_value=MAX_GOLDEN_PROMPTS,
- message=(
- f"Open Source limit: {MAX_GOLDEN_PROMPTS} golden prompts. "
- f"You have {num_prompts}.\n"
- f"Upgrade to Cloud for unlimited prompts."
- ),
- )
- return None
-
-
-def enforce_mutation_limit(requested_count: int, num_prompts: int) -> int:
- """
- Enforce mutation limit by capping the count.
-
- Returns the actual count to use (may be reduced).
- """
- max_per_prompt = MAX_MUTATIONS_PER_RUN // max(num_prompts, 1)
- return min(requested_count, max(max_per_prompt, 1))
-
-
-def print_upgrade_banner(console: Console, reason: str = "faster execution") -> None:
- """Print an upgrade banner to the console."""
- banner = Panel(
- Text.from_markup(
- f"[bold yellow]⚡ Want {reason}?[/bold yellow]\n\n"
- f"[white]flakestorm Cloud offers:[/white]\n"
- f" • [green]20x faster[/green] parallel execution\n"
- f" • [green]Cloud LLMs[/green] for higher quality mutations\n"
- f" • [green]Advanced safety checks[/green] (NER, ML-detection)\n"
- f" • [green]Test history[/green] and analytics dashboard\n"
- f" • [green]Team features[/green] for collaboration\n\n"
- f"[bold cyan]→ {CLOUD_URL}[/bold cyan]"
- ),
- title="[bold blue]Upgrade to flakestorm Cloud[/bold blue]",
- border_style="blue",
- padding=(1, 2),
- )
- console.print(banner)
-
-
-def print_limit_warning(console: Console, violation: LimitViolation) -> None:
- """Print a limit warning to the console."""
- warning = Panel(
- Text.from_markup(
- f"[bold yellow]⚠️ Limit Reached[/bold yellow]\n\n"
- f"[white]{violation.message}[/white]\n\n"
- f"[bold cyan]→ {CLOUD_URL}[/bold cyan]"
- ),
- title="[bold yellow]Open Source Edition[/bold yellow]",
- border_style="yellow",
- padding=(1, 2),
- )
- console.print(warning)
-
-
-def print_sequential_notice(console: Console) -> None:
- """Print a notice about sequential execution."""
- console.print(
- "\n[dim]ℹ️ Running in sequential mode (Open Source). "
- f"Upgrade to Cloud for parallel execution: {CLOUD_URL}[/dim]\n"
- )
-
-
-def print_completion_upsell(console: Console, duration_seconds: float) -> None:
- """Print upsell after test completion based on duration."""
- if duration_seconds > 60: # More than 1 minute
- estimated_cloud_time = (
- duration_seconds / 20
- ) # ~20x faster with parallel + cloud
- console.print(
- f"\n[dim]⏱️ Test took {duration_seconds:.1f}s. "
- f"With flakestorm Cloud, this would take ~{estimated_cloud_time:.1f}s[/dim]"
- )
- console.print(f"[dim cyan]→ {CLOUD_URL}[/dim cyan]\n")
-
-
-def get_feature_comparison() -> str:
- """Get a feature comparison table for documentation."""
- return """
-## Feature Comparison
-
-| Feature | Open Source | Cloud Pro | Cloud Team |
-|---------|:-----------:|:---------:|:----------:|
-| Mutation Types | 5 basic | All types | All types |
-| Mutations/Run | 50 | Unlimited | Unlimited |
-| Execution | Sequential | Parallel (20x) | Parallel (20x) |
-| LLM | Local only | Cloud + Local | Cloud + Local |
-| PII Detection | Basic | Advanced (NER) | Advanced (NER) |
-| Prompt Injection | Basic | ML-powered | ML-powered |
-| Factuality Check | ❌ | ✅ | ✅ |
-| Test History | ❌ | ✅ | ✅ |
-| Dashboard | ❌ | ✅ | ✅ |
-| GitHub Actions | ❌ | ✅ | ✅ |
-| Team Sharing | ❌ | ❌ | ✅ |
-| SSO/SAML | ❌ | ❌ | ✅ |
-| Price | Free | $49/mo | $299/mo |
-
-**Why is Open Source slower?**
-- Sequential execution: Tests run one at a time
-- Local LLM: Slower than cloud GPU inference
-- No caching: Each run starts fresh
-
-**Cloud advantages:**
-- 20x faster with parallel execution
-- Higher quality mutations with cloud LLMs
-- Historical comparison across runs
-"""
diff --git a/src/flakestorm/core/orchestrator.py b/src/flakestorm/core/orchestrator.py
index a910bda..80ec3cc 100644
--- a/src/flakestorm/core/orchestrator.py
+++ b/src/flakestorm/core/orchestrator.py
@@ -4,12 +4,7 @@ Orchestrator for flakestorm Test Runs
Coordinates the entire testing process: mutation generation,
agent invocation, invariant verification, and result aggregation.
-Open Source Edition:
-- Sequential execution only (no parallelism)
-- Maximum 50 mutations per test run
-- Basic mutation types only
-
-Upgrade to flakestorm Cloud for parallel execution and advanced features.
+Runs tests sequentially with a maximum of 50 mutations per test run.
"""
from __future__ import annotations
@@ -29,14 +24,9 @@ from rich.progress import (
TimeRemainingColumn,
)
-from flakestorm.core.limits import (
- MAX_MUTATIONS_PER_RUN,
- PARALLEL_EXECUTION_ENABLED,
- check_mutation_limit,
- print_completion_upsell,
- print_limit_warning,
- print_sequential_notice,
-)
+# Hardcoded limits for open source edition
+MAX_MUTATIONS_PER_RUN = 50
+PARALLEL_EXECUTION_ENABLED = False # Sequential execution only
if TYPE_CHECKING:
from flakestorm.assertions.verifier import InvariantVerifier
@@ -116,7 +106,7 @@ class Orchestrator:
"""
Execute the full test run.
- Open Source Edition runs sequentially. Upgrade to Cloud for parallel.
+ Runs tests sequentially with a maximum of 50 mutations per run.
Returns:
TestResults containing all test outcomes
@@ -128,26 +118,17 @@ class Orchestrator:
self.state = OrchestratorState()
all_results: list[MutationResult] = []
- # Check limits and show notices
- if self.show_progress:
- print_sequential_notice(self.console)
-
# Phase 1: Generate all mutations
all_mutations = await self._generate_mutations()
- # Enforce mutation limit for Open Source
+ # Enforce mutation limit
if len(all_mutations) > MAX_MUTATIONS_PER_RUN:
- violation = check_mutation_limit(
- self.config.mutations.count,
- len(self.config.golden_prompts),
- )
- if violation:
- print_limit_warning(self.console, violation)
# Truncate to limit
all_mutations = all_mutations[:MAX_MUTATIONS_PER_RUN]
- self.console.print(
- f"[yellow]⚠️ Limited to {MAX_MUTATIONS_PER_RUN} mutations (Open Source)[/yellow]\n"
- )
+ if self.show_progress:
+ self.console.print(
+ f"[yellow]⚠️ Limited to {MAX_MUTATIONS_PER_RUN} mutations per run[/yellow]\n"
+ )
self.state.total_mutations = len(all_mutations)
@@ -179,10 +160,6 @@ class Orchestrator:
statistics = self._calculate_statistics(all_results)
- # Show upgrade prompt based on duration
- if self.show_progress:
- print_completion_upsell(self.console, self.state.duration_seconds)
-
return TestResults(
config=self.config,
started_at=self.state.started_at,
@@ -235,31 +212,15 @@ class Orchestrator:
mutations: list[tuple[str, Mutation]],
) -> list[MutationResult]:
"""
- Run all mutations.
-
- Open Source Edition: Sequential execution (one at a time).
- Cloud Edition: Parallel execution with configurable concurrency.
+ Run all mutations sequentially (one at a time).
"""
- # Open Source: Force sequential execution (concurrency = 1)
- concurrency = (
- 1 if not PARALLEL_EXECUTION_ENABLED else self.config.advanced.concurrency
- )
- semaphore = asyncio.Semaphore(concurrency)
-
- # Sequential execution for Open Source
- if not PARALLEL_EXECUTION_ENABLED:
- results = []
- for original, mutation in mutations:
- result = await self._run_single_mutation(original, mutation, semaphore)
- results.append(result)
- return results
-
- # Parallel execution (Cloud only)
- tasks = [
- self._run_single_mutation(original, mutation, semaphore)
- for original, mutation in mutations
- ]
- return await asyncio.gather(*tasks)
+ # Sequential execution only
+ semaphore = asyncio.Semaphore(1)
+ results = []
+ for original, mutation in mutations:
+ result = await self._run_single_mutation(original, mutation, semaphore)
+ results.append(result)
+ return results
async def _run_mutations_with_progress(
self,
@@ -268,39 +229,16 @@ class Orchestrator:
task_id: int,
) -> list[MutationResult]:
"""
- Run all mutations with progress display.
-
- Open Source Edition: Sequential execution.
+ Run all mutations with progress display (sequential execution).
"""
- # Open Source: Force sequential execution
- concurrency = (
- 1 if not PARALLEL_EXECUTION_ENABLED else self.config.advanced.concurrency
- )
- semaphore = asyncio.Semaphore(concurrency)
+ # Sequential execution only
+ semaphore = asyncio.Semaphore(1)
results: list[MutationResult] = []
- # Sequential execution for Open Source
- if not PARALLEL_EXECUTION_ENABLED:
- for original, mutation in mutations:
- result = await self._run_single_mutation(original, mutation, semaphore)
- progress.update(task_id, advance=1)
- results.append(result)
- return results
-
- # Parallel execution (Cloud only)
- async def run_with_progress(
- original: str,
- mutation: Mutation,
- ) -> MutationResult:
+ for original, mutation in mutations:
result = await self._run_single_mutation(original, mutation, semaphore)
progress.update(task_id, advance=1)
- return result
-
- tasks = [
- run_with_progress(original, mutation) for original, mutation in mutations
- ]
-
- results = await asyncio.gather(*tasks)
+ results.append(result)
return results
async def _run_single_mutation(
diff --git a/src/flakestorm/integrations/__init__.py b/src/flakestorm/integrations/__init__.py
index 15cf6f3..ede209f 100644
--- a/src/flakestorm/integrations/__init__.py
+++ b/src/flakestorm/integrations/__init__.py
@@ -1,17 +1,15 @@
"""
flakestorm Integrations Module
-V2 features for integrating with external services:
+Features for integrating with external services:
- HuggingFace model downloading
-- GitHub Actions for CI/CD
- Local embeddings for semantic similarity
"""
-# V2 features - import guards for optional dependencies
+# Import guards for optional dependencies
__all__ = [
"HuggingFaceModelProvider",
- "GitHubActionsIntegration",
"LocalEmbedder",
]
@@ -22,10 +20,6 @@ def __getattr__(name: str):
from flakestorm.integrations.huggingface import HuggingFaceModelProvider
return HuggingFaceModelProvider
- elif name == "GitHubActionsIntegration":
- from flakestorm.integrations.github_actions import GitHubActionsIntegration
-
- return GitHubActionsIntegration
elif name == "LocalEmbedder":
from flakestorm.assertions.semantic import LocalEmbedder
diff --git a/src/flakestorm/integrations/github_actions.py b/src/flakestorm/integrations/github_actions.py
deleted file mode 100644
index fff5db4..0000000
--- a/src/flakestorm/integrations/github_actions.py
+++ /dev/null
@@ -1,255 +0,0 @@
-"""
-GitHub Actions Integration
-
-⚠️ CLOUD FEATURE: GitHub Actions integration is available in flakestorm Cloud.
-The Open Source edition provides documentation only.
-
-Upgrade to flakestorm Cloud for:
-- One-click CI/CD integration
-- Block PRs based on reliability score
-- Automated test history tracking
-- Team notifications
-
-→ https://flakestorm.cloud
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-
-from flakestorm.core.limits import CLOUD_URL, GITHUB_ACTIONS_ENABLED
-
-
-class GitHubActionsDisabledError(Exception):
- """Raised when trying to use GitHub Actions in Open Source edition."""
-
- def __init__(self):
- super().__init__(
- "GitHub Actions integration is available in flakestorm Cloud.\n"
- f"Upgrade at: {CLOUD_URL}"
- )
-
-
-# GitHub Action YAML template (for reference/documentation)
-ACTION_YAML = """# ⚠️ CLOUD FEATURE: This requires flakestorm Cloud
-# Upgrade at: https://flakestorm.cloud
-
-name: 'flakestorm Agent Test'
-description: 'Run chaos testing on AI agents to verify reliability'
-author: 'flakestorm'
-
-branding:
- icon: 'shield'
- color: 'purple'
-
-inputs:
- config:
- description: 'Path to flakestorm.yaml configuration file'
- required: false
- default: 'flakestorm.yaml'
- min_score:
- description: 'Minimum robustness score to pass (0.0-1.0)'
- required: false
- default: '0.9'
- api_key:
- description: 'flakestorm Cloud API key (required)'
- required: true
-
-outputs:
- score:
- description: 'The robustness score achieved'
- passed:
- description: 'Whether the test passed (true/false)'
- report_url:
- description: 'URL to the full report on flakestorm Cloud'
-
-runs:
- using: 'composite'
- steps:
- - name: Setup Python
- uses: actions/setup-python@v5
- with:
- python-version: '3.11'
-
- - name: Install flakestorm
- shell: bash
- run: pip install flakestorm
-
- - name: Run Cloud Tests
- shell: bash
- env:
- FLAKESTORM_API_KEY: ${{ inputs.api_key }}
- run: |
- flakestorm cloud run \\
- --config ${{ inputs.config }} \\
- --min-score ${{ inputs.min_score }} \\
- --ci
-"""
-
-
-# Example workflow YAML
-WORKFLOW_EXAMPLE = """# flakestorm Cloud CI/CD Integration
-# ⚠️ Requires flakestorm Cloud subscription
-# Get started: https://flakestorm.cloud
-
-name: Agent Reliability Check
-
-on:
- push:
- branches: [main]
- pull_request:
- branches: [main]
-
-jobs:
- reliability-test:
- runs-on: ubuntu-latest
-
- steps:
- - uses: actions/checkout@v4
-
- - name: Run flakestorm Cloud Tests
- uses: flakestorm/flakestorm-action@v1
- with:
- config: flakestorm.yaml
- min_score: '0.9'
- api_key: ${{ secrets.FLAKESTORM_API_KEY }}
-"""
-
-
-class GitHubActionsIntegration:
- """
- Helper class for GitHub Actions integration.
-
- ⚠️ NOTE: Full CI/CD integration requires flakestorm Cloud.
-
- The Open Source edition provides:
- - Documentation and examples
- - Local testing only
-
- flakestorm Cloud provides:
- - One-click GitHub Actions setup
- - Block PRs based on reliability score
- - Test history and comparison
- - Slack/Discord notifications
-
- Upgrade at: https://flakestorm.cloud
- """
-
- @staticmethod
- def _check_enabled() -> None:
- """Check if GitHub Actions is enabled."""
- if not GITHUB_ACTIONS_ENABLED:
- raise GitHubActionsDisabledError()
-
- @staticmethod
- def generate_action_yaml() -> str:
- """
- Generate the GitHub Action definition YAML.
-
- Note: This returns documentation only in Open Source edition.
- Full integration requires flakestorm Cloud.
-
- Returns:
- Action YAML content
- """
- return ACTION_YAML.strip()
-
- @staticmethod
- def generate_workflow_example() -> str:
- """
- Generate an example workflow that uses flakestorm.
-
- Note: Requires flakestorm Cloud for full functionality.
-
- Returns:
- Workflow YAML content
- """
- return WORKFLOW_EXAMPLE.strip()
-
- @staticmethod
- def save_action(output_dir: Path) -> Path:
- """
- Save the GitHub Action files to a directory.
-
- ⚠️ Cloud Feature: This creates documentation only.
- For working CI/CD, upgrade to flakestorm Cloud.
-
- Args:
- output_dir: Directory to save action files
-
- Returns:
- Path to the action.yml file
- """
- output_dir = Path(output_dir)
- output_dir.mkdir(parents=True, exist_ok=True)
-
- action_path = output_dir / "action.yml"
- action_path.write_text(ACTION_YAML.strip(), encoding="utf-8")
-
- # Also create a README explaining Cloud requirement
- readme_path = output_dir / "README.md"
- readme_path.write_text(
- f"""# flakestorm GitHub Action
-
-⚠️ **Cloud Feature**: Full CI/CD integration requires flakestorm Cloud.
-
-## What You Get with Cloud
-
-- ✅ One-click GitHub Actions setup
-- ✅ Block PRs based on reliability score
-- ✅ Test history and comparison across runs
-- ✅ Slack/Discord notifications
-- ✅ 20x faster parallel execution
-
-## Upgrade
-
-Get started at: {CLOUD_URL}
-
-## Local Testing
-
-For local-only testing, use the Open Source CLI:
-
-```bash
-flakestorm run --config flakestorm.yaml
-```
-
-Note: Local runs are sequential and may be slow for large test suites.
-""",
- encoding="utf-8",
- )
-
- return action_path
-
- @staticmethod
- def save_workflow_example(output_path: Path) -> Path:
- """
- Save an example workflow file.
-
- Args:
- output_path: Path to save the workflow file
-
- Returns:
- Path to the saved file
- """
- output_path = Path(output_path)
- output_path.parent.mkdir(parents=True, exist_ok=True)
- output_path.write_text(WORKFLOW_EXAMPLE.strip(), encoding="utf-8")
-
- return output_path
-
- @staticmethod
- def setup_ci(
- repo_path: Path,
- config_path: str = "flakestorm.yaml",
- min_score: float = 0.9,
- ) -> None:
- """
- Set up CI/CD integration for a repository.
-
- ⚠️ Cloud Feature: Requires flakestorm Cloud subscription.
-
- Raises:
- GitHubActionsDisabledError: Always in Open Source edition
- """
- GitHubActionsIntegration._check_enabled()
- # Cloud implementation would go here
diff --git a/src/flakestorm/integrations/huggingface.py b/src/flakestorm/integrations/huggingface.py
index c19a03d..59df2fd 100644
--- a/src/flakestorm/integrations/huggingface.py
+++ b/src/flakestorm/integrations/huggingface.py
@@ -154,7 +154,7 @@ class HuggingFaceModelProvider:
>>> model_name = provider.import_to_ollama(path, "mistral-attacker")
>>> # Now use with: ollama run mistral-attacker
"""
- import subprocess
+ import subprocess # nosec B404
import tempfile
model_path = Path(model_path)
@@ -196,7 +196,7 @@ SYSTEM You are a helpful assistant that generates text variations.
try:
# Run ollama create command
- result = subprocess.run(
+ result = subprocess.run( # nosec B603, B607
["ollama", "create", model_name, "-f", modelfile_path],
capture_output=True,
text=True,
@@ -270,7 +270,7 @@ SYSTEM You are a helpful assistant that generates text variations.
try:
req = urllib.request.Request(f"{host}/api/version")
- with urllib.request.urlopen(req, timeout=5) as response:
+ with urllib.request.urlopen(req, timeout=5) as response: # nosec B310
return response.status == 200
except (urllib.error.URLError, TimeoutError):
return False
@@ -297,7 +297,7 @@ SYSTEM You are a helpful assistant that generates text variations.
try:
req = urllib.request.Request(f"{host}/api/tags")
- with urllib.request.urlopen(req, timeout=10) as response:
+ with urllib.request.urlopen(req, timeout=10) as response: # nosec B310
data = json.loads(response.read().decode())
return [model["name"] for model in data.get("models", [])]
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
diff --git a/src/flakestorm/mutations/types.py b/src/flakestorm/mutations/types.py
index 55f1e14..ff372b0 100644
--- a/src/flakestorm/mutations/types.py
+++ b/src/flakestorm/mutations/types.py
@@ -16,15 +16,12 @@ class MutationType(str, Enum):
"""
Types of adversarial mutations.
- Open Source Edition includes 5 mutation types:
+ Includes 5 mutation types:
- PARAPHRASE: Semantic rewrites
- NOISE: Typos and spelling errors
- TONE_SHIFT: Tone changes
- PROMPT_INJECTION: Basic adversarial attacks
- CUSTOM: User-defined mutation templates
-
- Advanced mutations (sophisticated prompt injections, jailbreaks)
- are available in flakestorm Cloud.
"""
PARAPHRASE = "paraphrase"