release 0.4.29 (#998)

Co-authored-by: Cursor <cursoragent@cursor.com>

.github/workflows/ci.yml

@@ -117,7 +117,7 @@ jobs:
       # repo means find_repo_root() returns None, the local-build short-
       # circuit is skipped, and the CLI tries to download from a GitHub
       # release that does not yet exist for the in-flight version on
-      # release-bump PRs (e.g. 0.4.23 before publish-binaries has run).
+      # release-bump PRs (e.g. 0.4.29 before publish-binaries has run).
       - name: Seed ~/.plano cache for zero-config test
         run: |
           VERSION=$(sed -nE 's/^__version__ = "(.*)"$/\1/p' cli/planoai/__init__.py)
@@ -183,13 +183,13 @@ jobs:
           load: true
           tags: |
             ${{ env.PLANO_DOCKER_IMAGE }}
-            ${{ env.DOCKER_IMAGE }}:0.4.23
+            ${{ env.DOCKER_IMAGE }}:0.4.29
             ${{ env.DOCKER_IMAGE }}:latest
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
       - name: Save image as artifact
-        run: docker save ${{ env.PLANO_DOCKER_IMAGE }} ${{ env.DOCKER_IMAGE }}:0.4.23 ${{ env.DOCKER_IMAGE }}:latest -o /tmp/plano-image.tar
+        run: docker save ${{ env.PLANO_DOCKER_IMAGE }} ${{ env.DOCKER_IMAGE }}:0.4.29 ${{ env.DOCKER_IMAGE }}:latest -o /tmp/plano-image.tar
 
       - name: Upload image artifact
         uses: actions/upload-artifact@v6

.github/workflows/update-providers.yml

@@ -57,6 +57,7 @@ jobs:
           GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
           AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+          META_MODELS_API_KEY: ${{ secrets.META_MODELS_API_KEY }}
         run: cargo run --bin fetch_models --features model-fetch
 
       - name: Create pull request

Dockerfile

@@ -47,10 +47,13 @@ FROM docker.io/envoyproxy/envoy:${ENVOY_VERSION} AS envoy
 
 FROM python:3.14-slim AS arch
 
+# Install runtime deps first, then upgrade — so security patches also land on
+# packages pulled in transitively by the install, not just those already present
+# in the base image.
 RUN set -eux; \
   apt-get update; \
+  apt-get install -y --no-install-recommends gettext-base procps; \
   apt-get upgrade -y; \
-  apt-get install -y --no-install-recommends gettext-base curl procps; \
   apt-get clean; rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir supervisor

apps/www/src/components/Hero.tsx

@@ -24,7 +24,7 @@ export function Hero() {
             >
               <div className="inline-flex flex-wrap items-center gap-1.5 sm:gap-2 px-3 sm:px-4 py-1 rounded-full bg-[rgba(185,191,255,0.4)] border border-[var(--secondary)] shadow backdrop-blur hover:bg-[rgba(185,191,255,0.6)] transition-colors cursor-pointer">
                 <span className="text-xs sm:text-sm font-medium text-black/65">
-                  v0.4.23
+                  v0.4.29
                 </span>
                 <span className="text-xs sm:text-sm font-medium text-black ">
                   —

build_filter_image.sh

@@ -1 +1 @@
-docker build  -f Dockerfile . -t katanemo/plano -t katanemo/plano:0.4.23
+docker build  -f Dockerfile . -t katanemo/plano -t katanemo/plano:0.4.29

cli/planoai/__init__.py

@@ -1,3 +1,3 @@
 """Plano CLI - Intelligent Prompt Gateway."""
 
-__version__ = "0.4.23"
+__version__ = "0.4.29"

cli/planoai/consts.py

@@ -5,7 +5,7 @@ PLANO_COLOR = "#969FF4"
 
 SERVICE_NAME_ARCHGW = "plano"
 PLANO_DOCKER_NAME = "plano"
-PLANO_DOCKER_IMAGE = os.getenv("PLANO_DOCKER_IMAGE", "katanemo/plano:0.4.23")
+PLANO_DOCKER_IMAGE = os.getenv("PLANO_DOCKER_IMAGE", "katanemo/plano:0.4.29")
 DEFAULT_OTEL_TRACING_GRPC_ENDPOINT = "http://localhost:4317"
 
 # Native mode constants

cli/planoai/obs/pricing.py

@@ -1,7 +1,8 @@
-"""DigitalOcean Gradient pricing catalog for the obs console.
+"""Model pricing catalog for the obs console.
 
-Ported loosely from ``crates/brightstaff/src/router/model_metrics.rs::fetch_do_pricing``.
-Single-source: one fetch at startup, cached for the life of the process.
+Mirrors ``crates/brightstaff/src/router/model_metrics.rs``. The source is
+configurable: ``digitalocean`` (DO GenAI catalog) or ``models.dev``. A single
+fetch at startup is cached for the life of the process.
 """
 
 from __future__ import annotations
@@ -14,7 +15,18 @@ from typing import Any
 
 import requests
 
-DEFAULT_PRICING_URL = "https://api.digitalocean.com/v2/gen-ai/models/catalog"
+DO_PRICING_URL = "https://api.digitalocean.com/v2/gen-ai/models/catalog"
+MODELS_DEV_URL = "https://models.dev/api.json"
+
+# Backwards-compatible default (DigitalOcean) used when no provider is given.
+DEFAULT_PRICING_URL = DO_PRICING_URL
+DEFAULT_PRICING_PROVIDER = "digitalocean"
+
+_DEFAULT_URLS = {
+    "digitalocean": DO_PRICING_URL,
+    "models.dev": MODELS_DEV_URL,
+}
+
 FETCH_TIMEOUT_SECS = 5.0
 
 
@@ -51,36 +63,52 @@ class PricingCatalog:
             return list(self._prices.keys())[:n]
 
     @classmethod
-    def fetch(cls, url: str = DEFAULT_PRICING_URL) -> "PricingCatalog":
-        """Fetch pricing from DO's catalog endpoint. On failure, returns an
+    def fetch(
+        cls,
+        provider: str = DEFAULT_PRICING_PROVIDER,
+        url: str | None = None,
+    ) -> "PricingCatalog":
+        """Fetch pricing from the configured catalog. On failure, returns an
         empty catalog (cost column will be blank).
 
-        The catalog endpoint is public — no auth required, no signup — so
-        ``planoai obs`` gets cost data on first run out of the box.
+        ``provider`` selects the parser/default URL: ``digitalocean`` or
+        ``models.dev``. Both catalog endpoints are public — no auth required —
+        so ``planoai obs`` gets cost data on first run out of the box.
         """
+        provider = (provider or DEFAULT_PRICING_PROVIDER).strip().lower()
+        resolved_url = url or _DEFAULT_URLS.get(provider, DO_PRICING_URL)
         try:
-            resp = requests.get(url, timeout=FETCH_TIMEOUT_SECS)
+            resp = requests.get(resolved_url, timeout=FETCH_TIMEOUT_SECS)
             resp.raise_for_status()
             data = resp.json()
         except Exception as exc:  # noqa: BLE001 — best-effort; never fatal
             logger.warning(
-                "DO pricing fetch failed: %s; cost column will be blank.",
+                "%s pricing fetch failed: %s; cost column will be blank.",
+                provider,
                 exc,
             )
             return cls()
 
+        if provider == "models.dev":
+            prices = _parse_models_dev_pricing(data)
+        else:
             prices = _parse_do_pricing(data)
+
         if not prices:
-            # Dump the first entry's raw shape so we can see which fields DO
-            # actually returned — helps when the catalog adds new fields or
-            # the response doesn't match our parser.
+            # Dump a sample of the raw shape so we can see which fields the
+            # catalog returned — helps when it adds new fields or the response
+            # doesn't match our parser.
             import json as _json
 
+            if provider == "models.dev" and isinstance(data, dict):
+                sample = next(iter(data.values()), data)
+            else:
                 sample_items = _coerce_items(data)
                 sample = sample_items[0] if sample_items else data
             logger.warning(
-                "DO pricing response had no parseable entries; cost column "
+                "%s pricing response had no parseable entries; cost column "
                 "will be blank. Sample entry: %s",
+                provider,
                 _json.dumps(sample, default=str)[:400],
             )
         return cls(prices)
@@ -278,6 +306,75 @@ def _parse_do_pricing(data: Any) -> dict[str, ModelPrice]:
     return prices
 
 
+def _parse_models_dev_pricing(data: Any) -> dict[str, ModelPrice]:
+    """Parse a models.dev ``api.json`` response into a ModelPrice map.
+
+    models.dev shape (top-level object keyed by provider id)::
+
+        {
+          "anthropic": {
+            "models": {
+              "claude-opus-4-5": {
+                "cost": {"input": 5, "output": 25, "cache_read": 0.5}
+              }
+            }
+          },
+          ...
+        }
+
+    ``cost.*`` values are USD per *million* tokens, so we divide by 1e6 to get a
+    per-token rate. First-party providers use bare model keys, so we register
+    both ``provider/model`` (matching Plano's routing names) and the bare model
+    id as a fallback.
+    """
+    prices: dict[str, ModelPrice] = {}
+    if not isinstance(data, dict):
+        return prices
+
+    for provider_id, provider in data.items():
+        if not isinstance(provider, dict):
+            continue
+        models = provider.get("models")
+        if not isinstance(models, dict):
+            continue
+        for model_key, model in models.items():
+            if not isinstance(model, dict):
+                continue
+            cost = model.get("cost")
+            if not isinstance(cost, dict):
+                continue
+            input_pm = _as_float(cost.get("input"))
+            output_pm = _as_float(cost.get("output"))
+            if input_pm is None or output_pm is None:
+                continue
+            # Skip 0-rate entries so cost falls back to `—` rather than $0.0000.
+            if input_pm == 0 and output_pm == 0:
+                continue
+            cached_pm = _as_float(cost.get("cache_read"))
+            price = ModelPrice(
+                input_per_token_usd=input_pm / 1_000_000,
+                output_per_token_usd=output_pm / 1_000_000,
+                cached_input_per_token_usd=(
+                    cached_pm / 1_000_000 if cached_pm is not None else None
+                ),
+            )
+            composite = f"{provider_id}/{model_key}"
+            prices[composite] = price
+            prices.setdefault(composite.lower(), price)
+            prices.setdefault(str(model_key), price)
+            prices.setdefault(str(model_key).lower(), price)
+    return prices
+
+
+def _as_float(value: Any) -> float | None:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
 def _coerce_items(data: Any) -> list[dict]:
     if isinstance(data, list):
         return [x for x in data if isinstance(x, dict)]

cli/planoai/obs_cmd.py

@@ -2,9 +2,12 @@
 
 from __future__ import annotations
 
+import logging
+import os
 import time
 
 import rich_click as click
+import yaml
 from rich.console import Console
 from rich.live import Live
 
@@ -15,8 +18,50 @@ from planoai.obs.collector import (
     LLMCallStore,
     ObsCollector,
 )
-from planoai.obs.pricing import PricingCatalog
+from planoai.obs.pricing import DEFAULT_PRICING_PROVIDER, PricingCatalog
 from planoai.obs.render import render
+from planoai.utils import find_config_file
+
+logger = logging.getLogger(__name__)
+
+
+def _resolve_pricing_source(
+    config_file: str | None,
+    provider_override: str | None,
+    url_override: str | None,
+) -> tuple[str, str | None]:
+    """Pick the cost pricing source.
+
+    Precedence: explicit CLI overrides > the first ``type: cost`` entry in
+    ``model_metrics_sources`` from the Plano config > the DigitalOcean default.
+    """
+    provider = DEFAULT_PRICING_PROVIDER
+    url: str | None = None
+
+    config_path = find_config_file(file=config_file)
+    if config_path and os.path.exists(config_path):
+        try:
+            with open(config_path, "r") as f:
+                config = yaml.safe_load(f) or {}
+            sources = config.get("model_metrics_sources") or []
+            for source in sources:
+                if isinstance(source, dict) and source.get("type") == "cost":
+                    if source.get("provider"):
+                        provider = str(source["provider"])
+                    if source.get("url"):
+                        url = str(source["url"])
+                    break
+        except Exception as exc:  # noqa: BLE001 — config is optional for obs
+            logger.warning(
+                "could not read pricing source from %s: %s", config_path, exc
+            )
+
+    if provider_override:
+        provider = provider_override
+    if url_override:
+        url = url_override
+
+    return provider, url
 
 
 @click.command(name="obs", help="Live observability console for Plano LLM traffic.")
@@ -48,13 +93,42 @@ from planoai.obs.render import render
     show_default=True,
     help="TUI refresh interval.",
 )
-def obs(port: int, host: str, capacity: int, refresh_ms: int) -> None:
+@click.option(
+    "--config",
+    "config_file",
+    type=str,
+    default=None,
+    help="Path to the Plano config to read the pricing source from "
+    "(defaults to ./config.yaml or ./plano_config.yaml).",
+)
+@click.option(
+    "--pricing-provider",
+    type=click.Choice(["digitalocean", "models.dev"]),
+    default=None,
+    help="Override the cost pricing provider (otherwise read from config).",
+)
+@click.option(
+    "--pricing-url",
+    type=str,
+    default=None,
+    help="Override the pricing catalog URL (otherwise read from config / provider default).",
+)
+def obs(
+    port: int,
+    host: str,
+    capacity: int,
+    refresh_ms: int,
+    config_file: str | None,
+    pricing_provider: str | None,
+    pricing_url: str | None,
+) -> None:
     console = Console()
+    provider, url = _resolve_pricing_source(config_file, pricing_provider, pricing_url)
     console.print(
-        f"[bold {PLANO_COLOR}]planoai obs[/] — loading DO pricing catalog...",
+        f"[bold {PLANO_COLOR}]planoai obs[/] — loading {provider} pricing catalog...",
         end="",
     )
-    pricing = PricingCatalog.fetch()
+    pricing = PricingCatalog.fetch(provider=provider, url=url)
     if len(pricing):
         sample = ", ".join(pricing.sample_models(3))
         console.print(
@@ -63,7 +137,7 @@ def obs(port: int, host: str, capacity: int, refresh_ms: int) -> None:
     else:
         console.print(
             " [yellow]no pricing loaded[/] — "
-            "[dim]cost column will be blank (DO catalog unreachable)[/]"
+            f"[dim]cost column will be blank ({provider} catalog unreachable)[/]"
         )
 
     store = LLMCallStore(capacity=capacity)

cli/planoai/templates/conversational_state_v1_responses.yaml

@@ -11,7 +11,7 @@ model_providers:
     default: true
 
   # Anthropic Models
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 listeners:

cli/planoai/templates/preference_aware_routing.yaml

@@ -12,7 +12,7 @@ model_providers:
       - name: code understanding
         description: understand and explain existing code snippets, functions, or libraries
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     routing_preferences:
       - name: code generation

cli/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "planoai"
-version = "0.4.23"
+version = "0.4.29"
 description = "Python-based CLI tool to manage Plano."
 authors = [{name = "Katanemo Labs, Inc."}]
 readme = "README.md"

cli/test/test_config_generator.py

@@ -386,6 +386,33 @@ model_providers:
     access_key: $OPENAI_API_KEY
     default: true
 
+""",
+    },
+    {
+        "id": "valid_tracing_posthog_exporter",
+        "expected_error": None,
+        "plano_config": """
+version: v0.4.0
+
+listeners:
+  - name: llm
+    type: model
+    port: 12000
+
+model_providers:
+  - model: openai/gpt-4o-mini
+    access_key: $OPENAI_API_KEY
+    default: true
+
+tracing:
+  random_sampling: 100
+  exporters:
+    - type: posthog
+      url: https://us.i.posthog.com
+      api_key: $POSTHOG_API_KEY
+      distinct_id_header: x-user-id
+      capture_messages: false
+
 """,
     },
 ]
@@ -584,7 +611,7 @@ model_providers:
       - name: code understanding
         description: understand and explain existing code snippets, functions, or libraries
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     routing_preferences:
       - name: code generation
@@ -601,9 +628,7 @@ model_providers:
     by_name = {entry["name"]: entry for entry in top_level}
     assert set(by_name) == {"code understanding", "code generation"}
     assert by_name["code understanding"]["models"] == ["openai/gpt-4o"]
-    assert by_name["code generation"]["models"] == [
-        "anthropic/claude-sonnet-4-20250514"
-    ]
+    assert by_name["code generation"]["models"] == ["anthropic/claude-sonnet-4-6"]
     assert (
         by_name["code understanding"]["description"]
         == "understand and explain existing code snippets, functions, or libraries"
@@ -626,7 +651,7 @@ model_providers:
       - name: code generation
         description: generating new code snippets, functions, or boilerplate based on user prompts or requirements
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     routing_preferences:
       - name: code generation
@@ -641,7 +666,7 @@ model_providers:
     assert entry["name"] == "code generation"
     assert entry["models"] == [
         "openai/gpt-4o",
-        "anthropic/claude-sonnet-4-20250514",
+        "anthropic/claude-sonnet-4-6",
     ]
     assert config_yaml["version"] == "v0.4.0"
 
@@ -658,7 +683,7 @@ listeners:
 model_providers:
   - model: openai/gpt-4o
     access_key: $OPENAI_API_KEY
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 routing_preferences:
@@ -666,7 +691,7 @@ routing_preferences:
     description: generating new code snippets or boilerplate
     models:
       - openai/gpt-4o
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
 """
     config_yaml = yaml.safe_load(plano_config)
     before = yaml.safe_dump(config_yaml, sort_keys=True)

cli/test/test_obs_pricing.py

@@ -144,3 +144,68 @@ def test_parse_do_catalog_divides_large_values_as_per_million():
     prices = _parse_do_pricing(sample)
     assert prices["mystery-model"].input_per_token_usd == 5.0 / 1_000_000
     assert prices["mystery-model"].output_per_token_usd == 15.0 / 1_000_000
+
+
+_MODELS_DEV_SAMPLE = {
+    "anthropic": {
+        "id": "anthropic",
+        "models": {
+            "claude-opus-4-5": {
+                "id": "claude-opus-4-5",
+                "cost": {"input": 5, "output": 25, "cache_read": 0.5},
+            }
+        },
+    },
+    "groq": {
+        "id": "groq",
+        "models": {
+            "llama-3.3-70b-versatile": {
+                "id": "llama-3.3-70b-versatile",
+                "cost": {"input": 0.59, "output": 0.79},
+            },
+            # No cost block → skipped.
+            "whisper-large-v3-turbo": {"id": "whisper-large-v3-turbo"},
+        },
+    },
+}
+
+
+def test_parse_models_dev_composes_provider_keys_and_per_token_rates():
+    from planoai.obs.pricing import _parse_models_dev_pricing
+
+    prices = _parse_models_dev_pricing(_MODELS_DEV_SAMPLE)
+
+    # models.dev cost values are per-million → divided by 1e6.
+    opus = prices["anthropic/claude-opus-4-5"]
+    assert opus.input_per_token_usd == 5 / 1_000_000
+    assert opus.output_per_token_usd == 25 / 1_000_000
+    assert opus.cached_input_per_token_usd == 0.5 / 1_000_000
+
+    # Composite provider/model keys match Plano's routing names.
+    assert "groq/llama-3.3-70b-versatile" in prices
+    # Bare model id registered as a fallback.
+    assert "llama-3.3-70b-versatile" in prices
+    # Models without a cost block are skipped.
+    assert "groq/whisper-large-v3-turbo" not in prices
+
+
+def test_models_dev_catalog_cost_computation():
+    from planoai.obs.pricing import PricingCatalog, _parse_models_dev_pricing
+
+    catalog = PricingCatalog(_parse_models_dev_pricing(_MODELS_DEV_SAMPLE))
+    # 1000 input @ 5e-6 = 0.005; 500 output @ 25e-6 = 0.0125
+    cost = catalog.cost_for_call(_call("anthropic/claude-opus-4-5", 1000, 500))
+    assert cost == round(0.005 + 0.0125, 6)
+
+
+def test_models_dev_skips_zero_rate_entries():
+    from planoai.obs.pricing import _parse_models_dev_pricing
+
+    sample = {
+        "free": {
+            "models": {
+                "promo-model": {"cost": {"input": 0, "output": 0}},
+            }
+        }
+    }
+    assert _parse_models_dev_pricing(sample) == {}

cli/uv.lock

@@ -110,7 +110,7 @@ name = "exceptiongroup"
 version = "1.3.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+    { name = "typing-extensions" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
 wheels = [
@@ -337,7 +337,7 @@ wheels = [
 
 [[package]]
 name = "planoai"
-version = "0.4.23"
+version = "0.4.29"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

config/envoy.template.yaml

@@ -196,7 +196,11 @@ static_resources:
                         name: decompress
                         typed_config:
                           "@type": "type.googleapis.com/envoy.extensions.compression.gzip.decompressor.v3.Gzip"
-                          window_bits: 9
+                          # Must be >= the window_bits of whatever compressed the response
+                          # (upstreams use 15; our compressor uses 10). A smaller inflate window
+                          # makes zlib silently stop emitting data mid-stream, truncating SSE
+                          # responses after the first few KB. 15 handles any compliant gzip.
+                          window_bits: 15
                           chunk_size: 8192
                           # If this ratio is set too low, then body data will not be decompressed completely.
                           max_inflate_ratio: 1000
@@ -269,6 +273,13 @@ static_resources:
                             auto_host_rewrite: true
                             cluster: {{ cluster_name }}
                             timeout: 300s
+                            {% if cluster.prefix_affinity %}
+                            # Feeds the cluster's RING_HASH lb_policy: requests with the
+                            # same prompt-prefix hash stick to the same replica.
+                            hash_policy:
+                              - header:
+                                  header_name: x-plano-prefix-hash
+                            {% endif %}
                         {% endfor %}
                 http_filters:
                   - name: envoy.filters.http.router
@@ -361,7 +372,11 @@ static_resources:
                         name: decompress
                         typed_config:
                           "@type": "type.googleapis.com/envoy.extensions.compression.gzip.decompressor.v3.Gzip"
-                          window_bits: 9
+                          # Must be >= the window_bits of whatever compressed the response
+                          # (upstreams use 15; our compressor uses 10). A smaller inflate window
+                          # makes zlib silently stop emitting data mid-stream, truncating SSE
+                          # responses after the first few KB. 15 handles any compliant gzip.
+                          window_bits: 15
                           chunk_size: 8192
                           # If this ratio is set too low, then body data will not be decompressed completely.
                           max_inflate_ratio: 1000
@@ -442,7 +457,11 @@ static_resources:
                         name: decompress
                         typed_config:
                           "@type": "type.googleapis.com/envoy.extensions.compression.gzip.decompressor.v3.Gzip"
-                          window_bits: 9
+                          # Must be >= the window_bits of whatever compressed the response
+                          # (upstreams use 15; our compressor uses 10). A smaller inflate window
+                          # makes zlib silently stop emitting data mid-stream, truncating SSE
+                          # responses after the first few KB. 15 handles any compliant gzip.
+                          window_bits: 15
                           chunk_size: 8192
                           # If this ratio is set too low, then body data will not be decompressed completely.
                           max_inflate_ratio: 1000
@@ -577,6 +596,12 @@ static_resources:
                         name: decompress
                         typed_config:
                           "@type": "type.googleapis.com/envoy.extensions.compression.gzip.decompressor.v3.Gzip"
+                          # Must be >= the window_bits of whatever compressed the response
+                          # (upstreams use 15; our compressor uses 10; Envoy's default here is
+                          # only 12). A smaller inflate window makes zlib silently stop emitting
+                          # data mid-stream, truncating SSE responses after the first few KB.
+                          # 15 handles any compliant gzip.
+                          window_bits: 15
                           chunk_size: 8192
                           # If this ratio is set too low, then body data will not be decompressed completely.
                           max_inflate_ratio: 1000
@@ -977,9 +1002,18 @@ static_resources:
       {% else -%}
       connect_timeout: {{ upstream_connect_timeout | default('5s') }}
       {% endif -%}
+      {% if cluster.prefix_affinity -%}
+      # KV-aware replica stickiness: resolve every replica address and consistent-hash
+      # requests (by x-plano-prefix-hash, see the route-level hash_policy) so the same
+      # prompt prefix lands on the replica holding its warm KV cache.
+      type: STRICT_DNS
+      dns_lookup_family: V4_ONLY
+      lb_policy: RING_HASH
+      {% else -%}
       type: LOGICAL_DNS
       dns_lookup_family: V4_ONLY
       lb_policy: ROUND_ROBIN
+      {% endif -%}
       load_assignment:
         cluster_name: {{ cluster_name }}
         endpoints:

config/plano_config_schema.yaml

@@ -155,6 +155,13 @@ properties:
               - https
           http_host:
             type: string
+          prefix_affinity:
+            type: boolean
+            description: >
+              For self-hosted multi-replica backends (e.g. vLLM): consistent-hash requests
+              to replicas by the x-plano-prefix-hash header so the same prompt prefix lands
+              on the same replica and reuses its KV cache. Uses ring-hash load balancing
+              across all resolved endpoint addresses. Default false.
         additionalProperties: false
         required:
           - endpoint
@@ -316,6 +323,33 @@ properties:
       orchestrator_model_context_length:
         type: integer
         description: "Maximum token length for the orchestrator/routing model context window. Default is 8192."
+  prompt_caching:
+    type: object
+    description: >
+      Automatic provider prompt caching, configured once for the whole Plano instance.
+      Disabled by default; set enabled: true to opt in. Prompt caching never changes
+      which model routing selects — it only keeps a conversation on the same
+      model/provider so the upstream prompt cache stays warm across turns, and
+      auto-injects provider cache-control markers where supported.
+    properties:
+      enabled:
+        type: boolean
+        description: "Master switch. Default false (opt-in). Applies across the entire instance."
+      session_affinity:
+        type: boolean
+        description: "Auto-derive a session key from the prompt prefix (system + tools + first user message) when X-Model-Affinity is absent, so follow-up turns reuse the same warm cache. Default true when enabled."
+      inject_cache_control:
+        type: boolean
+        description: "Auto-insert ephemeral cache_control breakpoints for providers that need explicit markers (Anthropic). Default true when enabled."
+      min_prefix_tokens:
+        type: integer
+        minimum: 1
+        description: "Skip breakpoint injection when the estimated stable prefix is below this many tokens. Default 1024."
+      session_ttl_seconds:
+        type: integer
+        minimum: 1
+        description: "Pin lifetime for implicit/explicit sessions; align with the provider cache window (e.g. 3600 for Anthropic 1h caching). Defaults to routing.session_ttl_seconds."
+    additionalProperties: false
   system_prompt:
     type: string
   prompt_targets:
@@ -447,6 +481,28 @@ properties:
             additionalProperties:
               type: string
         additionalProperties: false
+      exporters:
+        type: array
+        items:
+          oneOf:
+            - type: object
+              properties:
+                type:
+                  type: string
+                  const: posthog
+                url:
+                  type: string
+                api_key:
+                  type: string
+                distinct_id_header:
+                  type: string
+                capture_messages:
+                  type: boolean
+              additionalProperties: false
+              required:
+                - type
+                - url
+                - api_key
     additionalProperties: false
   mode:
     type: string
@@ -488,6 +544,53 @@ properties:
               Optional HTTP header name whose value is used as a tenant prefix in the cache key.
               When set, keys are scoped as plano:affinity:{tenant_id}:{session_id}.
         additionalProperties: false
+      routing_budget:
+        type: object
+        description: >
+          Per-session cost gate on model switching. Self-sufficient and independent of
+          prompt_caching — it applies whenever configured (presence of this block turns
+          it on), derives implicit sessions on its own, and always prices warm anchors
+          at their cached input rate (provider caches are assumed real). The default
+          posture is to stick to the model a session is warm on. When routing proposes a
+          different model while that session's provider cache is plausibly still warm
+          (inferred from the idle gap vs. the provider's cache window), the actual
+          input-token cost of abandoning the cache —
+          context_tokens x (candidate_uncached_input_rate - anchor_cached_input_rate),
+          output cost deliberately excluded — accrues into the session's cumulative
+          switch spend. A paid switch is allowed only while that spend stays within
+          max_overhead_pct percent of the session's running never-switch baseline (what
+          staying on the anchor would have cost). An outright-cheaper switch is free but
+          never reduces the spend. Requires a cost source in model_metrics_sources.
+        properties:
+          max_overhead_pct:
+            type: number
+            minimum: 0
+            description: >
+              Cap on cumulative switching overhead, as a percentage of what the session
+              would have cost by never switching (a whole number: 20 = 20%). The promise
+              is "this conversation bills at most max_overhead_pct% above never-switching."
+              0 means never pay to switch (only outright-cheaper switches are allowed);
+              larger values buy more quality-driven switches. Typical range 10-30.
+          replenish_on_rebind:
+            type: boolean
+            description: "Reset the running baseline/spend totals when a cold session re-binds. Default true."
+          cache_read_discount:
+            type: number
+            minimum: 0
+            maximum: 1
+            description: >
+              Fallback used to estimate a model's cached input rate when the pricing feed
+              doesn't publish one (cached_rate = input_rate x discount). A pricing detail,
+              not a cost policy. Default 0.1.
+          record_counterfactual:
+            type: boolean
+            description: >
+              When true, a vetoed switch records the route the gate would have taken had
+              the switch been allowed, as the plano.switch.counterfactual_route span
+              attribute. Telemetry only — the counterfactual model is never dispatched.
+              Useful for evals/benchmarks. Default false.
+        required: ["max_overhead_pct"]
+        additionalProperties: false
     additionalProperties: false
   state_storage:
     type: object
@@ -582,13 +685,17 @@ properties:
               type: string
               enum:
                 - digitalocean
+                - models.dev
+            url:
+              type: string
+              description: "Optional override for the pricing catalog endpoint. Defaults per provider (digitalocean: DO GenAI catalog; models.dev: https://models.dev/api.json)."
             refresh_interval:
               type: integer
               minimum: 1
               description: "Refresh interval in seconds"
             model_aliases:
               type: object
-              description: "Map DO catalog keys (lowercase(creator)/model_id) to Plano model names used in routing_preferences. Example: 'openai/openai-gpt-oss-120b: openai/gpt-4o'"
+              description: "Map catalog keys to Plano model names used in routing_preferences. DigitalOcean keys are 'lowercase(creator)/model_id'; models.dev keys are 'creator/model_id'. Example: 'openai/openai-gpt-oss-120b: openai/gpt-4o'"
               additionalProperties:
                 type: string
           required:

crates/brightstaff/src/affinity.rs

@@ -0,0 +1,181 @@
+//! Implicit session-affinity key derivation.
+//!
+//! When a request carries no explicit `X-Model-Affinity` header, Plano derives a
+//! stable session key from the parts of the prompt that repeat verbatim at the head
+//! of every turn — the same bytes the provider's prompt cache is keyed on:
+//!
+//! ```text
+//! session_key  = hash(system + tools + first_user_message)
+//! prefix_hash  = hash(system + tools)
+//! ```
+//!
+//! The session key is constant for the life of a conversation (history grows at the
+//! tail, not the head), so turns 2+ reuse the same pin without any client changes.
+//! The prefix hash covers only the fully-stable segment and is stored with the pin
+//! for drift detection: if it changes, the provider cache is already lost and
+//! re-routing fresh is safe.
+//!
+//! Only salted hashes are ever stored — never prompt content.
+
+use hermesllm::apis::openai::{Message, Role};
+
+/// Salt folded into every hash so stored keys can't be trivially correlated with
+/// prompt content across systems. Deterministic across processes/replicas so a
+/// shared Redis session cache keys consistently.
+const HASH_SALT: &str = "plano-affinity-v1";
+
+/// Prefix distinguishing derived keys from client-supplied `X-Model-Affinity` ids.
+const IMPLICIT_KEY_PREFIX: &str = "implicit:";
+
+/// Derived affinity identifiers for one request.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ImplicitAffinity {
+    /// Session-cache key: `implicit:{hex}` over system + tools + first user message.
+    pub session_key: String,
+    /// Hash of the stable prefix only (system + tools), for drift detection.
+    pub prefix_hash: u64,
+}
+
+/// FNV-1a 64-bit — stable across processes and Rust versions (unlike `DefaultHasher`),
+/// dependency-free, and plenty for cache keying (collisions merely over-pin, which is
+/// cache-friendly).
+fn fnv1a64(chunks: &[&str]) -> u64 {
+    const OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
+    const PRIME: u64 = 0x0000_0100_0000_01b3;
+    let mut hash = OFFSET_BASIS;
+    let mut feed = |bytes: &[u8]| {
+        for &b in bytes {
+            hash ^= b as u64;
+            hash = hash.wrapping_mul(PRIME);
+        }
+        // Field separator so ("ab","c") and ("a","bc") hash differently.
+        hash ^= 0x1f;
+        hash = hash.wrapping_mul(PRIME);
+    };
+    feed(HASH_SALT.as_bytes());
+    for chunk in chunks {
+        feed(chunk.as_bytes());
+    }
+    hash
+}
+
+/// Derive the implicit affinity key from parsed request messages and tool names.
+///
+/// Returns `None` when there is no user message to anchor on (nothing distinguishes
+/// the conversation, so pinning would be meaningless).
+pub fn derive_implicit_affinity(
+    messages: &[Message],
+    tool_names: Option<&[String]>,
+    tenant_id: Option<&str>,
+) -> Option<ImplicitAffinity> {
+    let system_text: String = messages
+        .iter()
+        .filter(|m| matches!(m.role, Role::System | Role::Developer))
+        .filter_map(|m| m.content.as_ref().map(|c| c.to_string()))
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    let first_user = messages
+        .iter()
+        .find(|m| matches!(m.role, Role::User))
+        .and_then(|m| m.content.as_ref().map(|c| c.to_string()))?;
+
+    let tools_text = tool_names.map(|names| names.join(",")).unwrap_or_default();
+    let tenant = tenant_id.unwrap_or_default();
+
+    let prefix_hash = fnv1a64(&[tenant, &system_text, &tools_text]);
+    let session_hash = fnv1a64(&[tenant, &system_text, &tools_text, &first_user]);
+
+    Some(ImplicitAffinity {
+        session_key: format!("{IMPLICIT_KEY_PREFIX}{session_hash:016x}"),
+        prefix_hash,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use hermesllm::apis::openai::MessageContent;
+
+    fn msg(role: Role, text: &str) -> Message {
+        Message {
+            role,
+            content: Some(MessageContent::Text(text.to_string())),
+            name: None,
+            tool_call_id: None,
+            tool_calls: None,
+        }
+    }
+
+    #[test]
+    fn key_is_stable_as_history_grows() {
+        let turn1 = vec![
+            msg(Role::System, "You are a coding agent."),
+            msg(Role::User, "Fix the bug in main.rs"),
+        ];
+        let turn5 = vec![
+            msg(Role::System, "You are a coding agent."),
+            msg(Role::User, "Fix the bug in main.rs"),
+            msg(Role::Assistant, "Done. Anything else?"),
+            msg(Role::User, "Now add tests"),
+            msg(Role::Assistant, "Added."),
+        ];
+
+        let a1 = derive_implicit_affinity(&turn1, None, None).unwrap();
+        let a5 = derive_implicit_affinity(&turn5, None, None).unwrap();
+        assert_eq!(a1.session_key, a5.session_key);
+        assert_eq!(a1.prefix_hash, a5.prefix_hash);
+        assert!(a1.session_key.starts_with("implicit:"));
+    }
+
+    #[test]
+    fn different_first_user_message_yields_different_session() {
+        let base = msg(Role::System, "You are a coding agent.");
+        let a = derive_implicit_affinity(
+            &[base.clone(), msg(Role::User, "conversation A")],
+            None,
+            None,
+        )
+        .unwrap();
+        let b = derive_implicit_affinity(&[base, msg(Role::User, "conversation B")], None, None)
+            .unwrap();
+        // Same stable prefix, different conversations.
+        assert_eq!(a.prefix_hash, b.prefix_hash);
+        assert_ne!(a.session_key, b.session_key);
+    }
+
+    #[test]
+    fn changed_system_prompt_changes_prefix_hash() {
+        let a = derive_implicit_affinity(
+            &[msg(Role::System, "v1 prompt"), msg(Role::User, "hi")],
+            None,
+            None,
+        )
+        .unwrap();
+        let b = derive_implicit_affinity(
+            &[msg(Role::System, "v2 prompt"), msg(Role::User, "hi")],
+            None,
+            None,
+        )
+        .unwrap();
+        assert_ne!(a.prefix_hash, b.prefix_hash);
+        assert_ne!(a.session_key, b.session_key);
+    }
+
+    #[test]
+    fn tools_and_tenant_are_part_of_the_key() {
+        let messages = [msg(Role::System, "s"), msg(Role::User, "u")];
+        let plain = derive_implicit_affinity(&messages, None, None).unwrap();
+        let with_tools =
+            derive_implicit_affinity(&messages, Some(&["get_weather".to_string()]), None).unwrap();
+        let with_tenant = derive_implicit_affinity(&messages, None, Some("acme")).unwrap();
+        assert_ne!(plain.session_key, with_tools.session_key);
+        assert_ne!(plain.session_key, with_tenant.session_key);
+    }
+
+    #[test]
+    fn no_user_message_yields_none() {
+        assert!(derive_implicit_affinity(&[msg(Role::System, "s")], None, None).is_none());
+        assert!(derive_implicit_affinity(&[], None, None).is_none());
+    }
+}

crates/brightstaff/src/app_state.rs

@@ -1,7 +1,10 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use common::configuration::{Agent, FilterPipeline, Listener, ModelAlias, SpanAttributes};
+use common::configuration::{
+    Agent, EffectivePromptCaching, EffectiveRoutingBudget, FilterPipeline, Listener, ModelAlias,
+    SpanAttributes,
+};
 use common::llm_providers::LlmProviders;
 use tokio::sync::RwLock;
 
@@ -21,10 +24,20 @@ pub struct AppState {
     pub state_storage: Option<Arc<dyn StateStorage>>,
     pub llm_provider_url: String,
     pub span_attributes: Option<SpanAttributes>,
+    /// Request header whose value populates the observability `distinct_id`
+    /// (e.g. PostHog). Sourced from `tracing.exporters[].distinct_id_header`.
+    /// `None` means LLM events are captured anonymously.
+    pub distinct_id_header: Option<String>,
     /// Shared HTTP client for upstream LLM requests (connection pooling / keep-alive).
     pub http_client: reqwest::Client,
     pub filter_pipeline: Arc<FilterPipeline>,
     /// When false, agentic signal analysis is skipped on LLM responses to save CPU.
     /// Controlled by `overrides.disable_signals` in plano config.
     pub signals_enabled: bool,
+    /// Instance-wide automatic prompt-caching settings, resolved once from the
+    /// top-level `prompt_caching` config. Disabled by default (opt-in).
+    pub prompt_caching: EffectivePromptCaching,
+    /// Per-session model-switch cost gate, resolved from `routing.routing_budget`.
+    /// Independent of prompt caching; `None` when not configured (off by default).
+    pub routing_budget: Option<EffectiveRoutingBudget>,
 }

crates/brightstaff/src/handlers/function_calling.rs

@@ -1239,6 +1239,7 @@ impl ArchFunctionHandler {
                 total_tokens: 0,
                 prompt_tokens_details: None,
                 completion_tokens_details: None,
+                ..Default::default()
             },
             system_fingerprint: None,
             service_tier: None,

crates/brightstaff/src/handlers/llm/mod.rs

@@ -1,6 +1,9 @@
 use bytes::Bytes;
 use common::configuration::{FilterPipeline, ModelAlias};
-use common::consts::{ARCH_IS_STREAMING_HEADER, ARCH_PROVIDER_HINT_HEADER, MODEL_AFFINITY_HEADER};
+use common::consts::{
+    ARCH_IS_STREAMING_HEADER, ARCH_PROVIDER_HINT_HEADER, MODEL_AFFINITY_HEADER, PLANO_CACHE_HEADER,
+    PLANO_PREFIX_HASH_HEADER,
+};
 use common::llm_providers::LlmProviders;
 use hermesllm::apis::openai::Message;
 use hermesllm::apis::openai_responses::InputParam;
@@ -19,6 +22,8 @@ use tokio::sync::RwLock;
 use tracing::{debug, info, info_span, warn, Instrument};
 
 pub(crate) mod model_selection;
+pub(crate) mod prompt_caching;
+pub(crate) mod session_router;
 
 use crate::app_state::AppState;
 use crate::handlers::agents::pipeline::PipelineProcessor;
@@ -31,7 +36,7 @@ use crate::state::{
 };
 use crate::streaming::{
     create_streaming_response, create_streaming_response_with_output_filter, truncate_message,
-    LlmMetricsCtx, ObservableStreamProcessor, StreamProcessor,
+    LlmMetricsCtx, ObservableStreamProcessor, SessionUpdateCtx, StreamProcessor,
 };
 use crate::tracing::{
     collect_custom_trace_attributes, llm as tracing_llm, operation_component,
@@ -93,8 +98,26 @@ async fn llm_chat_inner(
         }
     });
 
-    // Session pinning: extract session ID and check cache before routing
-    let session_id: Option<String> = request_headers
+    // Stamp the caller identity for downstream exporters (e.g. PostHog
+    // `distinct_id`). Sourced from the configured `distinct_id_header`; when the
+    // header is absent the event is exported anonymously.
+    if let Some(header_name) = state.distinct_id_header.as_deref() {
+        if let Some(distinct_id) = request_headers
+            .get(header_name)
+            .and_then(|v| v.to_str().ok())
+            .map(str::trim)
+            .filter(|s| !s.is_empty())
+        {
+            get_active_span(|span| {
+                span.set_attribute(opentelemetry::KeyValue::new(
+                    tracing_plano::DISTINCT_ID,
+                    distinct_id.to_string(),
+                ));
+            });
+        }
+    }
+
+    let explicit_session_id: Option<String> = request_headers
         .get(MODEL_AFFINITY_HEADER)
         .and_then(|h| h.to_str().ok())
         .map(|s| s.to_string());
@@ -104,40 +127,17 @@ async fn llm_chat_inner(
         .and_then(|hdr| request_headers.get(hdr))
         .and_then(|v| v.to_str().ok())
         .map(|s| s.to_string());
-    let cached_route = if let Some(ref sid) = session_id {
-        state
-            .orchestrator_service
-            .get_cached_route(sid, tenant_id.as_deref())
-            .await
-    } else {
-        None
-    };
-    let (pinned_model, pinned_route_name): (Option<String>, Option<String>) = match cached_route {
-        Some(c) => (Some(c.model_name), c.route_name),
-        None => (None, None),
-    };
-
-    // Record session id on the LLM span for the observability console.
-    if let Some(ref sid) = session_id {
-        get_active_span(|span| {
-            span.set_attribute(opentelemetry::KeyValue::new(
-                tracing_plano::SESSION_ID,
-                sid.clone(),
-            ));
-        });
-    }
-    if let Some(ref route_name) = pinned_route_name {
-        get_active_span(|span| {
-            span.set_attribute(opentelemetry::KeyValue::new(
-                tracing_plano::ROUTE_NAME,
-                route_name.clone(),
-            ));
-        });
-    }
+    // `X-Plano-Cache: off` disables implicit pinning + marker injection for one call.
+    let cache_off_for_request = request_headers
+        .get(PLANO_CACHE_HEADER)
+        .and_then(|h| h.to_str().ok())
+        .is_some_and(|v| v.eq_ignore_ascii_case("off"));
 
     let full_qualified_llm_provider_url = format!("{}{}", state.llm_provider_url, request_path);
 
     // --- Phase 1: Parse and validate the incoming request ---
+    // (Parsing happens before session-key derivation: the implicit affinity key is
+    // computed from the request body's stable prompt prefix.)
     let parsed = match parse_and_validate_request(
         request,
         &request_path,
@@ -168,6 +168,14 @@ async fn llm_chat_inner(
         provider_id,
     } = parsed;
 
+    // Prompt caching is configured once for the whole instance and never influences
+    // which model routing selects — it only keeps a conversation on the same
+    // model/provider so the upstream prompt cache stays warm across turns. Note:
+    // session affinity and pin lookup are derived later (Phase 2a), *after* input
+    // filters and Responses-API state merge, so the prefix hash reflects the exact
+    // stable prefix sent upstream rather than the pre-filter request body.
+    let prompt_caching = state.prompt_caching;
+
     // Record LLM-specific span attributes
     let span = tracing::Span::current();
     if let Some(temp) = temperature {
@@ -264,6 +272,18 @@ async fn llm_chat_inner(
             *bad_request.status_mut() = StatusCode::BAD_REQUEST;
             return Ok(bad_request);
         }
+
+        // Auto-inject prompt-cache markers (grouped in `prompt_caching`). No-op unless
+        // caching is enabled for this request and the model needs explicit markers.
+        prompt_caching::inject_cache_markers(
+            &mut client_request,
+            provider_id,
+            &model_name_only,
+            &upstream_api,
+            &prompt_caching,
+            cache_off_for_request,
+            &alias_resolved_model,
+        );
     }
 
     // --- Phase 2: Resolve conversation state (v1/responses API) ---
@@ -282,6 +302,40 @@ async fn llm_chat_inner(
         Err(response) => return Ok(response),
     };
 
+    // --- Phase 2a: Session affinity (see `session_router`) ---
+    // Derived here, after input filters and Responses-API state merge, so the
+    // implicit session key and prefix hash are computed over the exact stable
+    // prefix (system + tools + first user message) that is actually sent upstream —
+    // the same bytes the provider's prompt cache is keyed on. The prefix hash is
+    // derived even when caching is off, so the `x-plano-prefix-hash` RING_HASH
+    // replica-stickiness header still works.
+    let routing_budget = state.routing_budget;
+    // Derive the implicit session key when either prompt-caching affinity or the
+    // routing budget is active — the budget needs a session anchor even with caching off.
+    let implicit_affinity_enabled = prompt_caching.session_affinity || routing_budget.is_some();
+    let request_messages = client_request.get_messages();
+    let session_router::SessionResolution {
+        request_prefix_hash,
+        session_id,
+    } = session_router::resolve_session(
+        explicit_session_id,
+        &request_messages,
+        tool_names.as_deref(),
+        tenant_id.as_deref(),
+        implicit_affinity_enabled,
+        cache_off_for_request,
+    );
+
+    // Record session id on the LLM span for the observability console.
+    if let Some(ref sid) = session_id {
+        get_active_span(|span| {
+            span.set_attribute(opentelemetry::KeyValue::new(
+                tracing_plano::SESSION_ID,
+                sid.clone(),
+            ));
+        });
+    }
+
     // Serialize request for upstream BEFORE router consumes it
     let client_request_bytes_for_upstream: Bytes =
         match ProviderRequestType::to_bytes(&client_request) {
@@ -294,15 +348,17 @@ async fn llm_chat_inner(
             }
         };
 
-    // --- Phase 3: Route the request (or use pinned model from session cache) ---
-    let resolved_model = if let Some(cached_model) = pinned_model {
-        info!(
-            session_id = %session_id.as_deref().unwrap_or(""),
-            model = %cached_model,
-            "using pinned routing decision from cache"
-        );
-        cached_model
+    // Context size for the switch-cost estimate, computed before the router consumes
+    // the request. Only needed when stickiness could act on a session.
+    let context_tokens: u64 = if session_id.is_some() && routing_budget.is_some() {
+        session_router::actual_context_tokens(&request_messages, &model_name_only)
     } else {
+        0
+    };
+
+    // --- Phase 3: Route the request (quality), then apply the session-cache decision ---
+    // Routing stays cache-blind: the quality router always picks a candidate. The
+    // session router then honors it or sticks to the warm anchor (see `session_router`).
     let routing_span = info_span!(
         "routing",
         component = "routing",
@@ -335,16 +391,32 @@ async fn llm_chat_inner(
         }
     };
 
-        let (router_selected_model, route_name) =
-            (routing_result.model_name, routing_result.route_name);
-        let model = if router_selected_model != "none" {
-            router_selected_model
+    let candidate_model = if routing_result.model_name != "none" {
+        routing_result.model_name
     } else {
         alias_resolved_model.clone()
     };
+    let candidate_route = routing_result.route_name;
 
-        // Record route name on the LLM span (only when the orchestrator produced one).
-        if let Some(ref rn) = route_name {
+    let decision = session_router::route(
+        &state.orchestrator_service,
+        routing_budget.as_ref(),
+        session_router::RouteFacts {
+            session_id: session_id.as_deref(),
+            tenant_id: tenant_id.as_deref(),
+            prefix_hash: request_prefix_hash,
+            context_tokens,
+            candidate_model: &candidate_model,
+            candidate_route: candidate_route.as_deref(),
+        },
+    )
+    .await;
+
+    let resolved_model = decision.model;
+    let resolved_route_name = decision.route_name;
+
+    // Record route name on the LLM span (only when a real route was produced).
+    if let Some(ref rn) = resolved_route_name {
         if !rn.is_empty() && rn != "none" {
             get_active_span(|span| {
                 span.set_attribute(opentelemetry::KeyValue::new(
@@ -355,16 +427,71 @@ async fn llm_chat_inner(
         }
     }
 
-        if let Some(ref sid) = session_id {
-            state
-                .orchestrator_service
-                .cache_route(sid.clone(), tenant_id.as_deref(), model.clone(), route_name)
-                .await;
+    tracing::Span::current().record(tracing_llm::MODEL_NAME, resolved_model.as_str());
+
+    // Record the provider (derived from the `provider/model` prefix) so
+    // observability exporters can populate provider fields (e.g. PostHog
+    // `$ai_provider`).
+    let (resolved_provider, _) = bs_metrics::split_provider_model(&resolved_model);
+    if resolved_provider != "unknown" {
+        get_active_span(|span| {
+            span.set_attribute(opentelemetry::KeyValue::new(
+                tracing_llm::PROVIDER,
+                resolved_provider.to_string(),
+            ));
+        });
     }
 
-        model
+    // Resolve the dispatched model's catalog rates now (this side is async; the
+    // response path that prices the turn is synchronous). `None` when no cost feed is
+    // configured → no per-request/session cost is computed.
+    let cost_rates = if session_id.is_some() {
+        state
+            .orchestrator_service
+            .model_rates(&resolved_model)
+            .await
+    } else {
+        None
     };
-    tracing::Span::current().record(tracing_llm::MODEL_NAME, resolved_model.as_str());
+    let cache_read_discount = state
+        .routing_budget
+        .as_ref()
+        .map(|b| b.cache_read_discount)
+        .unwrap_or(common::configuration::DEFAULT_CACHE_READ_DISCOUNT);
+
+    // Response-side refresh: update `last_used` + the context-size estimate from the
+    // real response. The routing decision itself was already persisted by `route()`.
+    let session_update_ctx: Option<SessionUpdateCtx> =
+        session_id.as_ref().map(|sid| SessionUpdateCtx {
+            orchestrator: Arc::clone(&state.orchestrator_service),
+            session_id: sid.clone(),
+            tenant_id: tenant_id.clone(),
+            anchor_model: resolved_model.clone(),
+            default_model: decision.default_model.clone(),
+            route_name: resolved_route_name.clone(),
+            prefix_hash: request_prefix_hash,
+            baseline_usd: decision.baseline_usd,
+            switch_spend_usd: decision.switch_spend_usd,
+            switches: decision.switches,
+            history: decision.history.clone(),
+            session_cost_usd: decision.session_cost_usd,
+            cost_rates,
+            cache_read_discount,
+            context_tokens: decision.cached_tokens,
+            gc_ttl: decision.gc_ttl,
+        });
+
+    // Forward the prefix hash so self-hosted multi-replica backends can do
+    // KV-aware replica stickiness (consistent-hash on this header at the
+    // cluster layer).
+    if let Some(hash) = request_prefix_hash {
+        if let Ok(val) = header::HeaderValue::from_str(&format!("{hash:016x}")) {
+            request_headers.insert(
+                header::HeaderName::from_static(PLANO_PREFIX_HASH_HEADER),
+                val,
+            );
+        }
+    }
 
     // --- Phase 4: Forward to upstream and stream back ---
     send_upstream(
@@ -383,6 +510,7 @@ async fn llm_chat_inner(
         state.state_storage.clone(),
         request_id,
         &state.filter_pipeline,
+        session_update_ctx,
     )
     .await
 }
@@ -660,6 +788,7 @@ async fn send_upstream(
     state_storage: Option<Arc<dyn StateStorage>>,
     request_id: String,
     filter_pipeline: &Arc<FilterPipeline>,
+    session_update_ctx: Option<SessionUpdateCtx>,
 ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
     let span_name = if model_from_request == resolved_model {
         format!("POST {} {}", request_path, resolved_model)
@@ -775,7 +904,7 @@ async fn send_upstream(
     let byte_stream = llm_response.bytes_stream();
 
     // Create base processor for metrics and tracing
-    let base_processor = ObservableStreamProcessor::new(
+    let mut base_processor = ObservableStreamProcessor::new(
         operation_component::LLM,
         span_name,
         request_start_time,
@@ -786,6 +915,13 @@ async fn send_upstream(
         model: metric_model.clone(),
         upstream_status: upstream_status.as_u16(),
     });
+    // Only refresh the session binding for successful responses — errors carry no
+    // usage block and shouldn't reset the warmth clock.
+    if upstream_status.is_success() {
+        if let Some(update_ctx) = session_update_ctx {
+            base_processor = base_processor.with_session_update(update_ctx);
+        }
+    }
 
     let output_filter_request_headers = if filter_pipeline.has_output_filters() {
         Some(request_headers.clone())

crates/brightstaff/src/handlers/llm/prompt_caching.rs

@@ -0,0 +1,70 @@
+//! Prompt-caching request handling for the LLM path.
+//!
+//! This module owns the "make the upstream provider's prompt cache work" concern:
+//! resolving the correct cache-marking strategy for the `(gateway × model family ×
+//! upstream API)` combination and injecting the markers into the outbound request.
+//! It never influences routing — see [`super::session_router`] for the session-cache
+//! lookup and the routing-budget switch-cost concern.
+
+use common::configuration::EffectivePromptCaching;
+use hermesllm::clients::SupportedUpstreamAPIs;
+use hermesllm::{cache_marker_strategy, CacheMarkerStrategy, ProviderId, ProviderRequestType};
+use tracing::debug;
+
+/// Auto-inject prompt-cache markers into `client_request`.
+///
+/// The strategy is resolved from `(gateway × model family × upstream API)` so that
+/// Anthropic-family models cache whether they arrive over the native Messages API or
+/// an OpenAI-compatible gateway (DigitalOcean, OpenRouter), while OpenAI-family models
+/// (which cache automatically) and unimplemented backends are left untouched.
+///
+/// A no-op when caching is disabled, `inject_cache_control` is off, the request opted
+/// out (`X-Plano-Cache: off`), or the provider caches automatically. Injection is
+/// idempotent (client-supplied markers are respected) and threshold-guarded against
+/// the provider's minimum cacheable prefix.
+pub fn inject_cache_markers(
+    client_request: &mut ProviderRequestType,
+    provider_id: ProviderId,
+    model_name_only: &str,
+    upstream_api: &SupportedUpstreamAPIs,
+    prompt_caching: &EffectivePromptCaching,
+    cache_off_for_request: bool,
+    alias_resolved_model: &str,
+) {
+    if !(prompt_caching.enabled && prompt_caching.inject_cache_control && !cache_off_for_request) {
+        return;
+    }
+
+    match cache_marker_strategy(provider_id, model_name_only, upstream_api) {
+        CacheMarkerStrategy::AnthropicMessagesBreakpoints {
+            min_prefix_tokens, ..
+        } => {
+            if let ProviderRequestType::MessagesRequest(req) = client_request {
+                let threshold = prompt_caching.min_prefix_tokens.max(min_prefix_tokens);
+                if req.inject_cache_breakpoints(threshold) {
+                    debug!(
+                        model = %alias_resolved_model,
+                        min_prefix_tokens = threshold,
+                        "injected anthropic ephemeral cache breakpoints"
+                    );
+                }
+            }
+        }
+        CacheMarkerStrategy::OpenAiContentPartCacheControl {
+            min_prefix_tokens,
+            ttl,
+        } => {
+            if let ProviderRequestType::ChatCompletionsRequest(req) = client_request {
+                let threshold = prompt_caching.min_prefix_tokens.max(min_prefix_tokens);
+                if req.inject_cache_control(ttl, threshold) {
+                    debug!(
+                        model = %alias_resolved_model,
+                        min_prefix_tokens = threshold,
+                        "injected openai content-part cache_control"
+                    );
+                }
+            }
+        }
+        CacheMarkerStrategy::Automatic | CacheMarkerStrategy::None => {}
+    }
+}

crates/brightstaff/src/handlers/routing_service.rs

@@ -1,9 +1,11 @@
 use bytes::Bytes;
-use common::configuration::{SpanAttributes, TopLevelRoutingPreference};
+use common::configuration::{
+    EffectivePromptCaching, EffectiveRoutingBudget, SpanAttributes, TopLevelRoutingPreference,
+};
 use common::consts::{MODEL_AFFINITY_HEADER, REQUEST_ID_HEADER};
 use common::errors::BrightStaffError;
 use hermesllm::clients::SupportedAPIsFromClient;
-use hermesllm::ProviderRequestType;
+use hermesllm::{ProviderRequest, ProviderRequestType};
 use http_body_util::combinators::BoxBody;
 use http_body_util::{BodyExt, Full};
 use hyper::{Request, Response, StatusCode};
@@ -12,6 +14,7 @@ use tracing::{debug, info, info_span, warn, Instrument};
 
 use super::extract_or_generate_traceparent;
 use crate::handlers::llm::model_selection::router_chat_get_upstream_model;
+use crate::handlers::llm::session_router;
 use crate::metrics as bs_metrics;
 use crate::metrics::labels as metric_labels;
 use crate::router::orchestrator::OrchestratorService;
@@ -60,11 +63,14 @@ struct RoutingDecisionResponse {
     pinned: bool,
 }
 
+#[allow(clippy::too_many_arguments)]
 pub async fn routing_decision(
     request: Request<hyper::body::Incoming>,
     orchestrator_service: Arc<OrchestratorService>,
     request_path: String,
     span_attributes: &Option<SpanAttributes>,
+    prompt_caching: EffectivePromptCaching,
+    routing_budget: Option<EffectiveRoutingBudget>,
 ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
     let request_headers = request.headers().clone();
     let request_id: String = request_headers
@@ -73,7 +79,7 @@ pub async fn routing_decision(
         .map(|s| s.to_string())
         .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
 
-    let session_id: Option<String> = request_headers
+    let explicit_session_id: Option<String> = request_headers
         .get(MODEL_AFFINITY_HEADER)
         .and_then(|h| h.to_str().ok())
         .map(|s| s.to_string());
@@ -101,8 +107,10 @@ pub async fn routing_decision(
         request_path,
         request_headers,
         custom_attrs,
-        session_id,
+        explicit_session_id,
         tenant_id,
+        prompt_caching,
+        routing_budget,
     )
     .instrument(request_span)
     .await
@@ -116,8 +124,10 @@ async fn routing_decision_inner(
     request_path: String,
     request_headers: hyper::HeaderMap,
     custom_attrs: std::collections::HashMap<String, String>,
-    session_id: Option<String>,
+    explicit_session_id: Option<String>,
     tenant_id: Option<String>,
+    prompt_caching: EffectivePromptCaching,
+    routing_budget: Option<EffectiveRoutingBudget>,
 ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
     set_service_name(operation_component::ROUTING);
     opentelemetry::trace::get_active_span(|span| {
@@ -135,37 +145,9 @@ async fn routing_decision_inner(
         .unwrap_or("unknown")
         .to_string();
 
-    if let Some(ref sid) = session_id {
-        if let Some(cached) = orchestrator_service
-            .get_cached_route(sid, tenant_id.as_deref())
-            .await
-        {
-            info!(
-                session_id = %sid,
-                model = %cached.model_name,
-                route = ?cached.route_name,
-                "returning pinned routing decision from cache"
-            );
-            let response = RoutingDecisionResponse {
-                models: vec![cached.model_name],
-                route: cached.route_name,
-                trace_id,
-                session_id: Some(sid.clone()),
-                pinned: true,
-            };
-            let json = serde_json::to_string(&response).unwrap();
-            let body = Full::new(Bytes::from(json))
-                .map_err(|never| match never {})
-                .boxed();
-            return Ok(Response::builder()
-                .status(StatusCode::OK)
-                .header("Content-Type", "application/json")
-                .body(body)
-                .unwrap());
-        }
-    }
-
-    // Parse request body
+    // Parse the request body up front so a pin can be validated against prefix drift
+    // and a fresh pin can be stored with its prefix hash. This endpoint shares the
+    // session cache with the LLM handler, so both must key drift the same way.
     let raw_bytes = request.collect().await?.to_bytes();
 
     debug!(
@@ -202,6 +184,39 @@ async fn routing_decision_inner(
         }
     };
 
+    // `X-Plano-Cache: off` opts this request out of implicit affinity (same sentinel
+    // the LLM handler honors), so callers can bypass stickiness per request.
+    let cache_off_for_request = request_headers
+        .get(common::consts::PLANO_CACHE_HEADER)
+        .and_then(|h| h.to_str().ok())
+        .is_some_and(|v| v.eq_ignore_ascii_case("off"));
+
+    let request_messages = client_request.get_messages();
+    let tool_names = client_request.get_tool_names();
+
+    // Session key + prefix hash resolved identically to the LLM handler so pins
+    // interoperate across the full-proxy and decision paths.
+    // Derive the implicit session key when either prompt-caching affinity or the routing
+    // budget is active, so the budget works the same way with caching off.
+    let implicit_affinity_enabled = prompt_caching.session_affinity || routing_budget.is_some();
+    let session_router::SessionResolution {
+        request_prefix_hash,
+        session_id,
+    } = session_router::resolve_session(
+        explicit_session_id,
+        &request_messages,
+        tool_names.as_deref(),
+        tenant_id.as_deref(),
+        implicit_affinity_enabled,
+        cache_off_for_request,
+    );
+
+    let context_tokens: u64 = if session_id.is_some() && routing_budget.is_some() {
+        session_router::actual_context_tokens(&request_messages, client_request.model())
+    } else {
+        0
+    };
+
     let routing_result = router_chat_get_upstream_model(
         Arc::clone(&orchestrator_service),
         client_request,
@@ -213,23 +228,37 @@ async fn routing_decision_inner(
 
     match routing_result {
         Ok(result) => {
-            if let Some(ref sid) = session_id {
-                orchestrator_service
-                    .cache_route(
-                        sid.clone(),
-                        tenant_id.as_deref(),
-                        result.model_name.clone(),
-                        result.route_name.clone(),
+            let candidate_model = result.model_name.clone();
+            let decision = session_router::route(
+                &orchestrator_service,
+                routing_budget.as_ref(),
+                session_router::RouteFacts {
+                    session_id: session_id.as_deref(),
+                    tenant_id: tenant_id.as_deref(),
+                    prefix_hash: request_prefix_hash,
+                    context_tokens,
+                    candidate_model: &candidate_model,
+                    candidate_route: result.route_name.as_deref(),
+                },
             )
             .await;
+
+            // Front the ranked fallback list with the decided model (the anchor, when a
+            // switch was vetoed), so 429/5xx fallbacks still work.
+            let mut models = result.models;
+            if models.first() != Some(&decision.model) {
+                models.retain(|m| m != &decision.model);
+                models.insert(0, decision.model.clone());
             }
 
             let response = RoutingDecisionResponse {
-                models: result.models,
-                route: result.route_name,
+                models,
+                route: decision.route_name,
                 trace_id,
                 session_id,
-                pinned: false,
+                // `pinned` signals a warm, stuck session — safe for callers to treat as
+                // "keep this provider's cache warm".
+                pinned: decision.warm,
             };
 
             // Distinguish "decision served" (a concrete model picked) from
@@ -247,6 +276,7 @@ async fn routing_decision_inner(
                 primary_model = %response.models.first().map(|s| s.as_str()).unwrap_or("none"),
                 total_models = response.models.len(),
                 route = ?response.route,
+                pinned = response.pinned,
                 "routing decision completed"
             );
 

crates/brightstaff/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod affinity;
 pub mod app_state;
 pub mod handlers;
 pub mod metrics;

crates/brightstaff/src/main.rs

@@ -216,7 +216,14 @@ async fn init_app_state(
         if latency_count > 1 {
             return Err("model_metrics_sources: only one latency metrics source is allowed".into());
         }
-        let svc = ModelMetricsService::new(sources, reqwest::Client::new()).await;
+        // The initial pricing fetch is awaited before listeners come up, so bound it —
+        // an unresponsive feed (models.dev / DO catalog) must not hang startup. The
+        // same client backs the periodic refresh loop.
+        let metrics_client = reqwest::Client::builder()
+            .connect_timeout(std::time::Duration::from_secs(5))
+            .timeout(std::time::Duration::from_secs(30))
+            .build()?;
+        let svc = ModelMetricsService::new(sources, metrics_client).await;
         Some(Arc::new(svc))
     } else {
         None
@@ -327,8 +334,71 @@ async fn init_app_state(
         .as_ref()
         .and_then(|tracing| tracing.span_attributes.clone());
 
+    // Resolve the distinct_id header from the first PostHog exporter that
+    // declares one, so the LLM handler can stamp `plano.distinct_id` on spans.
+    let distinct_id_header = config
+        .tracing
+        .as_ref()
+        .and_then(|tracing| tracing.exporters.as_ref())
+        .and_then(|exporters| {
+            exporters.iter().find_map(|exporter| match exporter {
+                common::configuration::Exporter::Posthog(posthog) => {
+                    posthog.distinct_id_header.clone()
+                }
+            })
+        });
+
     let signals_enabled = !overrides.disable_signals.unwrap_or(false);
 
+    let prompt_caching =
+        common::configuration::EffectivePromptCaching::from_config(config.prompt_caching.as_ref())?;
+
+    // Routing budget lives under `routing` and is independent of prompt caching.
+    let routing_budget = common::configuration::EffectiveRoutingBudget::from_config(
+        config
+            .routing
+            .as_ref()
+            .and_then(|r| r.routing_budget.as_ref()),
+    )?;
+
+    // The routing-budget cost gate needs per-model pricing to compute switch cost.
+    if routing_budget.is_some() {
+        use common::configuration::MetricsSource;
+        let has_cost_source = config
+            .model_metrics_sources
+            .as_deref()
+            .unwrap_or_default()
+            .iter()
+            .any(|s| matches!(s, MetricsSource::Cost(_)));
+        if !has_cost_source {
+            return Err(
+                "routing.routing_budget is configured but no cost metrics source is \
+                 configured — add a cost source (e.g. models.dev) to model_metrics_sources so \
+                 per-model input/cached rates are available for the switch-cost calculation"
+                    .into(),
+            );
+        }
+    }
+
+    // Session bindings (anchor model, budget spend) are keyed by tenant + session.
+    // Without a tenant header, sessions from different customers share one keyspace,
+    // so identical prompts from different tenants would collide on the same binding.
+    let session_state_in_use = prompt_caching.session_affinity || routing_budget.is_some();
+    if session_state_in_use
+        && config
+            .routing
+            .as_ref()
+            .and_then(|r| r.session_cache.as_ref())
+            .and_then(|c| c.tenant_header.as_ref())
+            .is_none()
+    {
+        warn!(
+            "session affinity / routing budget is enabled but routing.session_cache.tenant_header \
+             is not set — all requests share one session keyspace; set tenant_header to isolate \
+             sessions per customer in multi-tenant deployments"
+        );
+    }
+
     Ok(AppState {
         orchestrator_service,
         model_aliases: config.model_aliases.clone(),
@@ -338,9 +408,12 @@ async fn init_app_state(
         state_storage,
         llm_provider_url,
         span_attributes,
+        distinct_id_header,
         http_client: reqwest::Client::new(),
         filter_pipeline,
         signals_enabled,
+        prompt_caching,
+        routing_budget,
     })
 }
 
@@ -512,6 +585,8 @@ async fn dispatch(
                 Arc::clone(&state.orchestrator_service),
                 stripped,
                 &state.span_attributes,
+                state.prompt_caching,
+                state.routing_budget,
             )
             .with_context(parent_cx)
             .await;

crates/brightstaff/src/metrics/labels.rs

@@ -18,6 +18,10 @@ pub const ROUTE_LLM: &str = "llm";
 // Token kind for brightstaff_llm_tokens_total.
 pub const TOKEN_KIND_PROMPT: &str = "prompt";
 pub const TOKEN_KIND_COMPLETION: &str = "completion";
+/// Input tokens served from the provider's prompt cache (billed at the cached rate).
+pub const TOKEN_KIND_CACHE_READ: &str = "cache_read";
+/// Input tokens written into the provider's prompt cache (cache-creation surcharge).
+pub const TOKEN_KIND_CACHE_WRITE: &str = "cache_write";
 
 // LLM error_class values (match docstring in metrics/mod.rs).
 pub const LLM_ERR_NONE: &str = "none";
@@ -36,3 +40,31 @@ pub const ROUTING_SVC_POLICY_ERROR: &str = "policy_error";
 pub const SESSION_CACHE_HIT: &str = "hit";
 pub const SESSION_CACHE_MISS: &str = "miss";
 pub const SESSION_CACHE_STORE: &str = "store";
+
+// Prompt cache outcome values (brightstaff_prompt_cache_requests_total).
+pub const PROMPT_CACHE_HIT: &str = "hit";
+pub const PROMPT_CACHE_MISS: &str = "miss";
+
+// Session binding lifecycle events (brightstaff_session_binding_events_total).
+/// An existing binding was refreshed from observed usage (TTL extended, token counts
+/// and running cost updated) after a turn completed.
+pub const BINDING_EVENT_REFRESH: &str = "refresh";
+
+// Session-stickiness decisions (brightstaff_session_switch_decisions_total).
+// `decision` label — the coarse outcome:
+/// The proposed switch was honored (free, within the overhead cap, or unpriced fail-open).
+pub const SWITCH_DECISION_ALLOWED: &str = "allowed";
+/// The switch would have exceeded the session's overhead cap — the warm anchor was retained.
+pub const SWITCH_DECISION_RETAINED: &str = "retained";
+
+// `reason` label — why the decision was made:
+/// The router agreed with the warm anchor; no switch was needed.
+pub const SWITCH_REASON_SAME_ANCHOR: &str = "same_anchor";
+/// The candidate was outright cheaper (negative cost) — a free switch.
+pub const SWITCH_REASON_FREE: &str = "free";
+/// A paid switch that kept cumulative spend within the session's overhead cap.
+pub const SWITCH_REASON_WITHIN_CAP: &str = "within_cap";
+/// A paid switch that would have pushed cumulative spend over the overhead cap — retained.
+pub const SWITCH_REASON_OVER_CAP: &str = "over_cap";
+/// Pricing was missing for one side, so the switch was allowed without a cost gate.
+pub const SWITCH_REASON_NO_PRICING: &str = "no_pricing";

crates/brightstaff/src/metrics/mod.rs

@@ -18,6 +18,9 @@
 //!   `brightstaff_router_decision_duration_seconds`,
 //!   `brightstaff_routing_service_requests_total`,
 //!   `brightstaff_session_cache_events_total`.
+//! - Prompt caching: `brightstaff_prompt_cache_requests_total`,
+//!   `brightstaff_session_binding_events_total` (cache read/write tokens are emitted as
+//!   `kind=cache_read|cache_write` on `brightstaff_llm_tokens_total`).
 //! - Process: via `metrics-process`.
 //! - Build: `brightstaff_build_info`.
 
@@ -172,6 +175,16 @@ fn describe_all() {
         "brightstaff_session_cache_events_total",
         "Session affinity cache lookups and stores, by outcome."
     );
+    describe_counter!(
+        "brightstaff_prompt_cache_requests_total",
+        "LLM responses with usage data, by provider, model and prompt-cache outcome \
+         (hit = cache read/creation tokens reported, miss = none). The miss rate is \
+         the silent cache-miss baseline."
+    );
+    describe_counter!(
+        "brightstaff_session_binding_events_total",
+        "Session binding lifecycle events, by event (currently: refresh)."
+    );
 
     describe_gauge!(
         "brightstaff_build_info",
@@ -375,3 +388,37 @@ pub fn record_session_cache_event(outcome: &'static str) {
     )
     .increment(1);
 }
+
+/// Record whether a completed LLM response reported prompt-cache activity.
+/// `hit` means the provider billed cache-read or cache-creation tokens.
+pub fn record_prompt_cache_outcome(provider: &str, model: &str, outcome: &'static str) {
+    counter!(
+        "brightstaff_prompt_cache_requests_total",
+        "provider" => provider.to_string(),
+        "model" => model.to_string(),
+        "outcome" => outcome,
+    )
+    .increment(1);
+}
+
+/// Record a session binding lifecycle event (see `metrics::labels::BINDING_EVENT_*`).
+pub fn record_session_binding_event(event: &'static str) {
+    counter!(
+        "brightstaff_session_binding_events_total",
+        "event" => event,
+    )
+    .increment(1);
+}
+
+/// Record a session-stickiness decision on a proposed model switch. `decision` is the
+/// coarse outcome (`allowed`/`retained`, see `metrics::labels::SWITCH_DECISION_*`) and
+/// `reason` explains why (`same_anchor`/`free`/`within_cap`/`over_cap`/`no_pricing`,
+/// see `metrics::labels::SWITCH_REASON_*`).
+pub fn record_session_switch_decision(decision: &'static str, reason: &'static str) {
+    counter!(
+        "brightstaff_session_switch_decisions_total",
+        "decision" => decision,
+        "reason" => reason,
+    )
+    .increment(1);
+}

crates/brightstaff/src/router/model_metrics.rs

@@ -9,41 +9,124 @@ use tokio::sync::RwLock;
 use tracing::{debug, info, warn};
 
 const DO_PRICING_URL: &str = "https://api.digitalocean.com/v2/gen-ai/models/catalog";
+const MODELS_DEV_URL: &str = "https://models.dev/api.json";
+
+/// DigitalOcean publishes prices per token; scale to the per-million convention
+/// that `ModelRates` and the absolute-USD consumers (switch-cost gate) expect.
+const TOKENS_PER_MILLION: f64 = 1_000_000.0;
+
+/// Structured per-million-token USD rates for one model, as published by the cost
+/// feed. Kept alongside the blended ranking metric so cost-sensitive features (e.g.
+/// the session-stickiness switch-cost gate) can reason about input vs cached-input
+/// pricing without touching `rank_models`.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct ModelRates {
+    pub input_per_million: f64,
+    pub output_per_million: f64,
+    /// Cached (prompt-cache read) input rate. Present when the feed publishes it
+    /// (models.dev `cost.cache_read`, DO catalog `cache_read_input_price_per_million`
+    /// / `input_cache_read`); absent for models the feed prices without a cache SKU.
+    pub cache_read_per_million: Option<f64>,
+}
+
+impl ModelRates {
+    /// Blended input+output metric used for `prefer: cheapest` ranking.
+    fn blended(&self) -> f64 {
+        self.input_per_million + self.output_per_million
+    }
+
+    /// Cached input rate, falling back to `input * cache_read_discount` when the
+    /// feed doesn't publish a cached rate.
+    pub fn cached_input_rate(&self, cache_read_discount: f64) -> f64 {
+        self.cache_read_per_million
+            .unwrap_or(self.input_per_million * cache_read_discount)
+    }
+
+    /// Actual USD cost of one request from observed token usage, split into
+    /// `(input_cost, output_cost)`. Cache *creation* tokens are priced at the plain
+    /// input rate (no write premium). Cache *read* tokens use the cached rate.
+    ///
+    /// `prompt_includes_cached` reflects the provider convention: OpenAI-shape usage
+    /// folds cached tokens into `prompt_tokens` (uncached = prompt - cached), while
+    /// Anthropic-shape reports uncached input separately (uncached = prompt_tokens).
+    pub fn request_cost_usd(
+        &self,
+        prompt_tokens: u64,
+        cached_input_tokens: u64,
+        cache_creation_tokens: u64,
+        completion_tokens: u64,
+        prompt_includes_cached: bool,
+        cache_read_discount: f64,
+    ) -> (f64, f64) {
+        let read_rate = self.cached_input_rate(cache_read_discount);
+        let uncached = if prompt_includes_cached {
+            prompt_tokens.saturating_sub(cached_input_tokens)
+        } else {
+            prompt_tokens
+        };
+        let input = (uncached as f64 * self.input_per_million
+            + cached_input_tokens as f64 * read_rate
+            + cache_creation_tokens as f64 * self.input_per_million)
+            / TOKENS_PER_MILLION;
+        let output = completion_tokens as f64 * self.output_per_million / TOKENS_PER_MILLION;
+        (input, output)
+    }
+}
+
+/// Derive the blended ranking map from the structured rates map.
+fn blended_costs(rates: &HashMap<String, ModelRates>) -> HashMap<String, f64> {
+    rates
+        .iter()
+        .map(|(k, r)| (k.clone(), r.blended()))
+        .collect()
+}
 
 pub struct ModelMetricsService {
     cost: Arc<RwLock<HashMap<String, f64>>>,
+    rates: Arc<RwLock<HashMap<String, ModelRates>>>,
     latency: Arc<RwLock<HashMap<String, f64>>>,
 }
 
 impl ModelMetricsService {
     pub async fn new(sources: &[MetricsSource], client: reqwest::Client) -> Self {
         let cost_data = Arc::new(RwLock::new(HashMap::new()));
+        let rates_data = Arc::new(RwLock::new(HashMap::new()));
         let latency_data = Arc::new(RwLock::new(HashMap::new()));
 
         for source in sources {
             match source {
-                MetricsSource::Cost(cfg) => match cfg.provider {
-                    CostProvider::Digitalocean => {
+                MetricsSource::Cost(cfg) => {
+                    let provider = cfg.provider.clone();
+                    let url = cfg
+                        .url
+                        .clone()
+                        .unwrap_or_else(|| default_cost_url(&provider).to_string());
                     let aliases = cfg.model_aliases.clone().unwrap_or_default();
-                        let data = fetch_do_pricing(&client, &aliases).await;
-                        info!(models = data.len(), "fetched digitalocean pricing");
-                        *cost_data.write().await = data;
+                    let provider_name = cost_provider_name(&provider);
+
+                    let data = fetch_cost_pricing(&provider, &url, &client, &aliases).await;
+                    info!(models = data.len(), provider = provider_name, url = %url, "fetched cost pricing");
+                    *cost_data.write().await = blended_costs(&data);
+                    *rates_data.write().await = data;
 
                     if let Some(interval_secs) = cfg.refresh_interval {
                         let cost_clone = Arc::clone(&cost_data);
+                        let rates_clone = Arc::clone(&rates_data);
                         let client_clone = client.clone();
                         let interval = Duration::from_secs(interval_secs);
                         tokio::spawn(async move {
                             loop {
                                 tokio::time::sleep(interval).await;
-                                    let data = fetch_do_pricing(&client_clone, &aliases).await;
-                                    info!(models = data.len(), "refreshed digitalocean pricing");
-                                    *cost_clone.write().await = data;
+                                let data =
+                                    fetch_cost_pricing(&provider, &url, &client_clone, &aliases)
+                                        .await;
+                                info!(models = data.len(), provider = provider_name, url = %url, "refreshed cost pricing");
+                                *cost_clone.write().await = blended_costs(&data);
+                                *rates_clone.write().await = data;
                             }
                         });
                     }
                 }
-                },
                 MetricsSource::Latency(cfg) => match cfg.provider {
                     LatencyProvider::Prometheus => {
                         let data = fetch_prometheus_metrics(&cfg.url, &cfg.query, &client).await;
@@ -73,10 +156,34 @@ impl ModelMetricsService {
 
         ModelMetricsService {
             cost: cost_data,
+            rates: rates_data,
             latency: latency_data,
         }
     }
 
+    /// Build a service directly from a rates map (no network). Test-only: lets
+    /// handler/router tests exercise switch-cost math with deterministic pricing.
+    #[cfg(test)]
+    pub fn from_rates_for_test(rates: HashMap<String, ModelRates>) -> Self {
+        ModelMetricsService {
+            cost: Arc::new(RwLock::new(blended_costs(&rates))),
+            rates: Arc::new(RwLock::new(rates)),
+            latency: Arc::new(RwLock::new(HashMap::new())),
+        }
+    }
+
+    /// Structured per-million rates for a model, if the cost feed published them.
+    /// Falls back to the bare model id (without `provider/` prefix) like the
+    /// blended cost map does.
+    pub async fn model_rates(&self, model: &str) -> Option<ModelRates> {
+        let rates = self.rates.read().await;
+        rates.get(model).copied().or_else(|| {
+            model
+                .split_once('/')
+                .and_then(|(_, bare)| rates.get(bare).copied())
+        })
+    }
+
     /// Rank `models` by `policy`, returning them in preference order.
     /// Models with no metric data are appended at the end in their original order.
     pub async fn rank_models(&self, models: &[String], policy: &SelectionPolicy) -> Vec<String> {
@@ -159,42 +266,172 @@ struct DoModel {
     pricing: Option<DoPricing>,
 }
 
+/// DigitalOcean catalog pricing. Despite the `_per_million` field names, the DO
+/// API returns these as USD **per token** (e.g. gpt-4o input `2.5e-6`), so they
+/// are scaled to per-million at ingestion to match `ModelRates`' convention.
+///
+/// The catalog publishes the cached-read rate under two names (a migration in
+/// flight): `cache_read_input_price_per_million` and `input_cache_read`. Entries
+/// may carry either or both; both are parsed and the explicit name wins.
 #[derive(serde::Deserialize)]
 struct DoPricing {
     input_price_per_million: Option<f64>,
     output_price_per_million: Option<f64>,
+    cache_read_input_price_per_million: Option<f64>,
+    input_cache_read: Option<f64>,
+}
+
+impl DoPricing {
+    fn cache_read(&self) -> Option<f64> {
+        self.cache_read_input_price_per_million
+            .or(self.input_cache_read)
+    }
+}
+
+#[derive(serde::Deserialize)]
+struct ModelsDevProvider {
+    #[serde(default)]
+    models: HashMap<String, ModelsDevModel>,
+}
+
+#[derive(serde::Deserialize)]
+struct ModelsDevModel {
+    cost: Option<ModelsDevCost>,
+}
+
+#[derive(serde::Deserialize)]
+struct ModelsDevCost {
+    input: Option<f64>,
+    output: Option<f64>,
+    cache_read: Option<f64>,
+}
+
+fn default_cost_url(provider: &CostProvider) -> &'static str {
+    match provider {
+        CostProvider::Digitalocean => DO_PRICING_URL,
+        CostProvider::ModelsDev => MODELS_DEV_URL,
+    }
+}
+
+fn cost_provider_name(provider: &CostProvider) -> &'static str {
+    match provider {
+        CostProvider::Digitalocean => "digitalocean",
+        CostProvider::ModelsDev => "models.dev",
+    }
+}
+
+async fn fetch_cost_pricing(
+    provider: &CostProvider,
+    url: &str,
+    client: &reqwest::Client,
+    aliases: &HashMap<String, String>,
+) -> HashMap<String, ModelRates> {
+    match provider {
+        CostProvider::Digitalocean => fetch_do_pricing(url, client, aliases).await,
+        CostProvider::ModelsDev => fetch_models_dev_pricing(url, client, aliases).await,
+    }
 }
 
 async fn fetch_do_pricing(
+    url: &str,
     client: &reqwest::Client,
     aliases: &HashMap<String, String>,
-) -> HashMap<String, f64> {
-    match client.get(DO_PRICING_URL).send().await {
+) -> HashMap<String, ModelRates> {
+    match client.get(url).send().await {
         Ok(resp) => match resp.json::<DoModelList>().await {
-            Ok(list) => list
-                .data
+            Ok(list) => parse_do_pricing(list, aliases),
+            Err(err) => {
+                warn!(error = %err, url = %url, "failed to parse digitalocean pricing response");
+                HashMap::new()
+            }
+        },
+        Err(err) => {
+            warn!(error = %err, url = %url, "failed to fetch digitalocean pricing");
+            HashMap::new()
+        }
+    }
+}
+
+/// Map the DO catalog into `ModelRates`, scaling DO's per-token prices into the
+/// per-million convention the rest of the router uses.
+fn parse_do_pricing(
+    list: DoModelList,
+    aliases: &HashMap<String, String>,
+) -> HashMap<String, ModelRates> {
+    list.data
         .into_iter()
         .filter_map(|m| {
             let pricing = m.pricing?;
             let raw_key = m.model_id.clone();
             let key = aliases.get(&raw_key).cloned().unwrap_or(raw_key);
-                    let cost = pricing.input_price_per_million.unwrap_or(0.0)
-                        + pricing.output_price_per_million.unwrap_or(0.0);
-                    Some((key, cost))
+            // DO reports prices per token; scale to per-million so the
+            // absolute-USD consumers (the switch-cost gate) are correct.
+            // Relative ranking via `blended()` is unaffected either way.
+            let rates = ModelRates {
+                input_per_million: pricing.input_price_per_million.unwrap_or(0.0)
+                    * TOKENS_PER_MILLION,
+                output_per_million: pricing.output_price_per_million.unwrap_or(0.0)
+                    * TOKENS_PER_MILLION,
+                cache_read_per_million: pricing.cache_read().map(|r| r * TOKENS_PER_MILLION),
+            };
+            Some((key, rates))
         })
-                .collect(),
+        .collect()
+}
+
+/// models.dev publishes a top-level object keyed by provider id; each provider
+/// carries a `models` map whose keys are `creator/model` ids and whose `cost`
+/// block holds per-million USD rates. We keep the structured rates (including
+/// `cache_read` when published) and key the result by `creator/model_id` so it
+/// lines up with Plano's `provider/model` routing names.
+async fn fetch_models_dev_pricing(
+    url: &str,
+    client: &reqwest::Client,
+    aliases: &HashMap<String, String>,
+) -> HashMap<String, ModelRates> {
+    match client.get(url).send().await {
+        Ok(resp) => match resp.json::<HashMap<String, ModelsDevProvider>>().await {
+            Ok(providers) => parse_models_dev_pricing(providers, aliases),
             Err(err) => {
-                warn!(error = %err, url = DO_PRICING_URL, "failed to parse digitalocean pricing response");
+                warn!(error = %err, url = %url, "failed to parse models.dev pricing response");
                 HashMap::new()
             }
         },
         Err(err) => {
-            warn!(error = %err, url = DO_PRICING_URL, "failed to fetch digitalocean pricing");
+            warn!(error = %err, url = %url, "failed to fetch models.dev pricing");
             HashMap::new()
         }
     }
 }
 
+fn parse_models_dev_pricing(
+    providers: HashMap<String, ModelsDevProvider>,
+    aliases: &HashMap<String, String>,
+) -> HashMap<String, ModelRates> {
+    let mut out = HashMap::new();
+    for (provider_id, provider) in providers {
+        for (model_key, model) in provider.models {
+            let Some(cost) = model.cost else { continue };
+            let (Some(input), Some(output)) = (cost.input, cost.output) else {
+                continue;
+            };
+            // First-party providers use bare model keys (`claude-opus-4-5`),
+            // so compose `provider/model` to line up with Plano routing names.
+            let raw_key = format!("{provider_id}/{model_key}");
+            let rates = ModelRates {
+                input_per_million: input,
+                output_per_million: output,
+                cache_read_per_million: cost.cache_read,
+            };
+            let key = aliases.get(&raw_key).cloned().unwrap_or(raw_key);
+            out.insert(key, rates);
+            // Also register the bare model id as a fallback lookup.
+            out.entry(model_key).or_insert(rates);
+        }
+    }
+    out
+}
+
 #[derive(serde::Deserialize)]
 struct PrometheusResponse {
     data: PrometheusData,
@@ -255,6 +492,46 @@ mod tests {
         SelectionPolicy { prefer }
     }
 
+    fn service_with(
+        cost: HashMap<String, f64>,
+        latency: HashMap<String, f64>,
+    ) -> ModelMetricsService {
+        ModelMetricsService {
+            cost: Arc::new(RwLock::new(cost)),
+            rates: Arc::new(RwLock::new(HashMap::new())),
+            latency: Arc::new(RwLock::new(latency)),
+        }
+    }
+
+    #[test]
+    fn request_cost_openai_convention_subtracts_cached_from_prompt() {
+        // input $3/M, output $15/M, cached read $0.30/M. prompt_tokens (=1000) INCLUDES
+        // the 400 cached tokens → uncached = 600.
+        let rates = ModelRates {
+            input_per_million: 3.0,
+            output_per_million: 15.0,
+            cache_read_per_million: Some(0.30),
+        };
+        let (input, output) = rates.request_cost_usd(1000, 400, 0, 200, true, 0.1);
+        // 600*3 + 400*0.30 = 1800 + 120 = 1920 micro-$/M → /1e6
+        assert!((input - (1920.0 / 1_000_000.0)).abs() < 1e-12);
+        assert!((output - (200.0 * 15.0 / 1_000_000.0)).abs() < 1e-12);
+    }
+
+    #[test]
+    fn request_cost_anthropic_convention_prices_uncached_and_creation() {
+        // Anthropic: prompt_tokens (=input_tokens=100) EXCLUDES cached read (30) and
+        // creation (20). No published cached rate → falls back to input*discount.
+        let rates = ModelRates {
+            input_per_million: 3.0,
+            output_per_million: 15.0,
+            cache_read_per_million: None,
+        };
+        let (input, _output) = rates.request_cost_usd(100, 30, 20, 50, false, 0.1);
+        // uncached 100*3 + cached 30*(3*0.1=0.3) + creation 20*3 = 300 + 9 + 60 = 369
+        assert!((input - (369.0 / 1_000_000.0)).abs() < 1e-12);
+    }
+
     #[test]
     fn test_rank_by_ascending_metric_picks_lowest_first() {
         let models = vec!["a".to_string(), "b".to_string(), "c".to_string()];
@@ -285,15 +562,15 @@ mod tests {
 
     #[tokio::test]
     async fn test_rank_models_cheapest() {
-        let service = ModelMetricsService {
-            cost: Arc::new(RwLock::new({
+        let service = service_with(
+            {
                 let mut m = HashMap::new();
                 m.insert("gpt-4o".to_string(), 0.005);
                 m.insert("gpt-4o-mini".to_string(), 0.0001);
                 m
-            })),
-            latency: Arc::new(RwLock::new(HashMap::new())),
-        };
+            },
+            HashMap::new(),
+        );
         let models = vec!["gpt-4o".to_string(), "gpt-4o-mini".to_string()];
         let result = service
             .rank_models(&models, &make_policy(SelectionPreference::Cheapest))
@@ -303,15 +580,12 @@ mod tests {
 
     #[tokio::test]
     async fn test_rank_models_fastest() {
-        let service = ModelMetricsService {
-            cost: Arc::new(RwLock::new(HashMap::new())),
-            latency: Arc::new(RwLock::new({
+        let service = service_with(HashMap::new(), {
             let mut m = HashMap::new();
             m.insert("gpt-4o".to_string(), 200.0);
             m.insert("claude-sonnet".to_string(), 120.0);
             m
-            })),
-        };
+        });
         let models = vec!["gpt-4o".to_string(), "claude-sonnet".to_string()];
         let result = service
             .rank_models(&models, &make_policy(SelectionPreference::Fastest))
@@ -321,10 +595,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_rank_models_fallback_no_metrics() {
-        let service = ModelMetricsService {
-            cost: Arc::new(RwLock::new(HashMap::new())),
-            latency: Arc::new(RwLock::new(HashMap::new())),
-        };
+        let service = service_with(HashMap::new(), HashMap::new());
         let models = vec!["model-a".to_string(), "model-b".to_string()];
         let result = service
             .rank_models(&models, &make_policy(SelectionPreference::Cheapest))
@@ -334,14 +605,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_rank_models_partial_data_appended_last() {
-        let service = ModelMetricsService {
-            cost: Arc::new(RwLock::new({
+        let service = service_with(
+            {
                 let mut m = HashMap::new();
                 m.insert("gpt-4o".to_string(), 0.005);
                 m
-            })),
-            latency: Arc::new(RwLock::new(HashMap::new())),
-        };
+            },
+            HashMap::new(),
+        );
         let models = vec!["gpt-4o-mini".to_string(), "gpt-4o".to_string()];
         let result = service
             .rank_models(&models, &make_policy(SelectionPreference::Cheapest))
@@ -351,15 +622,15 @@ mod tests {
 
     #[tokio::test]
     async fn test_rank_models_none_preserves_order() {
-        let service = ModelMetricsService {
-            cost: Arc::new(RwLock::new({
+        let service = service_with(
+            {
                 let mut m = HashMap::new();
                 m.insert("gpt-4o-mini".to_string(), 0.0001);
                 m.insert("gpt-4o".to_string(), 0.005);
                 m
-            })),
-            latency: Arc::new(RwLock::new(HashMap::new())),
-        };
+            },
+            HashMap::new(),
+        );
         let models = vec!["gpt-4o".to_string(), "gpt-4o-mini".to_string()];
         let result = service
             .rank_models(&models, &make_policy(SelectionPreference::None))
@@ -368,6 +639,201 @@ mod tests {
         assert_eq!(result, vec!["gpt-4o", "gpt-4o-mini"]);
     }
 
+    #[test]
+    fn test_parse_do_pricing_scales_per_token_to_per_million() {
+        // DO's catalog returns USD *per token* despite the `_per_million` names.
+        // These are the real values from the live catalog.
+        let json = r#"{
+            "data": [
+                {
+                    "model_id": "digitalocean/anthropic-claude-4.6-sonnet",
+                    "pricing": {
+                        "input_price_per_million": 3e-6,
+                        "output_price_per_million": 15e-6,
+                        "cache_read_input_price_per_million": 3e-7
+                    }
+                },
+                {
+                    "model_id": "digitalocean/openai-gpt-4o",
+                    "pricing": {
+                        "input_price_per_million": 2.5e-6,
+                        "output_price_per_million": 10e-6
+                    }
+                }
+            ]
+        }"#;
+        let list: DoModelList = serde_json::from_str(json).unwrap();
+        let rates = parse_do_pricing(list, &HashMap::new());
+
+        // Scaled to per-million so the switch-cost gate compares real dollars.
+        let sonnet = rates
+            .get("digitalocean/anthropic-claude-4.6-sonnet")
+            .unwrap();
+        assert!((sonnet.input_per_million - 3.0).abs() < 1e-9);
+        assert!((sonnet.output_per_million - 15.0).abs() < 1e-9);
+        // DO *does* publish a cached-read rate — it must be captured, not dropped.
+        assert_eq!(sonnet.cache_read_per_million, Some(0.3));
+
+        let gpt4o = rates.get("digitalocean/openai-gpt-4o").unwrap();
+        assert!((gpt4o.input_per_million - 2.5).abs() < 1e-9);
+        assert_eq!(gpt4o.cache_read_per_million, None);
+
+        // Regression: a ~5k-token switch off warm sonnet to cold gpt-4o must now
+        // cost real dollars, not ~1e-8. This is the bug the live gate test caught.
+        let switch_cost =
+            5_000.0 / 1_000_000.0 * (gpt4o.input_per_million - sonnet.cached_input_rate(0.1));
+        assert!(
+            switch_cost > 0.01,
+            "expected ~$0.011 of switch cost, got {switch_cost}"
+        );
+    }
+
+    #[test]
+    fn test_parse_do_pricing_reads_cache_rate_from_either_field_name() {
+        // Real catalog shape (Jul 2026): DO publishes the cached-read rate under
+        // both `cache_read_input_price_per_million` and `input_cache_read` for
+        // some models, and only one of the two for others. All variants must
+        // resolve; the explicit name wins when both are present.
+        let json = r#"{
+            "data": [
+                {
+                    "model_id": "kimi-k2.5",
+                    "pricing": {
+                        "input_price_per_million": 3.75e-7,
+                        "output_price_per_million": 2.025e-6,
+                        "cache_read_input_price_per_million": 2.03e-7,
+                        "input_cache_read": 2.03e-7
+                    }
+                },
+                {
+                    "model_id": "glm-5",
+                    "pricing": {
+                        "input_price_per_million": 6e-7,
+                        "output_price_per_million": 2.2e-6,
+                        "input_cache_read": 1.1e-7
+                    }
+                },
+                {
+                    "model_id": "legacy-model",
+                    "pricing": {
+                        "input_price_per_million": 5e-7,
+                        "output_price_per_million": 1e-6
+                    }
+                }
+            ]
+        }"#;
+        let list: DoModelList = serde_json::from_str(json).unwrap();
+        let rates = parse_do_pricing(list, &HashMap::new());
+
+        // Both names present -> parsed (and scaled per-token -> per-million).
+        let kimi = rates.get("kimi-k2.5").unwrap();
+        assert!((kimi.input_per_million - 0.375).abs() < 1e-9);
+        assert!((kimi.cache_read_per_million.unwrap() - 0.203).abs() < 1e-9);
+
+        // Only the new `input_cache_read` name -> still parsed.
+        let glm = rates.get("glm-5").unwrap();
+        assert!((glm.cache_read_per_million.unwrap() - 0.11).abs() < 1e-9);
+
+        // Neither -> None, so the budget falls back to input x cache_read_discount.
+        let legacy = rates.get("legacy-model").unwrap();
+        assert_eq!(legacy.cache_read_per_million, None);
+        assert!((legacy.cached_input_rate(0.1) - 0.05).abs() < 1e-9);
+    }
+
+    #[test]
+    fn test_parse_models_dev_pricing_composes_provider_keys() {
+        let json = r#"{
+            "anthropic": {
+                "models": {
+                    "claude-opus-4-5": {"cost": {"input": 5.0, "output": 25.0, "cache_read": 0.5}}
+                }
+            },
+            "groq": {
+                "models": {
+                    "llama-3.3-70b-versatile": {"cost": {"input": 0.59, "output": 0.79}},
+                    "whisper-large-v3-turbo": {"cost": null}
+                }
+            }
+        }"#;
+        let providers: HashMap<String, ModelsDevProvider> = serde_json::from_str(json).unwrap();
+        let aliases = HashMap::new();
+        let rates = parse_models_dev_pricing(providers, &aliases);
+
+        let opus = rates.get("anthropic/claude-opus-4-5").unwrap();
+        assert_eq!(opus.blended(), 30.0);
+        assert_eq!(opus.cache_read_per_million, Some(0.5));
+        let llama = rates.get("groq/llama-3.3-70b-versatile").unwrap();
+        assert_eq!(llama.blended(), 1.38);
+        assert_eq!(llama.cache_read_per_million, None);
+        // bare fallback also registered
+        assert_eq!(
+            rates.get("claude-opus-4-5").map(|r| r.blended()),
+            Some(30.0)
+        );
+        // models with no cost block are skipped
+        assert!(!rates.contains_key("groq/whisper-large-v3-turbo"));
+    }
+
+    #[test]
+    fn test_parse_models_dev_pricing_applies_aliases() {
+        let json = r#"{
+            "openai": {"models": {"gpt-oss-120b": {"cost": {"input": 1.0, "output": 2.0}}}}
+        }"#;
+        let providers: HashMap<String, ModelsDevProvider> = serde_json::from_str(json).unwrap();
+        let mut aliases = HashMap::new();
+        aliases.insert(
+            "openai/gpt-oss-120b".to_string(),
+            "openai/gpt-4o".to_string(),
+        );
+        let rates = parse_models_dev_pricing(providers, &aliases);
+
+        assert_eq!(rates.get("openai/gpt-4o").map(|r| r.blended()), Some(3.0));
+        assert!(!rates.contains_key("openai/gpt-oss-120b"));
+    }
+
+    #[test]
+    fn test_cached_input_rate_prefers_published_rate() {
+        let with_feed = ModelRates {
+            input_per_million: 3.0,
+            output_per_million: 15.0,
+            cache_read_per_million: Some(0.3),
+        };
+        // Published cache_read wins; the discount is ignored.
+        assert_eq!(with_feed.cached_input_rate(0.5), 0.3);
+
+        let without_feed = ModelRates {
+            input_per_million: 3.0,
+            output_per_million: 15.0,
+            cache_read_per_million: None,
+        };
+        // Falls back to input * discount.
+        assert!((without_feed.cached_input_rate(0.1) - 0.3).abs() < 1e-9);
+    }
+
+    #[tokio::test]
+    async fn test_model_rates_falls_back_to_bare_model_id() {
+        let service = ModelMetricsService {
+            cost: Arc::new(RwLock::new(HashMap::new())),
+            rates: Arc::new(RwLock::new({
+                let mut m = HashMap::new();
+                m.insert(
+                    "claude-sonnet-4-5".to_string(),
+                    ModelRates {
+                        input_per_million: 3.0,
+                        output_per_million: 15.0,
+                        cache_read_per_million: Some(0.3),
+                    },
+                );
+                m
+            })),
+            latency: Arc::new(RwLock::new(HashMap::new())),
+        };
+        // provider-prefixed lookup falls back to the bare id
+        let rates = service.model_rates("vercel/claude-sonnet-4-5").await;
+        assert_eq!(rates.map(|r| r.input_per_million), Some(3.0));
+        assert!(service.model_rates("unknown/model").await.is_none());
+    }
+
     #[test]
     fn test_rank_by_ascending_metric_nan_treated_as_missing() {
         let models = vec![

crates/brightstaff/src/router/orchestrator.rs

@@ -20,9 +20,41 @@ use crate::metrics::labels as metric_labels;
 use crate::router::orchestrator_model_v1;
 use crate::session_cache::SessionCache;
 
-pub use crate::session_cache::CachedRoute;
+pub use crate::session_cache::SessionBinding;
 
 const DEFAULT_SESSION_TTL_SECONDS: u64 = 600;
+const TOKENS_PER_MILLION: f64 = 1_000_000.0;
+
+/// Input-cost of moving a session off its anchor onto `candidate`.
+///
+/// Staying re-reads the whole context at the anchor's read rate (`anchor_read_rate` — the
+/// cached rate when prompt caching keeps the anchor warm, the plain uncached rate when it
+/// doesn't). The candidate re-reads `candidate_warm_tokens` (whatever it still has cached
+/// from an earlier visit this session) at *its* cached rate, and the remaining,
+/// freshly-appended tokens at its uncached rate. When the candidate is cold
+/// (`candidate_warm_tokens == 0`) this reduces to the whole context at the uncached rate —
+/// i.e. a first-time switch. Crediting warm tokens is what makes an A→B→A return cheap:
+/// on the way back A still holds most of the context, so only the delta is re-ingested.
+///
+/// Deliberately input-only: output-token savings are unknowable before the response is
+/// generated (reasoning models can emit 5-10x the tokens for the same task), so they are
+/// never credited. Rates are USD per million tokens. Negative when the switch is outright
+/// cheaper than staying; the caller draws a positive cost down from the overhead cap.
+pub fn switch_cost_in_usd(
+    context_tokens: u64,
+    candidate_warm_tokens: u64,
+    anchor_read_rate: f64,
+    candidate_uncached_rate: f64,
+    candidate_cached_rate: f64,
+) -> f64 {
+    let warm = candidate_warm_tokens.min(context_tokens);
+    let fresh = context_tokens - warm;
+    let candidate_cost = (fresh as f64 * candidate_uncached_rate
+        + warm as f64 * candidate_cached_rate)
+        / TOKENS_PER_MILLION;
+    let anchor_cost = context_tokens as f64 * anchor_read_rate / TOKENS_PER_MILLION;
+    candidate_cost - anchor_cost
+}
 
 pub struct OrchestratorService {
     orchestrator_url: String,
@@ -126,43 +158,109 @@ impl OrchestratorService {
         }
     }
 
-    pub async fn get_cached_route(
+    /// Look up a session binding. Warmth is the caller's concern (time since
+    /// `last_used`); this only reports whether a binding exists.
+    pub async fn get_binding(
         &self,
         session_id: &str,
         tenant_id: Option<&str>,
-    ) -> Option<CachedRoute> {
+    ) -> Option<SessionBinding> {
         let cache = self.session_cache.as_ref()?;
         let result = cache.get(&Self::session_key(tenant_id, session_id)).await;
-        bs_metrics::record_session_cache_event(if result.is_some() {
-            metric_labels::SESSION_CACHE_HIT
-        } else {
-            metric_labels::SESSION_CACHE_MISS
+        bs_metrics::record_session_cache_event(match result {
+            Some(_) => metric_labels::SESSION_CACHE_HIT,
+            None => metric_labels::SESSION_CACHE_MISS,
         });
         result
     }
 
-    pub async fn cache_route(
+    /// The GC bound for a session binding: the per-scope override when provided,
+    /// otherwise the global `routing.session_ttl_seconds`. This only governs when an
+    /// idle binding is reclaimed from memory — not whether its cache is warm.
+    pub fn effective_session_ttl(&self, ttl_override_seconds: Option<u64>) -> Duration {
+        ttl_override_seconds
+            .map(Duration::from_secs)
+            .unwrap_or(self.session_ttl)
+    }
+
+    /// Persist a session binding with a GC bound (defaults to `routing.session_ttl_seconds`
+    /// when `gc_ttl` is `None`).
+    pub async fn store_binding(
         &self,
-        session_id: String,
+        session_id: &str,
         tenant_id: Option<&str>,
-        model_name: String,
-        route_name: Option<String>,
+        binding: SessionBinding,
+        gc_ttl: Option<Duration>,
     ) {
         if let Some(ref cache) = self.session_cache {
             cache
                 .put(
-                    &Self::session_key(tenant_id, &session_id),
-                    CachedRoute {
-                        model_name,
-                        route_name,
-                    },
-                    self.session_ttl,
+                    &Self::session_key(tenant_id, session_id),
+                    binding,
+                    gc_ttl.unwrap_or(self.session_ttl),
                 )
                 .await;
             bs_metrics::record_session_cache_event(metric_labels::SESSION_CACHE_STORE);
         }
     }
 
+    /// Structured per-million pricing for a model, from the configured cost feed.
+    /// `None` when no cost source is configured or the model is unknown to the feed.
+    pub async fn model_rates(&self, model: &str) -> Option<super::model_metrics::ModelRates> {
+        self.metrics_service.as_ref()?.model_rates(model).await
+    }
+
+    /// Estimate the input-cost (USD) of switching a session from `anchor_model`
+    /// (the model that handled the latest request) to `candidate_model`.
+    ///
+    /// The anchor is warm (this is only called for warm sessions), so staying re-reads the
+    /// context at its *cached* rate, while `candidate_warm_tokens` (context the candidate
+    /// still holds from an earlier visit this session) re-read at the candidate's cached
+    /// rate. Provider caches are assumed real whenever the routing budget is active —
+    /// most providers cache automatically, with or without Plano's marker injection
+    /// (`prompt_caching.enabled` is not consulted here).
+    ///
+    /// Fetches per-model rates from the configured cost feed; returns `None` when pricing
+    /// is missing for either side so the caller can fail open (switch freely) rather than
+    /// veto the router on guesswork. `cache_read_discount` estimates a model's cached-read
+    /// rate when the feed doesn't publish one. Negative when the switch is outright cheaper.
+    pub async fn estimate_switch_cost_in_usd(
+        &self,
+        context_tokens: u64,
+        anchor_model: &str,
+        candidate_model: &str,
+        candidate_warm_tokens: u64,
+        cache_read_discount: f64,
+    ) -> Option<f64> {
+        let anchor = self.model_rates(anchor_model).await?;
+        let candidate = self.model_rates(candidate_model).await?;
+        Some(switch_cost_in_usd(
+            context_tokens,
+            candidate_warm_tokens,
+            anchor.cached_input_rate(cache_read_discount),
+            candidate.input_per_million,
+            candidate.cached_input_rate(cache_read_discount),
+        ))
+    }
+
+    /// This turn's contribution to the session's *never-switch* baseline: the USD cost of
+    /// reading `context_tokens` on `model` — the session's `default_model`, i.e. what it
+    /// would have paid by never switching (not the possibly-drifted current anchor). Priced
+    /// at the cached input rate: the never-switch path stays warm, and provider caches are
+    /// assumed real whenever the routing budget is active. Summed across turns, this is
+    /// the denominator the percentage overhead cap is measured against. `None` when the
+    /// model has no pricing (the caller then can't grow the baseline this turn).
+    pub async fn context_read_cost_in_usd(
+        &self,
+        context_tokens: u64,
+        model: &str,
+        cache_read_discount: f64,
+    ) -> Option<f64> {
+        let rates = self.model_rates(model).await?;
+        let context_millions = context_tokens as f64 / TOKENS_PER_MILLION;
+        Some(context_millions * rates.cached_input_rate(cache_read_discount))
+    }
+
     // ---- LLM routing ----
 
     pub async fn determine_route(
@@ -348,86 +446,190 @@ mod tests {
         )
     }
 
-    #[tokio::test]
-    async fn test_cache_miss_returns_none() {
-        let svc = make_orchestrator_service(600, 100);
-        assert!(svc
-            .get_cached_route("unknown-session", None)
-            .await
-            .is_none());
+    fn binding(model: &str, route_name: Option<&str>) -> SessionBinding {
+        SessionBinding {
+            anchor_model: model.to_string(),
+            default_model: model.to_string(),
+            route_name: route_name.map(|r| r.to_string()),
+            prefix_hash: None,
+            last_used: std::time::SystemTime::now(),
+            cached_tokens: 0,
+            baseline_usd: 0.0,
+            switch_spend_usd: 0.0,
+            switches: 0,
+            session_cost_usd: 0.0,
+            history: Vec::new(),
+        }
     }
 
     #[tokio::test]
-    async fn test_cache_hit_returns_cached_route() {
+    async fn test_cache_miss_returns_none() {
         let svc = make_orchestrator_service(600, 100);
-        svc.cache_route(
-            "s1".to_string(),
-            None,
-            "gpt-4o".to_string(),
-            Some("code".to_string()),
-        )
+        assert!(svc.get_binding("unknown-session", None).await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_cache_hit_returns_binding() {
+        let svc = make_orchestrator_service(600, 100);
+        svc.store_binding("s1", None, binding("gpt-4o", Some("code")), None)
             .await;
 
-        let cached = svc.get_cached_route("s1", None).await.unwrap();
-        assert_eq!(cached.model_name, "gpt-4o");
+        let cached = svc.get_binding("s1", None).await.unwrap();
+        assert_eq!(cached.anchor_model, "gpt-4o");
         assert_eq!(cached.route_name, Some("code".to_string()));
     }
 
     #[tokio::test]
     async fn test_cache_expired_entry_returns_none() {
         let svc = make_orchestrator_service(0, 100);
-        svc.cache_route("s1".to_string(), None, "gpt-4o".to_string(), None)
+        svc.store_binding("s1", None, binding("gpt-4o", None), None)
             .await;
-        assert!(svc.get_cached_route("s1", None).await.is_none());
+        assert!(svc.get_binding("s1", None).await.is_none());
     }
 
     #[tokio::test]
     async fn test_expired_entries_not_returned() {
         let svc = make_orchestrator_service(0, 100);
-        svc.cache_route("s1".to_string(), None, "gpt-4o".to_string(), None)
+        svc.store_binding("s1", None, binding("gpt-4o", None), None)
             .await;
-        svc.cache_route("s2".to_string(), None, "claude".to_string(), None)
+        svc.store_binding("s2", None, binding("claude", None), None)
             .await;
 
-        assert!(svc.get_cached_route("s1", None).await.is_none());
-        assert!(svc.get_cached_route("s2", None).await.is_none());
+        assert!(svc.get_binding("s1", None).await.is_none());
+        assert!(svc.get_binding("s2", None).await.is_none());
     }
 
     #[tokio::test]
     async fn test_cache_evicts_oldest_when_full() {
         let svc = make_orchestrator_service(600, 2);
-        svc.cache_route("s1".to_string(), None, "model-a".to_string(), None)
+        svc.store_binding("s1", None, binding("model-a", None), None)
             .await;
         tokio::time::sleep(Duration::from_millis(10)).await;
-        svc.cache_route("s2".to_string(), None, "model-b".to_string(), None)
+        svc.store_binding("s2", None, binding("model-b", None), None)
             .await;
 
-        svc.cache_route("s3".to_string(), None, "model-c".to_string(), None)
+        svc.store_binding("s3", None, binding("model-c", None), None)
             .await;
 
-        assert!(svc.get_cached_route("s1", None).await.is_none());
-        assert!(svc.get_cached_route("s2", None).await.is_some());
-        assert!(svc.get_cached_route("s3", None).await.is_some());
+        assert!(svc.get_binding("s1", None).await.is_none());
+        assert!(svc.get_binding("s2", None).await.is_some());
+        assert!(svc.get_binding("s3", None).await.is_some());
     }
 
     #[tokio::test]
     async fn test_cache_update_existing_session_does_not_evict() {
         let svc = make_orchestrator_service(600, 2);
-        svc.cache_route("s1".to_string(), None, "model-a".to_string(), None)
+        svc.store_binding("s1", None, binding("model-a", None), None)
             .await;
-        svc.cache_route("s2".to_string(), None, "model-b".to_string(), None)
+        svc.store_binding("s2", None, binding("model-b", None), None)
             .await;
 
-        svc.cache_route(
-            "s1".to_string(),
+        svc.store_binding("s1", None, binding("model-a-updated", Some("route")), None)
+            .await;
+
+        let s1 = svc.get_binding("s1", None).await.unwrap();
+        assert_eq!(s1.anchor_model, "model-a-updated");
+        assert!(svc.get_binding("s2", None).await.is_some());
+    }
+
+    #[tokio::test]
+    async fn test_gc_ttl_override_extends_binding_lifetime() {
+        // Global GC bound of 0 would reclaim immediately; the per-call override keeps it.
+        let svc = make_orchestrator_service(0, 100);
+        svc.store_binding(
+            "s1",
             None,
-            "model-a-updated".to_string(),
-            Some("route".to_string()),
+            binding("gpt-4o", None),
+            Some(Duration::from_secs(600)),
         )
         .await;
+        let cached = svc.get_binding("s1", None).await.unwrap();
+        assert_eq!(cached.anchor_model, "gpt-4o");
+    }
 
-        let s1 = svc.get_cached_route("s1", None).await.unwrap();
-        assert_eq!(s1.model_name, "model-a-updated");
-        assert!(svc.get_cached_route("s2", None).await.is_some());
+    #[tokio::test]
+    async fn test_binding_fields_round_trip_through_cache() {
+        let svc = make_orchestrator_service(600, 100);
+        let mut b = binding("gpt-4o", None);
+        b.prefix_hash = Some(0xdead_beef);
+        b.cached_tokens = 12_345;
+        b.baseline_usd = 1.5;
+        b.switch_spend_usd = 0.42;
+        b.switches = 3;
+        b.session_cost_usd = 2.75;
+        svc.store_binding("s1", None, b, None).await;
+
+        let cached = svc.get_binding("s1", None).await.unwrap();
+        assert_eq!(cached.prefix_hash, Some(0xdead_beef));
+        assert_eq!(cached.cached_tokens, 12_345);
+        assert!((cached.baseline_usd - 1.5).abs() < 1e-9);
+        assert!((cached.switch_spend_usd - 0.42).abs() < 1e-9);
+        assert_eq!(cached.switches, 3);
+        assert!((cached.session_cost_usd - 2.75).abs() < 1e-9);
+    }
+
+    // ---- switch-cost math ----
+    //
+    // Real models.dev rates (USD per million input tokens):
+    //   claude-opus-4-1:   input 15,  cache_read 1.5
+    //   claude-sonnet-4-5: input 3,   cache_read 0.3
+    //   claude-haiku-4-5:  input 1,   cache_read 0.1
+    //   gpt-4.1:           input 2,   cache_read 0.5
+
+    #[test]
+    fn negative_cost_when_candidate_undercuts_cached_rate() {
+        // Anchor opus (cached 1.5) -> haiku (uncached 1.0) over 100k context, cold
+        // candidate: cost = 0.1M x (1.0 - 1.5) = -$0.05 — cheaper even after re-reading.
+        let cost = switch_cost_in_usd(100_000, 0, 1.5, 1.0, 0.1);
+        assert!((cost - (-0.05)).abs() < 1e-9);
+    }
+
+    #[test]
+    fn positive_cost_when_candidate_pricier_than_cached_rate() {
+        // Anchor opus (cached 1.5) -> gpt-4.1 (uncached 2.0) over 100k, cold candidate:
+        // cost = 0.1M x (2.0 - 1.5) = +$0.05.
+        let cost = switch_cost_in_usd(100_000, 0, 1.5, 2.0, 0.5);
+        assert!((cost - 0.05).abs() < 1e-9);
+    }
+
+    #[test]
+    fn large_context_amplifies_cost() {
+        // Anchor sonnet (cached 0.3) -> gpt-5.5-class (uncached 5.0) over 150k, cold:
+        // cost = 0.15M x (5.0 - 0.3) = +$0.705.
+        let cost = switch_cost_in_usd(150_000, 0, 0.3, 5.0, 0.5);
+        assert!((cost - 0.705).abs() < 1e-9);
+    }
+
+    #[test]
+    fn cost_scales_linearly_with_context() {
+        let small = switch_cost_in_usd(10_000, 0, 0.3, 0.8, 0.1);
+        let large = switch_cost_in_usd(1_000_000, 0, 0.3, 0.8, 0.1);
+        assert!((large / small - 100.0).abs() < 1e-6);
+    }
+
+    #[test]
+    fn tiny_context_cost_is_negligible() {
+        // 2k-token chat: even an expensive candidate costs ~$0.009.
+        let cost = switch_cost_in_usd(2_000, 0, 0.3, 5.0, 0.5);
+        assert!(cost < 0.01);
+    }
+
+    #[test]
+    fn warm_return_charges_only_the_delta() {
+        // Anchor sonnet (cached 0.3). Candidate gpt-4.1 (uncached 2.0, cached 0.5) is
+        // still warm from an earlier visit holding 90k of the 100k context. Only the
+        // 10k fresh tokens re-read at 2.0; the 90k warm tokens re-read at 0.5:
+        //   candidate = (10k x 2.0 + 90k x 0.5)/1M = 0.020 + 0.045 = $0.065
+        //   anchor    = 100k x 0.3 /1M            = $0.030
+        //   switch    = 0.065 - 0.030            = +$0.035
+        let warm = switch_cost_in_usd(100_000, 90_000, 0.3, 2.0, 0.5);
+        assert!(
+            (warm - 0.035).abs() < 1e-9,
+            "warm-return cost {warm} != 0.035"
+        );
+        // Cold, same switch re-reads the whole 100k at 2.0: (100k x 2.0)/1M - 0.03 = $0.17.
+        let cold = switch_cost_in_usd(100_000, 0, 0.3, 2.0, 0.5);
+        assert!((cold - 0.17).abs() < 1e-9, "cold cost {cold} != 0.17");
+        assert!(warm < cold, "returning to a warm model must be cheaper");
     }
 }

crates/brightstaff/src/session_cache/memory.rs

@@ -9,9 +9,9 @@ use lru::LruCache;
 use tokio::sync::Mutex;
 use tracing::info;
 
-use super::{CachedRoute, SessionCache};
+use super::{SessionBinding, SessionCache};
 
-type CacheStore = Mutex<LruCache<String, (CachedRoute, Instant, Duration)>>;
+type CacheStore = Mutex<LruCache<String, (SessionBinding, Instant, Duration)>>;
 
 pub struct MemorySessionCache {
     store: Arc<CacheStore>,
@@ -38,6 +38,8 @@ impl MemorySessionCache {
 
     async fn evict_expired(store: &CacheStore) {
         let mut cache = store.lock().await;
+        // The TTL is only a GC bound: drop bindings once they've outlived the window
+        // in which they could plausibly still be warm.
         let expired: Vec<String> = cache
             .iter()
             .filter(|(_, (_, inserted_at, ttl))| inserted_at.elapsed() >= *ttl)
@@ -59,21 +61,21 @@ impl MemorySessionCache {
 
 #[async_trait]
 impl SessionCache for MemorySessionCache {
-    async fn get(&self, key: &str) -> Option<CachedRoute> {
+    async fn get(&self, key: &str) -> Option<SessionBinding> {
         let mut cache = self.store.lock().await;
-        if let Some((route, inserted_at, ttl)) = cache.get(key) {
+        if let Some((binding, inserted_at, ttl)) = cache.get(key) {
             if inserted_at.elapsed() < *ttl {
-                return Some(route.clone());
+                return Some(binding.clone());
             }
         }
         None
     }
 
-    async fn put(&self, key: &str, route: CachedRoute, ttl: Duration) {
+    async fn put(&self, key: &str, binding: SessionBinding, ttl: Duration) {
         self.store
             .lock()
             .await
-            .put(key.to_string(), (route, Instant::now(), ttl));
+            .put(key.to_string(), (binding, Instant::now(), ttl));
     }
 
     async fn remove(&self, key: &str) {

crates/brightstaff/src/session_cache/mod.rs

@@ -1,4 +1,5 @@
 use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
 
 use async_trait::async_trait;
 use common::configuration::Configuration;
@@ -8,21 +9,155 @@ use tracing::{debug, info};
 pub mod memory;
 pub mod redis;
 
+/// A conversation's binding to a model, plus the state the session router needs to
+/// reason about cache warmth and switch affordability across turns.
+///
+/// Warmth is no longer derived from the cache's own expiry — the entry is kept alive
+/// as a plain KV value (subject only to a GC bound) and the router decides warmth from
+/// [`SessionBinding::last_used`] against the provider's cache window. This is what lets
+/// the decision path reason about warmth without ever seeing a provider response.
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-pub struct CachedRoute {
-    pub model_name: String,
+pub struct SessionBinding {
+    /// Provider-qualified model that handled the latest request (e.g. `openai/gpt-4o`).
+    /// This is what the session is currently *warm on*; a proposed switch's cost is
+    /// measured against reading the context at this model's cached rate. It tracks the
+    /// last dispatched model and changes whenever a switch is honored.
+    pub anchor_model: String,
+    /// Provider-qualified model the session started on this warm episode — the model it
+    /// would have stayed on had it *never switched*. The never-switch baseline is priced
+    /// against this (not `anchor_model`, which drifts as switches happen). Set when a warm
+    /// episode begins and preserved across its turns.
+    #[serde(default)]
+    pub default_model: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub route_name: Option<String>,
+    /// Hash of the stable prompt prefix (system + tools) observed when the binding was
+    /// stored. Used to detect prefix drift: if a later request's prefix hash differs,
+    /// the provider cache is already lost so a switch is free.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub prefix_hash: Option<u64>,
+    /// When this session was last dispatched. Warmth = `now - last_used` compared
+    /// against the provider's idle/hard cache window.
+    #[serde(default = "SystemTime::now", with = "epoch_secs")]
+    pub last_used: SystemTime,
+    /// Best estimate of the cacheable context size (input tokens) — the tokens a switch
+    /// would have to re-ingest at the uncached rate. Refined from real usage on the
+    /// full-proxy path; the tokenizer estimate on the decision path.
+    #[serde(default)]
+    pub cached_tokens: u64,
+    /// Cumulative *never-switch* baseline (USD) for this warm episode: the running cost
+    /// the session would have paid by staying on its `default_model`. Grows each warm
+    /// turn. This is the denominator the percentage overhead cap is measured against.
+    #[serde(default)]
+    pub baseline_usd: f64,
+    /// Cumulative overhead (USD) actually spent on paid switches this warm episode.
+    /// Monotonic: paid switches add to it, free/cheaper switches never subtract. A paid
+    /// switch is allowed only while `switch_spend_usd + cost <= pct * baseline_usd`.
+    #[serde(default)]
+    pub switch_spend_usd: f64,
+    /// Number of model switches taken during this warm session (observability).
+    #[serde(default)]
+    pub switches: u32,
+    /// Bounded most-recent-first-ish record of the models this session has been
+    /// dispatched to, each with when it was last used and how large the context was
+    /// then. Lets the switch-cost estimate credit a *return* to a still-warm model (it
+    /// re-reads only the tokens appended since, not the whole context) and gives future
+    /// routing policies per-model recency to reason about. Capped at
+    /// [`MAX_ROUTE_HISTORY`] distinct models (LRU-evicted).
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub history: Vec<RouteVisit>,
+    /// Cumulative *actual* cost (USD, input + output) of the whole conversation, priced
+    /// from the configured catalog rates and refined from real usage each turn on the
+    /// full-proxy path. Conversation-level (not per warm episode): it persists across
+    /// cold re-binds and is best-effort (resets only if the binding is evicted).
+    #[serde(default)]
+    pub session_cost_usd: f64,
+}
+
+/// One model this session has been dispatched to, with the recency and context size
+/// needed to estimate whether its provider cache is still warm on a later return.
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+pub struct RouteVisit {
+    /// Provider-qualified model id (e.g. `openai/gpt-4o`).
+    pub model: String,
+    /// When this model last handled a turn in this session.
+    #[serde(with = "epoch_secs")]
+    pub last_used: SystemTime,
+    /// Context size (input tokens) that model saw on its last turn — the prefix it may
+    /// still have cached if it's revisited before its cache window elapses.
+    #[serde(default)]
+    pub cached_tokens: u64,
+}
+
+/// Max distinct models retained in [`SessionBinding::history`]. Small: real sessions
+/// touch a handful of models, and the entry has to stay compact on the Redis wire.
+pub const MAX_ROUTE_HISTORY: usize = 8;
+
+/// Record (or refresh) a model visit in `history`, then LRU-evict down to
+/// [`MAX_ROUTE_HISTORY`]. Refreshing an existing model updates its recency and context
+/// size in place rather than appending a duplicate.
+pub fn record_route_visit(
+    history: &mut Vec<RouteVisit>,
+    model: &str,
+    last_used: SystemTime,
+    cached_tokens: u64,
+) {
+    if let Some(entry) = history.iter_mut().find(|e| e.model == model) {
+        entry.last_used = last_used;
+        entry.cached_tokens = cached_tokens;
+    } else {
+        history.push(RouteVisit {
+            model: model.to_string(),
+            last_used,
+            cached_tokens,
+        });
+    }
+    while history.len() > MAX_ROUTE_HISTORY {
+        if let Some(idx) = history
+            .iter()
+            .enumerate()
+            .min_by_key(|(_, e)| e.last_used)
+            .map(|(i, _)| i)
+        {
+            history.remove(idx);
+        } else {
+            break;
+        }
+    }
+}
+
+/// Serde helper: persist `SystemTime` as whole epoch seconds so the Redis wire format
+/// is stable and compact (the default `SystemTime` representation is version-fragile).
+mod epoch_secs {
+    use super::{Duration, SystemTime, UNIX_EPOCH};
+    use serde::{Deserialize, Deserializer, Serializer};
+
+    pub fn serialize<S: Serializer>(t: &SystemTime, s: S) -> Result<S::Ok, S::Error> {
+        let secs = t
+            .duration_since(UNIX_EPOCH)
+            .map(|d| d.as_secs())
+            .unwrap_or(0);
+        s.serialize_u64(secs)
+    }
+
+    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<SystemTime, D::Error> {
+        let secs = u64::deserialize(d)?;
+        Ok(UNIX_EPOCH + Duration::from_secs(secs))
+    }
 }
 
 #[async_trait]
 pub trait SessionCache: Send + Sync {
-    /// Look up a cached routing decision by key.
-    async fn get(&self, key: &str) -> Option<CachedRoute>;
+    /// Look up a session binding by key. `None` when absent or GC-evicted. Warmth is
+    /// the caller's concern (time since `last_used`), not the cache's.
+    async fn get(&self, key: &str) -> Option<SessionBinding>;
 
-    /// Store a routing decision in the session cache with the given TTL.
-    async fn put(&self, key: &str, route: CachedRoute, ttl: Duration);
+    /// Store a session binding with the given GC TTL. The TTL is only a memory bound
+    /// (keep the entry around at least as long as it could plausibly be warm); it does
+    /// not define warmth.
+    async fn put(&self, key: &str, binding: SessionBinding, ttl: Duration);
 
-    /// Remove a cached routing decision by key.
+    /// Remove a session binding by key.
     async fn remove(&self, key: &str);
 }
 

crates/brightstaff/src/session_cache/redis.rs

@@ -4,7 +4,7 @@ use async_trait::async_trait;
 use redis::aio::MultiplexedConnection;
 use redis::AsyncCommands;
 
-use super::{CachedRoute, SessionCache};
+use super::{SessionBinding, SessionCache};
 
 const KEY_PREFIX: &str = "plano:affinity:";
 
@@ -26,18 +26,20 @@ impl RedisSessionCache {
 
 #[async_trait]
 impl SessionCache for RedisSessionCache {
-    async fn get(&self, key: &str) -> Option<CachedRoute> {
+    async fn get(&self, key: &str) -> Option<SessionBinding> {
         let mut conn = self.conn.clone();
         let value: Option<String> = conn.get(Self::make_key(key)).await.ok()?;
         value.and_then(|v| serde_json::from_str(&v).ok())
     }
 
-    async fn put(&self, key: &str, route: CachedRoute, ttl: Duration) {
+    async fn put(&self, key: &str, binding: SessionBinding, ttl: Duration) {
         let mut conn = self.conn.clone();
-        let Ok(json) = serde_json::to_string(&route) else {
+        // The Redis TTL is only a GC bound; warmth is decided by the router from
+        // `binding.last_used`, not by expiry here.
+        let ttl_secs = ttl.as_secs().max(1);
+        let Ok(json) = serde_json::to_string(&binding) else {
             return;
         };
-        let ttl_secs = ttl.as_secs().max(1);
         let _: Result<(), _> = conn.set_ex(Self::make_key(key), json, ttl_secs).await;
     }
 

crates/brightstaff/src/signals/otel.rs

@@ -1,27 +1,21 @@
 //! Helpers for emitting `SignalReport` data to OpenTelemetry spans.
 //!
-//! Two sets of attributes are emitted:
-//!
-//! - **Legacy** keys under `signals.*` (e.g. `signals.frustration.count`),
-//!   computed from the new layered counts. Preserved for one release for
-//!   backward compatibility with existing dashboards.
-//! - **New** layered keys (e.g. `signals.interaction.misalignment.count`),
+//! Layered keys (e.g. `signals.interaction.misalignment.count`) are emitted,
 //! one set of `count`/`severity` attributes per category, plus per-instance
 //! span events named `signal.<dotted_signal_type>`.
 
 use opentelemetry::trace::SpanRef;
 use opentelemetry::KeyValue;
 
-use crate::signals::schemas::{SignalGroup, SignalReport, SignalType};
+use crate::signals::schemas::{SignalGroup, SignalReport};
 
-/// Emit both legacy and layered OTel attributes/events for a `SignalReport`.
+/// Emit layered OTel attributes/events for a `SignalReport`.
 ///
 /// Returns `true` if any "concerning" signal was found, mirroring the previous
 /// behavior used to flag the span operation name.
 pub fn emit_signals_to_span(span: &SpanRef<'_>, report: &SignalReport) -> bool {
     emit_overall(span, report);
     emit_layered_attributes(span, report);
-    emit_legacy_attributes(span, report);
     emit_signal_events(span, report);
 
     is_concerning(report)
@@ -90,69 +84,6 @@ fn emit_layered_attributes(span: &SpanRef<'_>, report: &SignalReport) {
     );
 }
 
-fn count_of(report: &SignalReport, t: SignalType) -> usize {
-    report.iter_signals().filter(|s| s.signal_type == t).count()
-}
-
-/// Emit the legacy attribute keys consumed by existing dashboards. These are
-/// derived from the new `SignalReport` so no detector contract is broken.
-fn emit_legacy_attributes(span: &SpanRef<'_>, report: &SignalReport) {
-    use crate::tracing::signals as legacy;
-
-    // signals.follow_up.repair.{count,ratio} - misalignment proxies repairs.
-    let repair_count = report.interaction.misalignment.count;
-    let user_turns = report.turn_metrics.user_turns.max(1) as f32;
-    if repair_count > 0 {
-        span.set_attribute(KeyValue::new(legacy::REPAIR_COUNT, repair_count as i64));
-        let ratio = repair_count as f32 / user_turns;
-        span.set_attribute(KeyValue::new(legacy::REPAIR_RATIO, format!("{:.3}", ratio)));
-    }
-
-    // signals.frustration.{count,severity} - disengagement.negative_stance is
-    // the closest legacy analog of "frustration".
-    let frustration_count = count_of(report, SignalType::DisengagementNegativeStance);
-    if frustration_count > 0 {
-        span.set_attribute(KeyValue::new(
-            legacy::FRUSTRATION_COUNT,
-            frustration_count as i64,
-        ));
-        let severity = match frustration_count {
-            0 => 0,
-            1..=2 => 1,
-            3..=4 => 2,
-            _ => 3,
-        };
-        span.set_attribute(KeyValue::new(legacy::FRUSTRATION_SEVERITY, severity as i64));
-    }
-
-    // signals.repetition.count - stagnation (repetition + dragging).
-    if report.interaction.stagnation.count > 0 {
-        span.set_attribute(KeyValue::new(
-            legacy::REPETITION_COUNT,
-            report.interaction.stagnation.count as i64,
-        ));
-    }
-
-    // signals.escalation.requested - any escalation/quit signal.
-    let escalated = report.interaction.disengagement.signals.iter().any(|s| {
-        matches!(
-            s.signal_type,
-            SignalType::DisengagementEscalation | SignalType::DisengagementQuit
-        )
-    });
-    if escalated {
-        span.set_attribute(KeyValue::new(legacy::ESCALATION_REQUESTED, true));
-    }
-
-    // signals.positive_feedback.count - satisfaction signals.
-    if report.interaction.satisfaction.count > 0 {
-        span.set_attribute(KeyValue::new(
-            legacy::POSITIVE_FEEDBACK_COUNT,
-            report.interaction.satisfaction.count as i64,
-        ));
-    }
-}
-
 fn emit_signal_events(span: &SpanRef<'_>, report: &SignalReport) {
     for sig in report.iter_signals() {
         let event_name = format!("signal.{}", sig.signal_type.as_str());
@@ -231,11 +162,4 @@ mod tests {
         let r = report_with_escalation();
         assert!(is_concerning(&r));
     }
-
-    #[test]
-    fn count_of_returns_per_type_count() {
-        let r = report_with_escalation();
-        assert_eq!(count_of(&r, SignalType::DisengagementEscalation), 1);
-        assert_eq!(count_of(&r, SignalType::DisengagementNegativeStance), 0);
-    }
 }

crates/brightstaff/src/streaming.rs

@@ -22,10 +22,15 @@ const STREAM_BUFFER_SIZE: usize = 16;
 const USAGE_BUFFER_MAX: usize = 2 * 1024 * 1024;
 use crate::metrics as bs_metrics;
 use crate::metrics::labels as metric_labels;
+use crate::router::model_metrics::ModelRates;
+use crate::router::orchestrator::OrchestratorService;
+use crate::session_cache::{record_route_visit, RouteVisit, SessionBinding};
 use crate::signals::otel::emit_signals_to_span;
 use crate::signals::{SignalAnalyzer, FLAG_MARKER};
 use crate::tracing::{llm, set_service_name};
 use hermesllm::apis::openai::Message;
+use std::sync::Arc;
+use std::time::{Duration, SystemTime};
 
 /// Parsed usage + resolved-model details from a provider response.
 #[derive(Debug, Default, Clone)]
@@ -39,6 +44,11 @@ struct ExtractedUsage {
     /// The model the upstream actually used. For router aliases (e.g.
     /// `router:software-engineering`), this differs from the request model.
     resolved_model: Option<String>,
+    /// Provider convention for `prompt_tokens`: OpenAI-shape usage folds cached tokens
+    /// *into* `prompt_tokens` (so uncached = prompt - cached), while Anthropic-shape
+    /// reports uncached `input_tokens` separately from `cache_read`/`cache_creation`.
+    /// Set at parse time from which field `prompt_tokens` was sourced. Drives cost math.
+    prompt_includes_cached: bool,
 }
 
 impl ExtractedUsage {
@@ -57,8 +67,9 @@ impl ExtractedUsage {
             }
         }
         if let Some(u) = value.get("usage") {
-            // OpenAI-shape usage
+            // OpenAI-shape usage: `prompt_tokens` includes the cached subset.
             out.prompt_tokens = u.get("prompt_tokens").and_then(|v| v.as_i64());
+            out.prompt_includes_cached = out.prompt_tokens.is_some();
             out.completion_tokens = u.get("completion_tokens").and_then(|v| v.as_i64());
             out.total_tokens = u.get("total_tokens").and_then(|v| v.as_i64());
             out.cached_input_tokens = u
@@ -187,6 +198,44 @@ pub struct LlmMetricsCtx {
     pub upstream_status: u16,
 }
 
+/// Response-side session-update context: refreshes the session binding once the real
+/// response is in hand, so `last_used` and the context-size estimate (`cached_tokens`)
+/// reflect the turn that just completed. The routing decision (model, budget, switches)
+/// was already made and persisted on the request side by [`super::handlers::llm::session_router`];
+/// this only refines the fields that need the response.
+pub struct SessionUpdateCtx {
+    pub orchestrator: Arc<OrchestratorService>,
+    pub session_id: String,
+    pub tenant_id: Option<String>,
+    /// Provider-qualified model this request actually ran on (the router's final pick).
+    pub anchor_model: String,
+    /// The session's never-switch model for this episode — preserved across the refresh.
+    pub default_model: String,
+    pub route_name: Option<String>,
+    pub prefix_hash: Option<u64>,
+    /// Cumulative never-switch baseline from the decision — preserved across the refresh.
+    pub baseline_usd: f64,
+    /// Cumulative switch spend from the decision — preserved across the refresh.
+    pub switch_spend_usd: f64,
+    /// Cumulative switch count from the routing decision — preserved across the refresh.
+    pub switches: u32,
+    /// Per-model route history from the decision — preserved across the refresh, with
+    /// the anchor's entry refined to the real prompt-token count.
+    pub history: Vec<RouteVisit>,
+    /// Cumulative actual conversation cost (USD) through prior turns. This turn's real
+    /// cost is added on top once usage is known.
+    pub session_cost_usd: f64,
+    /// Catalog rates for the dispatched model, resolved request-side (the response path
+    /// is synchronous). `None` when no cost source is configured → no cost is computed.
+    pub cost_rates: Option<ModelRates>,
+    /// Cached-read discount used to price cached input when the feed omits a cached rate.
+    pub cache_read_discount: f64,
+    /// Context-size count to fall back to when the response carries no usage block.
+    pub context_tokens: u64,
+    /// GC bound to store the refreshed binding with.
+    pub gc_ttl: Duration,
+}
+
 /// A processor that tracks streaming metrics
 pub struct ObservableStreamProcessor {
     service_name: String,
@@ -201,6 +250,7 @@ pub struct ObservableStreamProcessor {
     /// from the buffer (they still pass through to the client).
     response_buffer: Vec<u8>,
     llm_metrics: Option<LlmMetricsCtx>,
+    session_update: Option<SessionUpdateCtx>,
     metrics_recorded: bool,
 }
 
@@ -237,6 +287,7 @@ impl ObservableStreamProcessor {
             messages,
             response_buffer: Vec::new(),
             llm_metrics: None,
+            session_update: None,
             metrics_recorded: false,
         }
     }
@@ -247,6 +298,103 @@ impl ObservableStreamProcessor {
         self.llm_metrics = Some(ctx);
         self
     }
+
+    /// Attach session-update context so the processor refreshes `last_used` and the
+    /// context-size estimate from the real response once usage is known.
+    pub fn with_session_update(mut self, ctx: SessionUpdateCtx) -> Self {
+        self.session_update = Some(ctx);
+        self
+    }
+
+    /// Refresh the session binding from the response so warmth (`last_used`) and the
+    /// context-size estimate (`cached_tokens`) reflect the completed turn.
+    fn handle_session_update(&mut self, usage: &ExtractedUsage) {
+        let Some(update) = self.session_update.take() else {
+            return;
+        };
+        let SessionUpdateCtx {
+            orchestrator,
+            session_id,
+            tenant_id,
+            anchor_model,
+            default_model,
+            route_name,
+            prefix_hash,
+            baseline_usd,
+            switch_spend_usd,
+            switches,
+            mut history,
+            session_cost_usd,
+            cost_rates,
+            cache_read_discount,
+            context_tokens,
+            gc_ttl,
+        } = update;
+
+        // Prefer the real prompt-token count (the tokens a future switch would re-read)
+        // over the request-side estimate; fall back to the estimate when absent.
+        let cached_tokens = usage
+            .prompt_tokens
+            .filter(|&p| p > 0)
+            .map(|p| p as u64)
+            .unwrap_or(context_tokens);
+
+        // Refine the anchor's route-history entry with the real context size, so a later
+        // return to this model prices its warm portion off actual usage.
+        record_route_visit(
+            &mut history,
+            &anchor_model,
+            SystemTime::now(),
+            cached_tokens,
+        );
+
+        // Price this turn from the catalog rates and roll it into the conversation
+        // total. Emit per-request cost on the (llm) span and carry the running total
+        // into the binding so the next turn's routing span can surface it.
+        let session_cost_usd = if let Some(rates) = cost_rates {
+            let (input_cost, output_cost) = rates.request_cost_usd(
+                usage.prompt_tokens.unwrap_or(0).max(0) as u64,
+                usage.cached_input_tokens.unwrap_or(0).max(0) as u64,
+                usage.cache_creation_tokens.unwrap_or(0).max(0) as u64,
+                usage.completion_tokens.unwrap_or(0).max(0) as u64,
+                usage.prompt_includes_cached,
+                cache_read_discount,
+            );
+            let span = tracing::Span::current();
+            let otel_span = span.context();
+            let otel_span = otel_span.span();
+            otel_span.set_attribute(KeyValue::new(llm::INPUT_COST_IN_USD, input_cost));
+            otel_span.set_attribute(KeyValue::new(llm::OUTPUT_COST_IN_USD, output_cost));
+            otel_span.set_attribute(KeyValue::new(
+                llm::TOTAL_COST_IN_USD,
+                input_cost + output_cost,
+            ));
+            session_cost_usd + input_cost + output_cost
+        } else {
+            session_cost_usd
+        };
+
+        bs_metrics::record_session_binding_event(metric_labels::BINDING_EVENT_REFRESH);
+        let binding = SessionBinding {
+            anchor_model,
+            default_model,
+            route_name,
+            prefix_hash,
+            last_used: SystemTime::now(),
+            cached_tokens,
+            baseline_usd,
+            switch_spend_usd,
+            switches,
+            session_cost_usd,
+            history,
+        };
+        // Fire-and-forget: binding bookkeeping must not delay stream completion.
+        tokio::spawn(async move {
+            orchestrator
+                .store_binding(&session_id, tenant_id.as_deref(), binding, Some(gc_ttl))
+                .await;
+        });
+    }
 }
 
 impl StreamProcessor for ObservableStreamProcessor {
@@ -357,19 +505,53 @@ impl StreamProcessor for ObservableStreamProcessor {
                     v.max(0) as u64,
                 );
             }
+            // Prompt-cache token counters + hit/miss baseline (cache-blindness is
+            // invisible without these: a reroute that burns a warm cache shows up
+            // here as a miss with zero cache_read tokens).
+            if let Some(v) = usage.cached_input_tokens {
+                bs_metrics::record_llm_tokens(
+                    &ctx.provider,
+                    &ctx.model,
+                    metric_labels::TOKEN_KIND_CACHE_READ,
+                    v.max(0) as u64,
+                );
+            }
+            if let Some(v) = usage.cache_creation_tokens {
+                bs_metrics::record_llm_tokens(
+                    &ctx.provider,
+                    &ctx.model,
+                    metric_labels::TOKEN_KIND_CACHE_WRITE,
+                    v.max(0) as u64,
+                );
+            }
+            if usage.prompt_tokens.is_some() {
+                let cache_active = usage.cached_input_tokens.unwrap_or(0) > 0
+                    || usage.cache_creation_tokens.unwrap_or(0) > 0;
+                bs_metrics::record_prompt_cache_outcome(
+                    &ctx.provider,
+                    &ctx.model,
+                    if cache_active {
+                        metric_labels::PROMPT_CACHE_HIT
+                    } else {
+                        metric_labels::PROMPT_CACHE_MISS
+                    },
+                );
+            }
             if usage.prompt_tokens.is_none() && usage.completion_tokens.is_none() {
                 bs_metrics::record_llm_tokens_usage_missing(&ctx.provider, &ctx.model);
             }
             self.metrics_recorded = true;
         }
+
+        // Session-binding refresh: update `last_used` + the context-size estimate from
+        // the completed turn (the routing decision itself was made request-side).
+        self.handle_session_update(&usage);
         // Release the buffered bytes early; nothing downstream needs them.
         self.response_buffer.clear();
         self.response_buffer.shrink_to_fit();
 
         // Analyze signals if messages are available and record as span
-        // attributes + per-signal events. We dual-emit legacy aggregate keys
-        // and the new layered taxonomy so existing dashboards keep working
-        // while new consumers can opt into the richer hierarchy.
+        // attributes + per-signal events using the layered signal taxonomy.
         if let Some(ref messages) = self.messages {
             let analyzer = SignalAnalyzer::default();
             let report = analyzer.analyze_openai(messages);
@@ -614,6 +796,8 @@ mod usage_extraction_tests {
         assert_eq!(u.total_tokens, Some(46));
         assert_eq!(u.cached_input_tokens, Some(5));
         assert_eq!(u.reasoning_tokens, None);
+        // OpenAI folds cached tokens into `prompt_tokens`.
+        assert!(u.prompt_includes_cached);
     }
 
     #[test]
@@ -625,6 +809,8 @@ mod usage_extraction_tests {
         assert_eq!(u.total_tokens, Some(150));
         assert_eq!(u.cached_input_tokens, Some(30));
         assert_eq!(u.cache_creation_tokens, Some(20));
+        // Anthropic reports uncached `input_tokens` separately from cached reads.
+        assert!(!u.prompt_includes_cached);
     }
 
     #[test]

crates/brightstaff/src/tracing/constants.rs

@@ -92,6 +92,18 @@ pub mod llm {
     /// (OpenAI `completion_tokens_details.reasoning_tokens`, Google `thoughts_token_count`)
     pub const REASONING_TOKENS: &str = "llm.usage.reasoning_tokens";
 
+    /// This request's input-token cost (USD), priced from the catalog rates: uncached
+    /// input at the input rate, cached reads at the cached rate, cache creation at the
+    /// plain input rate. Present only when a cost source is configured.
+    pub const INPUT_COST_IN_USD: &str = "llm.usage.input_cost_usd";
+
+    /// This request's output-token cost (USD) = completion tokens x output rate.
+    pub const OUTPUT_COST_IN_USD: &str = "llm.usage.output_cost_usd";
+
+    /// This request's total cost (USD) = input + output. Sum across a session's turns
+    /// (group by `plano.session_id`) for the conversation total.
+    pub const TOTAL_COST_IN_USD: &str = "llm.usage.total_cost_usd";
+
     /// Temperature parameter used
     pub const TEMPERATURE: &str = "llm.temperature";
 
@@ -145,6 +157,66 @@ pub mod plano {
     /// "software-engineering"). Absent when the client routed directly
     /// to a concrete model.
     pub const ROUTE_NAME: &str = "plano.route.name";
+
+    /// Caller identity used to populate downstream observability `distinct_id`
+    /// fields (e.g. PostHog). Sourced from the configured
+    /// `tracing.exporters[].distinct_id_header`. Absent for anonymous calls.
+    pub const DISTINCT_ID: &str = "plano.distinct_id";
+
+    /// Whether the session's provider cache was inferred warm at decision time
+    /// (from the idle gap vs. the provider's cache window).
+    pub const CACHE_WARM: &str = "plano.cache.warm";
+
+    /// How long (ms) since the session was last used — the idle gap warmth is measured
+    /// against.
+    pub const CACHE_IDLE_MS: &str = "plano.cache.idle_ms";
+
+    /// Cumulative switching overhead consumed this session, as a percentage of the
+    /// never-switch baseline (`100 * switch_spend / baseline`). Directly comparable to
+    /// the configured `routing.routing_budget.max_overhead_pct`.
+    pub const SESSION_OVERHEAD_PCT: &str = "plano.session.overhead_pct";
+
+    /// Cumulative overhead (USD) actually spent on paid switches this session — the
+    /// numerator behind `plano.session.overhead_pct`.
+    pub const SESSION_SWITCH_SPEND_IN_USD: &str = "plano.session.switch_spend_in_usd";
+
+    /// Cumulative never-switch baseline (USD) — what staying on the anchor would have
+    /// cost so far. The denominator behind `plano.session.overhead_pct`.
+    pub const SESSION_BASELINE_IN_USD: &str = "plano.session.baseline_in_usd";
+
+    /// Cumulative number of model switches taken during this warm session.
+    pub const SESSION_SWITCHES: &str = "plano.session.switches";
+
+    /// Cumulative *actual* cost (USD, input + output) of the whole conversation, priced
+    /// from the configured catalog rates and refined from real usage each turn. Emitted
+    /// on the routing span; reflects cost through the previous turn (this turn isn't
+    /// billed yet at decision time). Full-proxy path only.
+    pub const SESSION_TOTAL_COST_IN_USD: &str = "plano.session.total_cost_in_usd";
+
+    /// Actual input-cost (USD) of the proposed model switch — computed from input-token
+    /// pricing only, output-token cost deliberately excluded. Negative when the candidate
+    /// is outright cheaper than staying on the warm anchor.
+    pub const SWITCH_COST_IN_USD: &str = "plano.switch.cost_in_usd";
+
+    /// Tokens the switch candidate still has cached from an earlier visit this session
+    /// (a return to a still-warm model). These re-read at the candidate's cached rate
+    /// instead of its uncached rate, which is why the switch cost can be far below a
+    /// full re-ingest. Zero for a first-time (cold) switch.
+    pub const SWITCH_CANDIDATE_WARM_TOKENS: &str = "plano.switch.candidate_warm_tokens";
+
+    /// The overhead ceiling (USD) available when the switch was evaluated —
+    /// `max_overhead_pct% * baseline`. A paid switch is allowed while cumulative spend
+    /// plus this switch's cost stays under it. Directly comparable to `cost_in_usd`.
+    pub const SWITCH_OVERHEAD_CEILING_IN_USD: &str = "plano.switch.overhead_ceiling_in_usd";
+
+    /// Switch outcome: "allowed" or "retained".
+    pub const SWITCH_DECISION: &str = "plano.switch.decision";
+
+    /// The route (`provider/model`, plus route name when routed) the routing-budget gate
+    /// *would* have selected had the switch been allowed. Recorded only on a `retained`
+    /// decision when `routing.routing_budget.record_counterfactual` is enabled.
+    /// Telemetry only — the counterfactual model is never dispatched.
+    pub const SWITCH_COUNTERFACTUAL_ROUTE: &str = "plano.switch.counterfactual_route";
 }
 
 // =============================================================================
@@ -183,27 +255,6 @@ pub mod signals {
 
     /// Efficiency score (0.0-1.0)
     pub const EFFICIENCY_SCORE: &str = "signals.efficiency_score";
-
-    /// Number of repair attempts detected
-    pub const REPAIR_COUNT: &str = "signals.follow_up.repair.count";
-
-    /// Ratio of repairs to user turns
-    pub const REPAIR_RATIO: &str = "signals.follow_up.repair.ratio";
-
-    /// Number of frustration indicators detected
-    pub const FRUSTRATION_COUNT: &str = "signals.frustration.count";
-
-    /// Frustration severity level (0-3)
-    pub const FRUSTRATION_SEVERITY: &str = "signals.frustration.severity";
-
-    /// Number of repetition instances detected
-    pub const REPETITION_COUNT: &str = "signals.repetition.count";
-
-    /// Whether escalation was requested (user asked for human help)
-    pub const ESCALATION_REQUESTED: &str = "signals.escalation.requested";
-
-    /// Number of positive feedback indicators detected
-    pub const POSITIVE_FEEDBACK_COUNT: &str = "signals.positive_feedback.count";
 }
 
 // =============================================================================

crates/brightstaff/src/tracing/init.rs

@@ -11,8 +11,8 @@ use tracing_subscriber::registry::LookupSpan;
 use tracing_subscriber::util::SubscriberInitExt;
 use tracing_subscriber::EnvFilter;
 
-use super::ServiceNameOverrideExporter;
-use common::configuration::Tracing;
+use super::{PostHogExporter, ServiceNameOverrideExporter};
+use common::configuration::{Exporter, PosthogExporter, Tracing};
 
 struct BracketedTime;
 
@@ -90,26 +90,53 @@ pub fn init_tracer(tracing_config: Option<&Tracing>) -> &'static SdkTracerProvid
 
         let random_sampling = tracing_config.and_then(|t| t.random_sampling).unwrap_or(0);
 
-        let tracing_enabled = random_sampling > 0 && otel_endpoint.is_some();
+        // Collect PostHog export destinations from `tracing.exporters`.
+        let posthog_exporters: Vec<PosthogExporter> = tracing_config
+            .and_then(|t| t.exporters.as_ref())
+            .map(|exporters| {
+                exporters
+                    .iter()
+                    .map(|Exporter::Posthog(posthog)| posthog.clone())
+                    .collect()
+            })
+            .unwrap_or_default();
+
+        // Tracing is enabled when sampling is on and there is at least one
+        // destination — an OTLP collector and/or a configured exporter.
+        let has_destination = otel_endpoint.is_some() || !posthog_exporters.is_empty();
+        let tracing_enabled = random_sampling > 0 && has_destination;
         eprintln!(
-            "initializing tracing: tracing_enabled={}, otel_endpoint={:?}, random_sampling={}",
-            tracing_enabled, otel_endpoint, random_sampling
+            "initializing tracing: tracing_enabled={}, otel_endpoint={:?}, random_sampling={}, posthog_exporters={}",
+            tracing_enabled, otel_endpoint, random_sampling, posthog_exporters.len()
         );
 
-        // Create OTLP exporter to send spans to collector.
-        // Use `if let` to destructure the endpoint, avoiding an unwrap.
-        if let Some(endpoint) = otel_endpoint.as_deref().filter(|_| tracing_enabled) {
+        if tracing_enabled {
             if std::env::var("OTEL_SERVICE_NAME").is_err() {
                 std::env::set_var("OTEL_SERVICE_NAME", "plano");
             }
+
+            // Compose the tracer provider from all configured destinations. Each
+            // `with_batch_exporter` registers an independent span processor, so
+            // every span fans out to the OTLP collector and every exporter.
+            let mut builder = SdkTracerProvider::builder();
+
             // Create ServiceNameOverrideExporter to support per-span service names
             // This allows spans to have different service names (e.g., plano(orchestrator),
             // plano(filter), plano(llm)) by setting the "service.name.override" attribute
-            let exporter = ServiceNameOverrideExporter::new(endpoint);
+            if let Some(endpoint) = otel_endpoint.as_deref() {
+                builder = builder.with_batch_exporter(ServiceNameOverrideExporter::new(endpoint));
+            }
 
-            let provider = SdkTracerProvider::builder()
-                .with_batch_exporter(exporter)
-                .build();
+            // PostHog exporters translate LLM spans into `$ai_generation` events.
+            for posthog in &posthog_exporters {
+                builder = builder.with_batch_exporter(PostHogExporter::new(
+                    &posthog.url,
+                    &posthog.api_key,
+                    posthog.capture_messages.unwrap_or(false),
+                ));
+            }
+
+            let provider = builder.build();
 
             global::set_tracer_provider(provider.clone());
 

crates/brightstaff/src/tracing/mod.rs

@@ -1,6 +1,7 @@
 mod constants;
 mod custom_attributes;
 mod init;
+mod posthog_exporter;
 mod service_name_exporter;
 
 pub use constants::{
@@ -8,6 +9,7 @@ pub use constants::{
 };
 pub use custom_attributes::collect_custom_trace_attributes;
 pub use init::init_tracer;
+pub use posthog_exporter::PostHogExporter;
 pub use service_name_exporter::{ServiceNameOverrideExporter, SERVICE_NAME_OVERRIDE_KEY};
 
 use opentelemetry::trace::get_active_span;

crates/brightstaff/src/tracing/posthog_exporter.rs

@@ -0,0 +1,402 @@
+//! PostHog Span Exporter
+//!
+//! A custom [`SpanExporter`] that translates Plano's LLM spans into PostHog
+//! [`$ai_generation`](https://posthog.com/docs/ai-observability/generations)
+//! events and POSTs them to PostHog's capture API (`{url}/batch/`).
+//!
+//! This makes PostHog a first-class, provider-agnostic export target: a user
+//! only points `tracing.exporters` at their PostHog URL + project token and
+//! every LLM call is captured — mirroring LiteLLM's `posthog` callback.
+//!
+//! # Behaviour
+//!
+//! - Receives every span in the provider (like all batch exporters do) and
+//!   keeps only LLM generation spans, identified by the presence of the
+//!   [`llm::MODEL_NAME`] (`llm.model`) attribute.
+//! - Maps span attributes onto `$ai_*` PostHog properties (model, provider,
+//!   latency, tokens, http status, ...).
+//! - `distinct_id` is read from the [`plano::DISTINCT_ID`] span attribute (set
+//!   by the LLM handler from the configured `distinct_id_header`). When absent
+//!   the event is captured anonymously (`$process_person_profile = false`).
+//! - Network failures are logged and dropped — telemetry export never blocks or
+//!   fails request processing.
+
+use std::time::Duration;
+
+use opentelemetry::{Array, Value};
+use opentelemetry_sdk::error::OTelSdkResult;
+use opentelemetry_sdk::trace::{SpanData, SpanExporter};
+use opentelemetry_sdk::Resource;
+use serde_json::{json, Map, Value as JsonValue};
+use time::format_description::well_known::Rfc3339;
+use time::OffsetDateTime;
+
+use super::{http, llm, plano};
+
+/// PostHog event name for an individual LLM call.
+const AI_GENERATION_EVENT: &str = "$ai_generation";
+
+/// PostHog capture path appended to the configured host.
+const CAPTURE_PATH: &str = "batch/";
+
+/// A [`SpanExporter`] that ships LLM spans to PostHog as `$ai_generation` events.
+pub struct PostHogExporter {
+    client: reqwest::Client,
+    /// Fully-qualified capture endpoint, e.g. `https://us.i.posthog.com/batch/`.
+    endpoint: String,
+    /// PostHog project API key (token).
+    api_key: String,
+    /// Whether to attach the truncated user message preview as `$ai_input`.
+    capture_messages: bool,
+}
+
+impl std::fmt::Debug for PostHogExporter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PostHogExporter")
+            .field("endpoint", &self.endpoint)
+            .field("capture_messages", &self.capture_messages)
+            .finish()
+    }
+}
+
+impl PostHogExporter {
+    /// Create a new PostHog exporter.
+    ///
+    /// # Arguments
+    /// * `url` – PostHog host (e.g. `https://us.i.posthog.com`). The `/batch/`
+    ///   capture path is appended automatically.
+    /// * `api_key` – PostHog project API key (token).
+    /// * `capture_messages` – when true, send the user message preview as
+    ///   `$ai_input`.
+    pub fn new(url: &str, api_key: &str, capture_messages: bool) -> Self {
+        let endpoint = format!("{}/{}", url.trim_end_matches('/'), CAPTURE_PATH);
+        let client = reqwest::Client::builder()
+            .timeout(Duration::from_secs(10))
+            .build()
+            .unwrap_or_default();
+        Self {
+            client,
+            endpoint,
+            api_key: api_key.to_string(),
+            capture_messages,
+        }
+    }
+
+    /// Build the PostHog `batch` payload from a batch of spans, keeping only LLM
+    /// generation spans. Returns `None` when no LLM spans are present.
+    fn build_payload(&self, batch: &[SpanData]) -> Option<JsonValue> {
+        let events: Vec<JsonValue> = batch
+            .iter()
+            .filter_map(|span| self.build_generation_event(span))
+            .collect();
+
+        if events.is_empty() {
+            return None;
+        }
+
+        Some(json!({
+            "api_key": self.api_key,
+            "batch": events,
+        }))
+    }
+
+    /// Translate a single span into a PostHog `$ai_generation` event, or `None`
+    /// if the span is not an LLM generation span.
+    fn build_generation_event(&self, span: &SpanData) -> Option<JsonValue> {
+        // Only LLM generation spans carry `llm.model`.
+        let model = find_attr(span, llm::MODEL_NAME)?;
+
+        let mut props = Map::new();
+        props.insert("$ai_model".to_string(), otel_value_to_json(model));
+        props.insert(
+            "$ai_trace_id".to_string(),
+            json!(span.span_context.trace_id().to_string()),
+        );
+        if span.parent_span_id != opentelemetry::trace::SpanId::INVALID {
+            props.insert(
+                "$ai_parent_id".to_string(),
+                json!(span.parent_span_id.to_string()),
+            );
+        }
+
+        if let Some(provider) = find_attr(span, llm::PROVIDER) {
+            props.insert("$ai_provider".to_string(), otel_value_to_json(provider));
+        }
+
+        // Latency / TTFT are stored in milliseconds; PostHog wants seconds.
+        if let Some(ms) = find_i64(span, llm::DURATION_MS) {
+            props.insert("$ai_latency".to_string(), json!(ms as f64 / 1000.0));
+        }
+        if let Some(ms) = find_i64(span, llm::TIME_TO_FIRST_TOKEN_MS) {
+            props.insert(
+                "$ai_time_to_first_token".to_string(),
+                json!(ms as f64 / 1000.0),
+            );
+            props.insert("$ai_stream".to_string(), json!(true));
+        }
+
+        if let Some(tokens) = find_i64(span, llm::PROMPT_TOKENS) {
+            props.insert("$ai_input_tokens".to_string(), json!(tokens));
+        }
+        if let Some(tokens) = find_i64(span, llm::COMPLETION_TOKENS) {
+            props.insert("$ai_output_tokens".to_string(), json!(tokens));
+        }
+
+        if let Some(status) = find_i64(span, http::STATUS_CODE) {
+            props.insert("$ai_http_status".to_string(), json!(status));
+            if status >= 400 {
+                props.insert("$ai_is_error".to_string(), json!(true));
+            }
+        }
+
+        if self.capture_messages {
+            if let Some(preview) = find_attr(span, llm::USER_MESSAGE_PREVIEW) {
+                props.insert(
+                    "$ai_input".to_string(),
+                    json!([{ "role": "user", "content": value_to_string(preview) }]),
+                );
+            }
+        }
+
+        // distinct_id: identified when the configured header was present,
+        // otherwise anonymous (do not create/update a person profile).
+        match find_attr(span, plano::DISTINCT_ID) {
+            Some(id) => {
+                props.insert("distinct_id".to_string(), otel_value_to_json(id));
+            }
+            None => {
+                props.insert(
+                    "distinct_id".to_string(),
+                    json!(span.span_context.trace_id().to_string()),
+                );
+                props.insert("$process_person_profile".to_string(), json!(false));
+            }
+        }
+
+        // Pass through any other non-reserved attributes (custom span attributes
+        // such as static tags or header-derived tenant ids) as plain properties.
+        for kv in span.attributes.iter() {
+            let key = kv.key.as_str();
+            if is_reserved_attr(key) {
+                continue;
+            }
+            props
+                .entry(key.to_string())
+                .or_insert_with(|| otel_value_to_json(&kv.value));
+        }
+
+        let mut event = Map::new();
+        event.insert("event".to_string(), json!(AI_GENERATION_EVENT));
+        event.insert("properties".to_string(), JsonValue::Object(props));
+        if let Ok(ts) = OffsetDateTime::from(span.end_time).format(&Rfc3339) {
+            event.insert("timestamp".to_string(), json!(ts));
+        }
+
+        Some(JsonValue::Object(event))
+    }
+}
+
+impl SpanExporter for PostHogExporter {
+    fn export(
+        &self,
+        batch: Vec<SpanData>,
+    ) -> impl std::future::Future<Output = OTelSdkResult> + Send {
+        let payload = self.build_payload(&batch);
+        let client = self.client.clone();
+        let endpoint = self.endpoint.clone();
+        async move {
+            let Some(payload) = payload else {
+                return Ok(());
+            };
+            match client.post(&endpoint).json(&payload).send().await {
+                Ok(resp) if resp.status().is_success() => {}
+                Ok(resp) => {
+                    tracing::warn!(
+                        status = %resp.status(),
+                        endpoint = %endpoint,
+                        "PostHog exporter: non-success response"
+                    );
+                }
+                Err(e) => {
+                    tracing::warn!(error = ?e, endpoint = %endpoint, "PostHog exporter: request failed");
+                }
+            }
+            Ok(())
+        }
+    }
+
+    fn shutdown_with_timeout(&mut self, _timeout: Duration) -> OTelSdkResult {
+        Ok(())
+    }
+
+    fn set_resource(&mut self, _resource: &Resource) {}
+}
+
+/// Span attributes that are mapped to dedicated `$ai_*` properties (or are
+/// internal plumbing) and should not be duplicated as raw properties.
+fn is_reserved_attr(key: &str) -> bool {
+    matches!(
+        key,
+        k if k == llm::MODEL_NAME
+            || k == llm::PROVIDER
+            || k == llm::DURATION_MS
+            || k == llm::TIME_TO_FIRST_TOKEN_MS
+            || k == llm::PROMPT_TOKENS
+            || k == llm::COMPLETION_TOKENS
+            || k == llm::USER_MESSAGE_PREVIEW
+            || k == http::STATUS_CODE
+            || k == plano::DISTINCT_ID
+            || k == super::SERVICE_NAME_OVERRIDE_KEY
+    )
+}
+
+fn find_attr<'a>(span: &'a SpanData, key: &str) -> Option<&'a Value> {
+    span.attributes
+        .iter()
+        .find(|kv| kv.key.as_str() == key)
+        .map(|kv| &kv.value)
+}
+
+fn find_i64(span: &SpanData, key: &str) -> Option<i64> {
+    match find_attr(span, key)? {
+        Value::I64(i) => Some(*i),
+        _ => None,
+    }
+}
+
+fn value_to_string(value: &Value) -> String {
+    match value {
+        Value::String(s) => s.as_str().to_string(),
+        other => other.to_string(),
+    }
+}
+
+fn otel_value_to_json(value: &Value) -> JsonValue {
+    match value {
+        Value::Bool(b) => json!(b),
+        Value::I64(i) => json!(i),
+        Value::F64(f) => json!(f),
+        Value::String(s) => json!(s.as_str()),
+        Value::Array(arr) => match arr {
+            Array::Bool(v) => json!(v),
+            Array::I64(v) => json!(v),
+            Array::F64(v) => json!(v),
+            Array::String(v) => json!(v.iter().map(|s| s.as_str()).collect::<Vec<_>>()),
+            _ => JsonValue::Null,
+        },
+        _ => json!(value.to_string()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use opentelemetry::trace::{
+        SpanContext, SpanId, SpanKind, Status, TraceFlags, TraceId, TraceState,
+    };
+    use opentelemetry::KeyValue;
+    use opentelemetry_sdk::trace::{SpanData, SpanEvents, SpanLinks};
+    use std::borrow::Cow;
+    use std::time::SystemTime;
+
+    fn span_with_attrs(attrs: Vec<KeyValue>) -> SpanData {
+        SpanData {
+            span_context: SpanContext::new(
+                TraceId::from_bytes([
+                    0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0, 0x12, 0x34, 0x56, 0x78, 0x9a,
+                    0xbc, 0xde, 0xf0,
+                ]),
+                SpanId::from_bytes([0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]),
+                TraceFlags::SAMPLED,
+                false,
+                TraceState::default(),
+            ),
+            parent_span_id: SpanId::INVALID,
+            parent_span_is_remote: false,
+            span_kind: SpanKind::Client,
+            name: Cow::Borrowed("llm"),
+            start_time: SystemTime::UNIX_EPOCH,
+            end_time: SystemTime::UNIX_EPOCH,
+            attributes: attrs,
+            dropped_attributes_count: 0,
+            events: SpanEvents::default(),
+            links: SpanLinks::default(),
+            status: Status::Unset,
+            instrumentation_scope: Default::default(),
+        }
+    }
+
+    fn props(event: &JsonValue) -> &Map<String, JsonValue> {
+        event["properties"].as_object().unwrap()
+    }
+
+    #[test]
+    fn non_llm_span_is_skipped() {
+        let exporter = PostHogExporter::new("https://us.i.posthog.com", "phc_x", false);
+        let span = span_with_attrs(vec![KeyValue::new("routing.strategy", "least-latency")]);
+        assert!(exporter.build_generation_event(&span).is_none());
+    }
+
+    #[test]
+    fn maps_llm_attributes_to_ai_properties() {
+        let exporter = PostHogExporter::new("https://us.i.posthog.com/", "phc_x", false);
+        let span = span_with_attrs(vec![
+            KeyValue::new(llm::MODEL_NAME, "gpt-5-mini"),
+            KeyValue::new(llm::PROVIDER, "openai"),
+            KeyValue::new(llm::DURATION_MS, 1500_i64),
+            KeyValue::new(llm::TIME_TO_FIRST_TOKEN_MS, 250_i64),
+            KeyValue::new(llm::PROMPT_TOKENS, 10_i64),
+            KeyValue::new(llm::COMPLETION_TOKENS, 20_i64),
+            KeyValue::new(http::STATUS_CODE, 200_i64),
+            KeyValue::new("tenant.id", "acme"),
+        ]);
+
+        let event = exporter.build_generation_event(&span).unwrap();
+        assert_eq!(event["event"], json!("$ai_generation"));
+        let p = props(&event);
+        assert_eq!(p["$ai_model"], json!("gpt-5-mini"));
+        assert_eq!(p["$ai_provider"], json!("openai"));
+        assert_eq!(p["$ai_latency"], json!(1.5));
+        assert_eq!(p["$ai_time_to_first_token"], json!(0.25));
+        assert_eq!(p["$ai_stream"], json!(true));
+        assert_eq!(p["$ai_input_tokens"], json!(10));
+        assert_eq!(p["$ai_output_tokens"], json!(20));
+        assert_eq!(p["$ai_http_status"], json!(200));
+        // Anonymous (no distinct id header captured).
+        assert_eq!(p["$process_person_profile"], json!(false));
+        // Custom passthrough attribute preserved.
+        assert_eq!(p["tenant.id"], json!("acme"));
+        // No $ai_input unless capture_messages is enabled.
+        assert!(!p.contains_key("$ai_input"));
+    }
+
+    #[test]
+    fn uses_distinct_id_and_flags_errors() {
+        let exporter = PostHogExporter::new("https://us.i.posthog.com", "phc_x", true);
+        let span = span_with_attrs(vec![
+            KeyValue::new(llm::MODEL_NAME, "gpt-5-mini"),
+            KeyValue::new(plano::DISTINCT_ID, "user_123"),
+            KeyValue::new(llm::USER_MESSAGE_PREVIEW, "hello"),
+            KeyValue::new(http::STATUS_CODE, 500_i64),
+        ]);
+
+        let event = exporter.build_generation_event(&span).unwrap();
+        let p = props(&event);
+        assert_eq!(p["distinct_id"], json!("user_123"));
+        assert!(!p.contains_key("$process_person_profile"));
+        assert_eq!(p["$ai_is_error"], json!(true));
+        assert_eq!(
+            p["$ai_input"],
+            json!([{ "role": "user", "content": "hello" }])
+        );
+    }
+
+    #[test]
+    fn payload_wraps_events_with_api_key() {
+        let exporter = PostHogExporter::new("https://us.i.posthog.com", "phc_secret", false);
+        let span = span_with_attrs(vec![KeyValue::new(llm::MODEL_NAME, "gpt-5-mini")]);
+        let payload = exporter.build_payload(&[span]).unwrap();
+        assert_eq!(payload["api_key"], json!("phc_secret"));
+        assert_eq!(payload["batch"].as_array().unwrap().len(), 1);
+    }
+}

crates/brightstaff/src/tracing/service_name_exporter.rs

@@ -172,7 +172,7 @@ impl SpanExporter for ServiceNameOverrideExporter {
     }
 
     fn shutdown_with_timeout(&mut self, timeout: Duration) -> OTelSdkResult {
-        for (_, exporter_mutex) in self.exporters.iter() {
+        for exporter_mutex in self.exporters.values() {
             if let Ok(mut exporter) = exporter_mutex.try_lock() {
                 let _ = exporter.shutdown_with_timeout(timeout);
             }

crates/common/src/configuration.rs

@@ -33,6 +33,11 @@ pub struct Routing {
     pub session_ttl_seconds: Option<u64>,
     pub session_max_entries: Option<usize>,
     pub session_cache: Option<SessionCacheConfig>,
+    /// Cost gate on model switching within a session. Self-sufficient and independent
+    /// of prompt caching: presence of this block turns it on, implicit sessions are
+    /// derived on its own, and warm anchors are always priced at cached rates —
+    /// `prompt_caching` only controls marker injection and affinity-without-budget.
+    pub routing_budget: Option<RoutingBudget>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -133,7 +138,7 @@ pub enum StateStorageType {
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(rename_all = "lowercase")]
+#[serde(rename_all = "snake_case")]
 pub enum SelectionPreference {
     Cheapest,
     Fastest,
@@ -177,8 +182,13 @@ pub enum MetricsSource {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct CostMetricsConfig {
     pub provider: CostProvider,
+    /// Optional override for the pricing catalog endpoint. When omitted, a
+    /// sensible default is used per provider.
+    pub url: Option<String>,
     pub refresh_interval: Option<u64>,
-    /// Map DO catalog keys (`lowercase(creator)/model_id`) to Plano model names.
+    /// Map catalog keys to Plano model names used in `routing_preferences`.
+    /// DigitalOcean keys look like `lowercase(creator)/model_id`; models.dev
+    /// keys look like `creator/model_id`.
     /// Example: `openai/openai-gpt-oss-120b: openai/gpt-4o`
     pub model_aliases: Option<HashMap<String, String>>,
 }
@@ -187,6 +197,8 @@ pub struct CostMetricsConfig {
 #[serde(rename_all = "snake_case")]
 pub enum CostProvider {
     Digitalocean,
+    #[serde(rename = "models.dev")]
+    ModelsDev,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -211,6 +223,10 @@ pub struct Configuration {
     pub model_aliases: Option<HashMap<String, ModelAlias>>,
     pub overrides: Option<Overrides>,
     pub routing: Option<Routing>,
+    /// Automatic provider prompt caching. Disabled by default; opt in globally with
+    /// `prompt_caching: { enabled: true }`. Applies across the entire Plano instance
+    /// and never changes which model routing selects.
+    pub prompt_caching: Option<PromptCaching>,
     pub system_prompt: Option<String>,
     pub prompt_guards: Option<PromptGuards>,
     pub prompt_targets: Option<Vec<PromptTarget>>,
@@ -237,6 +253,181 @@ pub struct Overrides {
     pub disable_signals: Option<bool>,
 }
 
+/// Automatic prompt caching, configured once for the whole Plano instance.
+///
+/// Prompt caching keeps a multi-turn conversation's stable prefix warm in the
+/// upstream provider's cache. It never influences which model routing selects — it
+/// only (a) auto-injects provider cache-control markers where supported and
+/// (b) derives an implicit session key from the stable prompt prefix so follow-up
+/// turns reuse the same warm cache. An explicit `X-Model-Affinity` header always wins.
+///
+/// Disabled by default; opt in with `enabled: true`. The remaining knobs are optional
+/// tuning that only take effect while caching is enabled.
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct PromptCaching {
+    /// Master switch. Defaults to `false` (opt-in).
+    #[serde(default)]
+    pub enabled: bool,
+    /// Derive an implicit session key from the stable prompt prefix so caches survive
+    /// across turns without client changes. Defaults to `true` when caching is enabled.
+    pub session_affinity: Option<bool>,
+    /// Auto-inject provider cache-control markers (e.g. Anthropic `cache_control`).
+    /// Defaults to `true` when caching is enabled.
+    pub inject_cache_control: Option<bool>,
+    /// Minimum estimated prefix tokens before a cache breakpoint is injected.
+    pub min_prefix_tokens: Option<u32>,
+    /// Session pin TTL; falls back to `routing.session_ttl_seconds` when unset.
+    pub session_ttl_seconds: Option<u64>,
+}
+
+/// A cumulative per-session overhead cap governing when routing may switch models.
+///
+/// This is a routing concern, not a caching one: it applies whenever configured,
+/// regardless of whether `prompt_caching` is enabled. The default posture is to stick
+/// to the model a session is warm on. When routing proposes a *different* model,
+/// switching forces the candidate to re-ingest the whole context at its uncached input
+/// rate. With prompt caching the anchor is warm, so the cost is the cache-loss delta
+/// `context_tokens x (candidate_uncached_input_rate - anchor_cached_input_rate)`; without
+/// caching there is no warm cache to lose, so it is
+/// `context_tokens x (candidate_uncached_input_rate - anchor_uncached_input_rate)`. That
+/// input-token cost accrues into the session's cumulative switch spend. The gate allows a paid switch
+/// only while that spend stays within `max_overhead_pct` percent of the session's
+/// running *never-switch* baseline (the cost the session would have paid by staying
+/// on its anchor). A switch that is outright cheaper (negative cost) is free but
+/// never credits the spend back — the "saving" is vs a path we didn't take, not real
+/// spendable money. Requires a cost source in `model_metrics_sources` so per-model
+/// rates are available. Presence of the block turns it on.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
+pub struct RoutingBudget {
+    /// Cap on cumulative switching overhead, as a **percentage** of what the session
+    /// would have cost by never switching (a whole number: `20` = 20%). The promise
+    /// is "this conversation bills at most `max_overhead_pct`% above never-switching."
+    /// `0` means "never pay to switch" (only outright-cheaper switches are ever
+    /// allowed); larger values buy more quality-driven switches. Typical range 10–30.
+    pub max_overhead_pct: f64,
+    /// Reset the running baseline/spend totals when a session goes cold and re-binds
+    /// (a fresh warm episode). Defaults to `true`.
+    #[serde(default = "default_true")]
+    pub replenish_on_rebind: bool,
+    /// Fallback used to estimate a model's cached input rate when the pricing feed
+    /// doesn't publish one: `cached_rate = input_rate * cache_read_discount`. A
+    /// pricing detail, not a cost policy. Defaults to 0.1 (cached reads at 10% of
+    /// input, typical for Anthropic-style caches).
+    pub cache_read_discount: Option<f64>,
+    /// When true, a vetoed switch records the route the gate *would* have taken had
+    /// the switch been allowed, as the `plano.switch.counterfactual_route` span
+    /// attribute. Telemetry only — the counterfactual model is never dispatched.
+    /// Useful for evals/benchmarks that want to quantify the road not taken.
+    /// Defaults to `false`.
+    #[serde(default)]
+    pub record_counterfactual: bool,
+}
+
+fn default_true() -> bool {
+    true
+}
+
+/// Fully-resolved routing-budget settings (present only when configured and valid).
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct EffectiveRoutingBudget {
+    /// Cumulative switching-overhead cap, as a percentage of the never-switch
+    /// baseline (a whole number: `20` = 20%).
+    pub max_overhead_pct: f64,
+    /// Reset the running baseline/spend totals on cold->warm re-bind.
+    pub replenish_on_rebind: bool,
+    pub cache_read_discount: f64,
+    /// Emit `plano.switch.counterfactual_route` on vetoed switches. Telemetry only.
+    pub record_counterfactual: bool,
+}
+
+pub const DEFAULT_CACHE_READ_DISCOUNT: f64 = 0.1;
+
+impl RoutingBudget {
+    /// Resolve to effective settings, validating the overhead cap and cache-read
+    /// discount.
+    pub fn resolve(&self) -> Result<EffectiveRoutingBudget, String> {
+        if !self.max_overhead_pct.is_finite() || self.max_overhead_pct < 0.0 {
+            return Err(format!(
+                "routing.routing_budget.max_overhead_pct: must be a non-negative number (percent, e.g. 20 for 20%), got {}",
+                self.max_overhead_pct
+            ));
+        }
+        let cache_read_discount = self
+            .cache_read_discount
+            .unwrap_or(DEFAULT_CACHE_READ_DISCOUNT);
+        if !(0.0..=1.0).contains(&cache_read_discount) {
+            return Err(format!(
+                "routing.routing_budget.cache_read_discount: must be between 0.0 and 1.0, got {cache_read_discount}"
+            ));
+        }
+        Ok(EffectiveRoutingBudget {
+            max_overhead_pct: self.max_overhead_pct,
+            replenish_on_rebind: self.replenish_on_rebind,
+            cache_read_discount,
+            record_counterfactual: self.record_counterfactual,
+        })
+    }
+}
+
+impl EffectiveRoutingBudget {
+    /// Resolve from an optional config block; `None` means the gate is off.
+    pub fn from_config(config: Option<&RoutingBudget>) -> Result<Option<Self>, String> {
+        config.map(RoutingBudget::resolve).transpose()
+    }
+}
+
+/// Fully-resolved, instance-wide prompt-caching settings.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct EffectivePromptCaching {
+    pub enabled: bool,
+    pub session_affinity: bool,
+    pub inject_cache_control: bool,
+    pub min_prefix_tokens: u32,
+    /// Pin TTL override; `None` uses `routing.session_ttl_seconds`.
+    pub session_ttl_seconds: Option<u64>,
+}
+
+pub const DEFAULT_MIN_PREFIX_TOKENS: u32 = 1024;
+
+impl Default for EffectivePromptCaching {
+    fn default() -> Self {
+        EffectivePromptCaching {
+            enabled: false,
+            session_affinity: false,
+            inject_cache_control: false,
+            min_prefix_tokens: DEFAULT_MIN_PREFIX_TOKENS,
+            session_ttl_seconds: None,
+        }
+    }
+}
+
+impl PromptCaching {
+    /// Resolve the instance-wide effective settings. When caching is disabled every
+    /// sub-feature is off, regardless of the individual knobs.
+    pub fn resolve(&self) -> Result<EffectivePromptCaching, String> {
+        if !self.enabled {
+            return Ok(EffectivePromptCaching::default());
+        }
+        Ok(EffectivePromptCaching {
+            enabled: true,
+            session_affinity: self.session_affinity.unwrap_or(true),
+            inject_cache_control: self.inject_cache_control.unwrap_or(true),
+            min_prefix_tokens: self.min_prefix_tokens.unwrap_or(DEFAULT_MIN_PREFIX_TOKENS),
+            session_ttl_seconds: self.session_ttl_seconds,
+        })
+    }
+}
+
+impl EffectivePromptCaching {
+    /// Resolve from an optional config block; `None` means caching is off.
+    pub fn from_config(config: Option<&PromptCaching>) -> Result<Self, String> {
+        config
+            .map(PromptCaching::resolve)
+            .transpose()
+            .map(Option::unwrap_or_default)
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct Tracing {
     pub sampling_rate: Option<f64>,
@@ -244,6 +435,11 @@ pub struct Tracing {
     pub random_sampling: Option<u32>,
     pub opentracing_grpc_endpoint: Option<String>,
     pub span_attributes: Option<SpanAttributes>,
+    /// Provider-agnostic telemetry export destinations. Each entry is tagged by
+    /// its `type` (e.g. `posthog`) so new backends can be added without breaking
+    /// existing configs. LLM spans are translated into each backend's native
+    /// event format and streamed in addition to any `opentracing_grpc_endpoint`.
+    pub exporters: Option<Vec<Exporter>>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -253,6 +449,36 @@ pub struct SpanAttributes {
     pub static_attributes: Option<HashMap<String, String>>,
 }
 
+/// A telemetry export destination configured under `tracing.exporters`.
+///
+/// The list is provider-agnostic; each variant is internally tagged by its
+/// `type` field (e.g. `type: posthog`). Additional backends (datadog, raw
+/// otlp, ...) can be added as new variants without breaking existing configs.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum Exporter {
+    /// PostHog AI observability. LLM spans are converted into PostHog
+    /// `$ai_generation` events and POSTed to the configured `url`.
+    Posthog(PosthogExporter),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PosthogExporter {
+    /// PostHog host, e.g. `https://us.i.posthog.com`. The `/batch/` capture
+    /// path is appended automatically.
+    pub url: String,
+    /// PostHog project API key (token). Supports `$ENV_VAR` expansion at render
+    /// time, e.g. `$POSTHOG_API_KEY`.
+    pub api_key: String,
+    /// Optional request header whose value is used as the PostHog `distinct_id`.
+    /// When unset (or the header is missing on a request) events are captured
+    /// anonymously.
+    pub distinct_id_header: Option<String>,
+    /// When true, include the truncated user message preview as `$ai_input`.
+    /// Defaults to `false` to avoid sending prompt content off-box.
+    pub capture_messages: Option<bool>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, Default)]
 pub enum GatewayMode {
     #[serde(rename = "llm")]
@@ -657,7 +883,10 @@ mod test {
     use pretty_assertions::assert_eq;
     use std::fs;
 
-    use super::{IntoModels, LlmProvider, LlmProviderType};
+    use super::{
+        EffectivePromptCaching, EffectiveRoutingBudget, IntoModels, LlmProvider, LlmProviderType,
+        PromptCaching, RoutingBudget, DEFAULT_CACHE_READ_DISCOUNT, DEFAULT_MIN_PREFIX_TOKENS,
+    };
     use crate::api::open_ai::ToolType;
 
     #[test]
@@ -741,6 +970,51 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_deserialize_models_dev_cost_source() {
+        let yaml = r#"
+- type: cost
+  provider: models.dev
+  url: https://models.dev/api.json
+  refresh_interval: 3600
+  model_aliases:
+    openai/gpt-oss-120b: openai/gpt-4o
+"#;
+        let sources: Vec<super::MetricsSource> = serde_yaml::from_str(yaml).unwrap();
+        assert_eq!(sources.len(), 1);
+        match &sources[0] {
+            super::MetricsSource::Cost(cfg) => {
+                assert!(matches!(cfg.provider, super::CostProvider::ModelsDev));
+                assert_eq!(cfg.url.as_deref(), Some("https://models.dev/api.json"));
+                assert_eq!(cfg.refresh_interval, Some(3600));
+                assert_eq!(
+                    cfg.model_aliases
+                        .as_ref()
+                        .and_then(|m| m.get("openai/gpt-oss-120b"))
+                        .map(String::as_str),
+                    Some("openai/gpt-4o")
+                );
+            }
+            other => panic!("expected cost source, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn test_deserialize_digitalocean_cost_source_without_url() {
+        let yaml = r#"
+- type: cost
+  provider: digitalocean
+"#;
+        let sources: Vec<super::MetricsSource> = serde_yaml::from_str(yaml).unwrap();
+        match &sources[0] {
+            super::MetricsSource::Cost(cfg) => {
+                assert!(matches!(cfg.provider, super::CostProvider::Digitalocean));
+                assert_eq!(cfg.url, None);
+            }
+            other => panic!("expected cost source, got {other:?}"),
+        }
+    }
+
     #[test]
     fn test_into_models_filters_internal_providers() {
         let providers = vec![
@@ -813,4 +1087,169 @@ disable_signals: false
         let overrides: super::Overrides = serde_yaml::from_str(yaml_missing).unwrap();
         assert_eq!(overrides.disable_signals, None);
     }
+
+    #[test]
+    fn test_prompt_caching_disabled_by_default() {
+        // Absent config → everything off.
+        let effective = EffectivePromptCaching::from_config(None).unwrap();
+        assert!(!effective.enabled);
+        assert!(!effective.session_affinity);
+        assert!(!effective.inject_cache_control);
+
+        // Present but not enabled → still off.
+        let cfg: PromptCaching = serde_yaml::from_str("enabled: false").unwrap();
+        let effective = cfg.resolve().unwrap();
+        assert!(!effective.enabled);
+        assert!(!effective.session_affinity);
+        assert!(!effective.inject_cache_control);
+    }
+
+    #[test]
+    fn test_prompt_caching_enabled_defaults() {
+        // A bare `enabled: true` turns everything on with sensible defaults.
+        let cfg: PromptCaching = serde_yaml::from_str("enabled: true").unwrap();
+        let effective = cfg.resolve().unwrap();
+        assert!(effective.enabled);
+        assert!(effective.session_affinity);
+        assert!(effective.inject_cache_control);
+        assert_eq!(effective.min_prefix_tokens, DEFAULT_MIN_PREFIX_TOKENS);
+        assert_eq!(effective.session_ttl_seconds, None);
+    }
+
+    #[test]
+    fn test_prompt_caching_optional_knobs() {
+        let yaml = r#"
+enabled: true
+session_affinity: false
+inject_cache_control: false
+min_prefix_tokens: 2048
+session_ttl_seconds: 3600
+"#;
+        let cfg: PromptCaching = serde_yaml::from_str(yaml).unwrap();
+        let effective = cfg.resolve().unwrap();
+        assert!(effective.enabled);
+        assert!(!effective.session_affinity);
+        assert!(!effective.inject_cache_control);
+        assert_eq!(effective.min_prefix_tokens, 2048);
+        assert_eq!(effective.session_ttl_seconds, Some(3600));
+    }
+
+    #[test]
+    fn test_prompt_caching_knobs_ignored_when_disabled() {
+        // Knobs only take effect while caching is enabled.
+        let yaml = r#"
+enabled: false
+session_affinity: true
+inject_cache_control: true
+"#;
+        let cfg: PromptCaching = serde_yaml::from_str(yaml).unwrap();
+        let effective = cfg.resolve().unwrap();
+        assert!(!effective.enabled);
+        assert!(!effective.session_affinity);
+        assert!(!effective.inject_cache_control);
+    }
+
+    #[test]
+    fn test_routing_budget_parses() {
+        let yaml = r#"
+max_overhead_pct: 20
+"#;
+        let cfg: RoutingBudget = serde_yaml::from_str(yaml).unwrap();
+        let budget = cfg.resolve().unwrap();
+        assert_eq!(budget.max_overhead_pct, 20.0);
+        // Replenish defaults on.
+        assert!(budget.replenish_on_rebind);
+        assert_eq!(budget.cache_read_discount, DEFAULT_CACHE_READ_DISCOUNT);
+        // Counterfactual recording is opt-in; off unless requested.
+        assert!(!budget.record_counterfactual);
+    }
+
+    #[test]
+    fn test_routing_budget_record_counterfactual_parses() {
+        let yaml = r#"
+max_overhead_pct: 20
+record_counterfactual: true
+"#;
+        let cfg: RoutingBudget = serde_yaml::from_str(yaml).unwrap();
+        let budget = cfg.resolve().unwrap();
+        assert!(budget.record_counterfactual);
+    }
+
+    #[test]
+    fn test_routing_budget_flags_parse() {
+        let yaml = r#"
+max_overhead_pct: 15
+replenish_on_rebind: false
+cache_read_discount: 0.25
+"#;
+        let cfg: RoutingBudget = serde_yaml::from_str(yaml).unwrap();
+        let budget = cfg.resolve().unwrap();
+        assert_eq!(budget.max_overhead_pct, 15.0);
+        assert!(!budget.replenish_on_rebind);
+        assert_eq!(budget.cache_read_discount, 0.25);
+    }
+
+    #[test]
+    fn test_routing_budget_absent_is_off() {
+        // No block configured → gate is off.
+        assert!(EffectiveRoutingBudget::from_config(None).unwrap().is_none());
+    }
+
+    #[test]
+    fn test_routing_budget_invalid_values_rejected() {
+        let negative: RoutingBudget = serde_yaml::from_str("max_overhead_pct: -1.0").unwrap();
+        assert!(negative.resolve().is_err());
+
+        let bad_discount: RoutingBudget = serde_yaml::from_str(
+            r#"
+max_overhead_pct: 20
+cache_read_discount: 1.5
+"#,
+        )
+        .unwrap();
+        assert!(bad_discount.resolve().is_err());
+    }
+
+    #[test]
+    fn test_tracing_posthog_exporter_deserialize() {
+        let yaml = r#"
+random_sampling: 100
+exporters:
+  - type: posthog
+    url: https://us.i.posthog.com
+    api_key: phc_secret
+    distinct_id_header: x-user-id
+    capture_messages: true
+"#;
+        let tracing: super::Tracing = serde_yaml::from_str(yaml).unwrap();
+        let exporters = tracing.exporters.expect("exporters should be parsed");
+        assert_eq!(exporters.len(), 1);
+        match &exporters[0] {
+            super::Exporter::Posthog(posthog) => {
+                assert_eq!(posthog.url, "https://us.i.posthog.com");
+                assert_eq!(posthog.api_key, "phc_secret");
+                assert_eq!(posthog.distinct_id_header.as_deref(), Some("x-user-id"));
+                assert_eq!(posthog.capture_messages, Some(true));
+            }
+        }
+    }
+
+    #[test]
+    fn test_tracing_posthog_exporter_minimal() {
+        let yaml = r#"
+exporters:
+  - type: posthog
+    url: https://eu.i.posthog.com
+    api_key: phc_eu
+"#;
+        let tracing: super::Tracing = serde_yaml::from_str(yaml).unwrap();
+        let exporters = tracing.exporters.unwrap();
+        match &exporters[0] {
+            super::Exporter::Posthog(posthog) => {
+                assert_eq!(posthog.url, "https://eu.i.posthog.com");
+                assert_eq!(posthog.distinct_id_header, None);
+                assert_eq!(posthog.capture_messages, None);
+            }
+        }
+    }
 }

crates/common/src/consts.rs

@@ -23,6 +23,12 @@ pub const X_ARCH_FC_MODEL_RESPONSE: &str = "x-arch-fc-model-response";
 pub const ARCH_FC_MODEL_NAME: &str = "Arch-Function";
 pub const REQUEST_ID_HEADER: &str = "x-request-id";
 pub const MODEL_AFFINITY_HEADER: &str = "x-model-affinity";
+/// Per-request prompt-caching control. `off` disables implicit session affinity and
+/// cache-control injection for that single request.
+pub const PLANO_CACHE_HEADER: &str = "x-plano-cache";
+/// Hash of the stable prompt prefix, forwarded upstream so self-hosted multi-replica
+/// backends can do KV-aware (consistent-hash) replica routing at the LB/Envoy layer.
+pub const PLANO_PREFIX_HASH_HEADER: &str = "x-plano-prefix-hash";
 pub const ENVOY_ORIGINAL_PATH_HEADER: &str = "x-envoy-original-path";
 pub const TRACE_PARENT_HEADER: &str = "traceparent";
 pub const ARCH_INTERNAL_CLUSTER_NAME: &str = "arch_internal";

crates/hermesllm/src/apis/anthropic.rs

@@ -128,6 +128,7 @@ pub struct MessagesRequest {
 pub enum MessagesRole {
     User,
     Assistant,
+    System,
 }
 
 /// Cache control types for content blocks
@@ -284,6 +285,7 @@ pub struct MessagesTool {
     pub name: String,
     pub description: Option<String>,
     pub input_schema: Value,
+    pub cache_control: Option<MessagesCacheControl>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
@@ -304,6 +306,155 @@ pub struct MessagesToolChoice {
     pub disable_parallel_tool_use: Option<bool>,
 }
 
+/// Rough chars-per-token heuristic used to threshold-guard cache-marker injection.
+/// Injection below the provider's minimum cacheable prefix is a provider-side no-op,
+/// so a cheap estimate is sufficient here.
+const CHARS_PER_TOKEN: usize = 4;
+
+fn block_cache_control(block: &MessagesContentBlock) -> Option<&MessagesCacheControl> {
+    match block {
+        MessagesContentBlock::Text { cache_control, .. }
+        | MessagesContentBlock::Thinking { cache_control, .. }
+        | MessagesContentBlock::ToolUse { cache_control, .. }
+        | MessagesContentBlock::ToolResult { cache_control, .. } => cache_control.as_ref(),
+        _ => None,
+    }
+}
+
+/// Set `cache_control: ephemeral` on the last block that supports it.
+/// Returns true if a marker was placed.
+fn mark_last_cacheable_block(blocks: &mut [MessagesContentBlock]) -> bool {
+    for block in blocks.iter_mut().rev() {
+        match block {
+            MessagesContentBlock::Text { cache_control, .. }
+            | MessagesContentBlock::Thinking { cache_control, .. }
+            | MessagesContentBlock::ToolUse { cache_control, .. }
+            | MessagesContentBlock::ToolResult { cache_control, .. } => {
+                *cache_control = Some(MessagesCacheControl::Ephemeral);
+                return true;
+            }
+            _ => continue,
+        }
+    }
+    false
+}
+
+impl MessagesRequest {
+    /// True when the client already supplied any `cache_control` marker
+    /// (on system blocks, tools, or message content blocks).
+    pub fn has_cache_markers(&self) -> bool {
+        let system_marked = match &self.system {
+            Some(MessagesSystemPrompt::Blocks(blocks)) => {
+                blocks.iter().any(|b| block_cache_control(b).is_some())
+            }
+            _ => false,
+        };
+        let tools_marked = self
+            .tools
+            .as_ref()
+            .is_some_and(|tools| tools.iter().any(|t| t.cache_control.is_some()));
+        let messages_marked = self.messages.iter().any(|m| match &m.content {
+            MessagesMessageContent::Blocks(blocks) => {
+                blocks.iter().any(|b| block_cache_control(b).is_some())
+            }
+            MessagesMessageContent::Single(_) => false,
+        });
+        system_marked || tools_marked || messages_marked
+    }
+
+    /// Estimated token length of the stable prompt prefix (tools + system), which is
+    /// what precedes conversation history in Anthropic's prompt ordering.
+    pub fn estimated_prefix_tokens(&self) -> u32 {
+        let mut chars = 0usize;
+        if let Some(tools) = &self.tools {
+            for tool in tools {
+                chars += tool.name.len();
+                chars += tool.description.as_deref().map_or(0, str::len);
+                chars += tool.input_schema.to_string().len();
+            }
+        }
+        match &self.system {
+            Some(MessagesSystemPrompt::Single(text)) => chars += text.len(),
+            Some(MessagesSystemPrompt::Blocks(blocks)) => {
+                chars += blocks.extract_text().len();
+            }
+            None => {}
+        }
+        (chars / CHARS_PER_TOKEN) as u32
+    }
+
+    /// Auto-inject ephemeral cache breakpoints for providers that require explicit
+    /// markers (Anthropic-shaped requests):
+    ///
+    /// 1. at the end of the system prompt (covering the fully-stable tools + system
+    ///    prefix), falling back to the last tool when there is no system prompt, and
+    /// 2. a rolling breakpoint on the last content block of the final message, so each
+    ///    turn's cache write becomes the next turn's cache read as history grows.
+    ///
+    /// Idempotent: a request that already carries any client-supplied `cache_control`
+    /// marker is left untouched. Threshold-guarded: no-op when the estimated stable
+    /// prefix is below `min_prefix_tokens` (injection below the provider's minimum
+    /// cacheable prefix is wasted bytes).
+    ///
+    /// Returns true if any marker was injected.
+    pub fn inject_cache_breakpoints(&mut self, min_prefix_tokens: u32) -> bool {
+        if self.has_cache_markers() {
+            return false;
+        }
+        if self.estimated_prefix_tokens() < min_prefix_tokens {
+            return false;
+        }
+
+        let mut injected = false;
+
+        // Breakpoint 1: end of the stable prefix.
+        match self.system.take() {
+            Some(MessagesSystemPrompt::Single(text)) => {
+                self.system = Some(MessagesSystemPrompt::Blocks(vec![
+                    MessagesContentBlock::Text {
+                        text,
+                        cache_control: Some(MessagesCacheControl::Ephemeral),
+                    },
+                ]));
+                injected = true;
+            }
+            Some(MessagesSystemPrompt::Blocks(mut blocks)) => {
+                injected |= mark_last_cacheable_block(&mut blocks);
+                self.system = Some(MessagesSystemPrompt::Blocks(blocks));
+            }
+            None => {
+                if let Some(last_tool) = self.tools.as_mut().and_then(|t| t.last_mut()) {
+                    last_tool.cache_control = Some(MessagesCacheControl::Ephemeral);
+                    injected = true;
+                }
+            }
+        }
+
+        // Breakpoint 2: rolling tail of conversation history.
+        if let Some(last_msg) = self.messages.last_mut() {
+            let content = std::mem::replace(
+                &mut last_msg.content,
+                MessagesMessageContent::Single(String::new()),
+            );
+            last_msg.content = match content {
+                MessagesMessageContent::Single(text) => {
+                    injected = true;
+                    MessagesMessageContent::Blocks(vec![MessagesContentBlock::Text {
+                        text,
+                        cache_control: Some(MessagesCacheControl::Ephemeral),
+                    }])
+                }
+                MessagesMessageContent::Blocks(mut blocks) => {
+                    injected |= mark_last_cacheable_block(&mut blocks);
+                    MessagesMessageContent::Blocks(blocks)
+                }
+            };
+        }
+
+        injected
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum MessagesStopReason {
@@ -632,6 +783,7 @@ impl MessagesRole {
         match self {
             MessagesRole::User => "user",
             MessagesRole::Assistant => "assistant",
+            MessagesRole::System => "system",
         }
     }
 }
@@ -678,6 +830,134 @@ mod tests {
     use super::*;
     use serde_json::json;
 
+    fn cache_test_request(system: Option<MessagesSystemPrompt>) -> MessagesRequest {
+        MessagesRequest {
+            model: "claude-sonnet-4".to_string(),
+            messages: vec![
+                MessagesMessage {
+                    role: MessagesRole::User,
+                    content: MessagesMessageContent::Single("first user turn".to_string()),
+                },
+                MessagesMessage {
+                    role: MessagesRole::Assistant,
+                    content: MessagesMessageContent::Single("assistant reply".to_string()),
+                },
+                MessagesMessage {
+                    role: MessagesRole::User,
+                    content: MessagesMessageContent::Single("second user turn".to_string()),
+                },
+            ],
+            max_tokens: 100,
+            container: None,
+            mcp_servers: None,
+            system,
+            metadata: None,
+            service_tier: None,
+            thinking: None,
+            temperature: None,
+            top_p: None,
+            top_k: None,
+            stream: None,
+            stop_sequences: None,
+            tools: None,
+            tool_choice: None,
+        }
+    }
+
+    #[test]
+    fn test_inject_cache_breakpoints_marks_system_and_tail() {
+        let long_system = "x".repeat(8192); // ~2048 estimated tokens
+        let mut req = cache_test_request(Some(MessagesSystemPrompt::Single(long_system)));
+
+        assert!(req.inject_cache_breakpoints(1024));
+
+        // System converted to a marked block.
+        match &req.system {
+            Some(MessagesSystemPrompt::Blocks(blocks)) => {
+                assert_eq!(blocks.len(), 1);
+                assert!(matches!(
+                    &blocks[0],
+                    MessagesContentBlock::Text {
+                        cache_control: Some(MessagesCacheControl::Ephemeral),
+                        ..
+                    }
+                ));
+            }
+            other => panic!("expected marked system blocks, got {:?}", other),
+        }
+
+        // Rolling breakpoint on the last message.
+        match &req.messages.last().unwrap().content {
+            MessagesMessageContent::Blocks(blocks) => {
+                assert!(matches!(
+                    &blocks[0],
+                    MessagesContentBlock::Text {
+                        cache_control: Some(MessagesCacheControl::Ephemeral),
+                        ..
+                    }
+                ));
+            }
+            other => panic!("expected marked tail blocks, got {:?}", other),
+        }
+    }
+
+    #[test]
+    fn test_inject_cache_breakpoints_below_threshold_is_noop() {
+        let mut req = cache_test_request(Some(MessagesSystemPrompt::Single(
+            "short system".to_string(),
+        )));
+        assert!(!req.inject_cache_breakpoints(1024));
+        assert!(matches!(req.system, Some(MessagesSystemPrompt::Single(_))));
+    }
+
+    #[test]
+    fn test_inject_cache_breakpoints_respects_client_markers() {
+        let long_system = "x".repeat(8192);
+        let mut req = cache_test_request(Some(MessagesSystemPrompt::Blocks(vec![
+            MessagesContentBlock::Text {
+                text: long_system,
+                cache_control: Some(MessagesCacheControl::Ephemeral),
+            },
+        ])));
+        // Client already placed a marker: injection must be a no-op.
+        assert!(!req.inject_cache_breakpoints(1024));
+        // Tail message untouched.
+        assert!(matches!(
+            req.messages.last().unwrap().content,
+            MessagesMessageContent::Single(_)
+        ));
+    }
+
+    #[test]
+    fn test_inject_cache_breakpoints_marks_last_tool_when_no_system() {
+        let mut req = cache_test_request(None);
+        req.tools = Some(vec![MessagesTool {
+            name: "big_tool".to_string(),
+            description: Some("d".repeat(8192)),
+            input_schema: json!({"type": "object"}),
+            cache_control: None,
+        }]);
+
+        assert!(req.inject_cache_breakpoints(1024));
+        assert_eq!(
+            req.tools.as_ref().unwrap().last().unwrap().cache_control,
+            Some(MessagesCacheControl::Ephemeral)
+        );
+    }
+
+    #[test]
+    fn test_tool_cache_control_roundtrip() {
+        // Client-supplied cache_control on tools must survive serde passthrough.
+        let tool_json = json!({
+            "name": "get_weather",
+            "input_schema": {"type": "object"},
+            "cache_control": {"type": "ephemeral"}
+        });
+        let tool: MessagesTool = serde_json::from_value(tool_json.clone()).unwrap();
+        assert_eq!(tool.cache_control, Some(MessagesCacheControl::Ephemeral));
+        assert_eq!(serde_json::to_value(&tool).unwrap(), tool_json);
+    }
+
     #[test]
     fn test_anthropic_required_fields() {
         // Create a JSON object with only required fields

crates/hermesllm/src/apis/openai.rs

@@ -163,6 +163,112 @@ impl ChatCompletionsRequest {
             self.store = None;
         }
     }
+
+    /// True when any message content part already carries a `cache_control` marker
+    /// (client-supplied or previously injected).
+    pub fn has_cache_control_markers(&self) -> bool {
+        self.messages.iter().any(|m| match &m.content {
+            Some(MessageContent::Parts(parts)) => parts.iter().any(|p| {
+                matches!(
+                    p,
+                    ContentPart::Text {
+                        cache_control: Some(_),
+                        ..
+                    }
+                )
+            }),
+            _ => false,
+        })
+    }
+
+    /// Estimated token length of the stable prompt prefix — the system/developer
+    /// message(s) plus tool definitions — using a chars/4 heuristic. Precision is
+    /// not required; this only gates whether marking the prefix is worthwhile.
+    pub fn estimated_cache_prefix_tokens(&self) -> u32 {
+        const CHARS_PER_TOKEN: usize = 4;
+        let mut chars = 0usize;
+        for m in &self.messages {
+            if matches!(m.role, Role::System | Role::Developer) {
+                if let Some(content) = &m.content {
+                    chars += content.extract_text().len();
+                }
+            }
+        }
+        if let Some(tools) = &self.tools {
+            for t in tools {
+                chars += t.function.name.len();
+                chars += t.function.description.as_deref().map_or(0, str::len);
+                chars += t.function.parameters.to_string().len();
+            }
+        }
+        (chars / CHARS_PER_TOKEN) as u32
+    }
+
+    /// Auto-inject a single ephemeral `cache_control` breakpoint at the end of the
+    /// stable prompt prefix, for OpenAI-compatible gateways that proxy Anthropic-family
+    /// models (DigitalOcean, OpenRouter). The marker is attached to the last text
+    /// content part of the system/developer message (falling back to the first message
+    /// when there is no system prompt), normalizing a plain-string content into a
+    /// one-element content-part array as needed.
+    ///
+    /// Idempotent: a request that already carries any `cache_control` marker is left
+    /// untouched. Threshold-guarded: no-op when the estimated stable prefix is below
+    /// `min_prefix_tokens`. Returns true if a marker was injected.
+    pub fn inject_cache_control(&mut self, ttl: Option<String>, min_prefix_tokens: u32) -> bool {
+        if self.has_cache_control_markers() {
+            return false;
+        }
+        if self.estimated_cache_prefix_tokens() < min_prefix_tokens {
+            return false;
+        }
+
+        let target_idx = self
+            .messages
+            .iter()
+            .position(|m| matches!(m.role, Role::System | Role::Developer))
+            .or(if self.messages.is_empty() {
+                None
+            } else {
+                Some(0)
+            });
+
+        let Some(idx) = target_idx else {
+            return false;
+        };
+        let marker = CacheControl {
+            kind: "ephemeral".to_string(),
+            ttl,
+        };
+        attach_cache_control(&mut self.messages[idx], marker)
+    }
+}
+
+/// Attach a `cache_control` marker to the last text content part of `message`,
+/// normalizing string content into a one-element parts array. Returns false when
+/// there is no text part to mark (e.g. an image-only message).
+fn attach_cache_control(message: &mut Message, marker: CacheControl) -> bool {
+    let mut parts = match message.content.take() {
+        Some(MessageContent::Text(text)) => vec![ContentPart::Text {
+            text,
+            cache_control: None,
+        }],
+        Some(MessageContent::Parts(parts)) => parts,
+        None => return false,
+    };
+
+    let marked = if let Some(ContentPart::Text { cache_control, .. }) = parts
+        .iter_mut()
+        .rev()
+        .find(|p| matches!(p, ContentPart::Text { .. }))
+    {
+        *cache_control = Some(marker);
+        true
+    } else {
+        false
+    };
+
+    message.content = Some(MessageContent::Parts(parts));
+    marked
 }
 
 /// True when the upstream model id is Moonshot's Kimi Code endpoint model.
@@ -277,7 +383,7 @@ impl ExtractText for Vec<ContentPart> {
     fn extract_text(&self) -> String {
         self.iter()
             .filter_map(|part| match part {
-                ContentPart::Text { text } => Some(text.as_str()),
+                ContentPart::Text { text, .. } => Some(text.as_str()),
                 _ => None,
             })
             .collect::<Vec<_>>()
@@ -296,11 +402,32 @@ impl Display for MessageContent {
 #[serde(tag = "type")]
 pub enum ContentPart {
     #[serde(rename = "text")]
-    Text { text: String },
+    Text {
+        text: String,
+        /// Prompt-cache breakpoint for OpenAI-compatible gateways that proxy
+        /// Anthropic-family models (e.g. DigitalOcean, OpenRouter). Round-trips so
+        /// client-supplied markers survive deserialization and are respected by the
+        /// idempotent injector.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        cache_control: Option<CacheControl>,
+    },
     #[serde(rename = "image_url")]
     ImageUrl { image_url: ImageUrl },
 }
 
+/// Prompt-cache control marker carried on an OpenAI content part. Mirrors the
+/// Anthropic `cache_control` object as accepted by OpenAI-compatible gateways that
+/// front Anthropic models.
+#[skip_serializing_none]
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+pub struct CacheControl {
+    #[serde(rename = "type")]
+    pub kind: String, // "ephemeral"
+    /// Cache lifetime hint: "5m" | "1h" (DigitalOcean / OpenRouter). Omitted for
+    /// plain Anthropic ephemeral caching (defaults to 5 minutes upstream).
+    pub ttl: Option<String>,
+}
+
 /// Image URL configuration for vision capabilities
 #[skip_serializing_none]
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -445,11 +572,35 @@ pub struct Usage {
     pub total_tokens: u32,
     pub prompt_tokens_details: Option<PromptTokensDetails>,
     pub completion_tokens_details: Option<CompletionTokensDetails>,
+    /// Anthropic-style cache-read counter emitted by OpenAI-compatible gateways that
+    /// front Anthropic models (e.g. DigitalOcean), which do *not* populate
+    /// `prompt_tokens_details.cached_tokens`. Captured here so it can be surfaced to
+    /// OpenAI clients via [`Usage::normalize_cache_tokens`].
+    pub cache_read_input_tokens: Option<u32>,
+    /// Anthropic-style cache-write counter (see `cache_read_input_tokens`).
+    pub cache_creation_input_tokens: Option<u32>,
+}
+
+impl Usage {
+    /// Fold gateway-specific Anthropic-style `cache_read_input_tokens` into the
+    /// OpenAI-standard `prompt_tokens_details.cached_tokens` so OpenAI-compatible
+    /// clients (and downstream cost accounting) observe the cache hit. No-op when the
+    /// standard field is already populated or no cache-read counter is present.
+    pub fn normalize_cache_tokens(&mut self) {
+        if let Some(read) = self.cache_read_input_tokens {
+            let details = self
+                .prompt_tokens_details
+                .get_or_insert_with(Default::default);
+            if details.cached_tokens.is_none() {
+                details.cached_tokens = Some(read);
+            }
+        }
+    }
 }
 
 /// Detailed breakdown of prompt tokens
 #[skip_serializing_none]
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
 pub struct PromptTokensDetails {
     pub cached_tokens: Option<u32>,
     pub audio_tokens: Option<u32>,
@@ -632,7 +783,15 @@ impl TokenUsage for Usage {
     fn cached_input_tokens(&self) -> Option<usize> {
         self.prompt_tokens_details
             .as_ref()
-            .and_then(|d| d.cached_tokens.map(|t| t as usize))
+            .and_then(|d| d.cached_tokens)
+            // Gateways fronting Anthropic (e.g. DigitalOcean) report cache reads here
+            // instead of prompt_tokens_details.cached_tokens.
+            .or(self.cache_read_input_tokens)
+            .map(|t| t as usize)
+    }
+
+    fn cache_creation_tokens(&self) -> Option<usize> {
+        self.cache_creation_input_tokens.map(|t| t as usize)
     }
 
     fn reasoning_tokens(&self) -> Option<usize> {
@@ -666,7 +825,7 @@ impl ProviderRequest for ChatCompletionsRequest {
                     MessageContent::Parts(parts) => parts
                         .iter()
                         .map(|part| match part {
-                            ContentPart::Text { text } => text.clone(),
+                            ContentPart::Text { text, .. } => text.clone(),
                             ContentPart::ImageUrl { .. } => "[Image]".to_string(),
                         })
                         .collect::<Vec<_>>()
@@ -796,6 +955,137 @@ mod tests {
     use super::*;
     use serde_json::json;
 
+    /// Build a request with a long system prompt (well over the token threshold) plus
+    /// a short user turn.
+    fn request_with_system(system: &str, user: &str) -> ChatCompletionsRequest {
+        ChatCompletionsRequest {
+            model: "anthropic/claude-3.5-sonnet".to_string(),
+            messages: vec![
+                Message {
+                    role: Role::System,
+                    content: Some(MessageContent::Text(system.to_string())),
+                    name: None,
+                    tool_calls: None,
+                    tool_call_id: None,
+                },
+                Message {
+                    role: Role::User,
+                    content: Some(MessageContent::Text(user.to_string())),
+                    name: None,
+                    tool_calls: None,
+                    tool_call_id: None,
+                },
+            ],
+            ..Default::default()
+        }
+    }
+
+    fn system_cache_control(req: &ChatCompletionsRequest) -> Option<&CacheControl> {
+        match &req.messages[0].content {
+            Some(MessageContent::Parts(parts)) => parts.iter().find_map(|p| match p {
+                ContentPart::Text { cache_control, .. } => cache_control.as_ref(),
+                _ => None,
+            }),
+            _ => None,
+        }
+    }
+
+    #[test]
+    fn inject_cache_control_normalizes_string_content_and_marks_prefix() {
+        let mut req = request_with_system(&"x".repeat(8000), "hi");
+        let injected = req.inject_cache_control(None, 1024);
+        assert!(injected);
+        // The plain-string system content was normalized into a one-element parts array.
+        let marker = system_cache_control(&req).expect("system content part should be marked");
+        assert_eq!(marker.kind, "ephemeral");
+        assert_eq!(marker.ttl, None);
+        // The user message is untouched.
+        assert!(matches!(
+            req.messages[1].content,
+            Some(MessageContent::Text(_))
+        ));
+    }
+
+    #[test]
+    fn inject_cache_control_is_idempotent() {
+        let mut req = request_with_system(&"x".repeat(8000), "hi");
+        assert!(req.inject_cache_control(Some("1h".to_string()), 1024));
+        // A second pass must be a no-op because a marker already exists.
+        assert!(!req.inject_cache_control(None, 1024));
+        assert_eq!(
+            system_cache_control(&req).and_then(|c| c.ttl.clone()),
+            Some("1h".to_string())
+        );
+    }
+
+    #[test]
+    fn inject_cache_control_respects_min_prefix_threshold() {
+        // A tiny prefix (below the threshold) is not worth marking.
+        let mut req = request_with_system("short system", "hi");
+        assert!(!req.inject_cache_control(None, 1024));
+        assert!(system_cache_control(&req).is_none());
+    }
+
+    #[test]
+    fn client_supplied_cache_control_round_trips_and_blocks_injection() {
+        // A client that already set cache_control must survive deserialization and be
+        // respected (no double injection).
+        let raw = json!({
+            "model": "anthropic/claude-3.5-sonnet",
+            "messages": [
+                {
+                    "role": "system",
+                    "content": [
+                        {"type": "text", "text": "big stable prefix",
+                         "cache_control": {"type": "ephemeral", "ttl": "5m"}}
+                    ]
+                },
+                {"role": "user", "content": "hi"}
+            ]
+        });
+        let mut req: ChatCompletionsRequest = serde_json::from_value(raw).unwrap();
+        assert!(req.has_cache_control_markers());
+        // Injection is a no-op, and re-serialization preserves the client's marker.
+        assert!(!req.inject_cache_control(None, 0));
+        let reserialized = serde_json::to_value(&req).unwrap();
+        let marker = &reserialized["messages"][0]["content"][0]["cache_control"];
+        assert_eq!(marker["type"], "ephemeral");
+        assert_eq!(marker["ttl"], "5m");
+    }
+
+    #[test]
+    fn cache_control_omitted_when_absent() {
+        // A plain text part must not emit a null cache_control field.
+        let part = ContentPart::Text {
+            text: "hello".to_string(),
+            cache_control: None,
+        };
+        let v = serde_json::to_value(&part).unwrap();
+        assert!(v.get("cache_control").is_none());
+    }
+
+    #[test]
+    fn usage_normalizes_gateway_cache_read_into_cached_tokens() {
+        // DigitalOcean-style usage: Anthropic cache_read with no prompt_tokens_details.
+        let raw = json!({
+            "prompt_tokens": 100,
+            "completion_tokens": 20,
+            "total_tokens": 120,
+            "cache_read_input_tokens": 80,
+            "cache_creation_input_tokens": 12
+        });
+        let mut usage: Usage = serde_json::from_value(raw).unwrap();
+        // TokenUsage falls back to the gateway field.
+        assert_eq!(usage.cached_input_tokens(), Some(80));
+        assert_eq!(usage.cache_creation_tokens(), Some(12));
+        // Normalization surfaces it as the OpenAI-standard cached_tokens.
+        usage.normalize_cache_tokens();
+        assert_eq!(
+            usage.prompt_tokens_details.and_then(|d| d.cached_tokens),
+            Some(80)
+        );
+    }
+
     #[test]
     fn test_required_fields() {
         // Create a JSON object with only required fields
@@ -995,7 +1285,7 @@ mod tests {
             assert_eq!(content_parts.len(), 2);
 
             // Validate text content part
-            if let ContentPart::Text { text } = &content_parts[0] {
+            if let ContentPart::Text { text, .. } = &content_parts[0] {
                 assert_eq!(text, "What can you see in this image and what's the weather like in the location shown?");
             } else {
                 panic!("Expected text content part");

crates/hermesllm/src/apis/openai_responses.rs

@@ -183,9 +183,13 @@ pub enum MessageRole {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(tag = "type", rename_all = "snake_case")]
 pub enum InputContent {
-    /// Text input
-    #[serde(rename = "input_text", alias = "text", alias = "output_text")]
+    /// Text input (input-role message content)
+    #[serde(rename = "input_text", alias = "text")]
     InputText { text: String },
+    /// Text produced by the model in a prior turn. This must round-trip as
+    /// `output_text` because the Responses API rejects `input_text` for
+    /// output-role (assistant) message content.
+    OutputText { text: String },
     /// Image input via URL
     InputImage {
         image_url: String,
@@ -595,7 +599,18 @@ pub enum OutputItem {
     /// Reasoning item
     Reasoning {
         id: String,
+        #[serde(default)]
         summary: Vec<serde_json::Value>,
+        /// Reasoning content parts (present on some providers/models).
+        #[serde(default, skip_serializing_if = "Vec::is_empty")]
+        content: Vec<serde_json::Value>,
+        /// Opaque encrypted reasoning payload (present when reasoning is stored
+        /// server-side and returned encrypted).
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        encrypted_content: Option<String>,
+        /// Reasoning item status.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        status: Option<OutputItemStatus>,
     },
 }
 
@@ -830,6 +845,9 @@ pub enum ResponsesAPIStreamEvent {
         output_index: i32,
         content_index: i32,
         delta: String,
+        // Tolerate an omitted `logprobs` (defaults to empty). Note: an explicit
+        // `null` still fails, since serde cannot deserialize null into a Vec.
+        #[serde(default)]
         logprobs: Vec<serde_json::Value>,
         obfuscation: Option<String>,
         sequence_number: i32,
@@ -842,6 +860,9 @@ pub enum ResponsesAPIStreamEvent {
         output_index: i32,
         content_index: i32,
         text: String,
+        // Tolerate an omitted `logprobs` (defaults to empty). Note: an explicit
+        // `null` still fails, since serde cannot deserialize null into a Vec.
+        #[serde(default)]
         logprobs: Vec<serde_json::Value>,
         sequence_number: i32,
     },
@@ -1051,6 +1072,7 @@ pub struct ListInputItemsResponse {
 fn append_input_content_text(buffer: &mut String, content: &InputContent) {
     match content {
         InputContent::InputText { text } => buffer.push_str(text),
+        InputContent::OutputText { text } => buffer.push_str(text),
         InputContent::InputImage { .. } => buffer.push_str("[Image]"),
         InputContent::InputFile { .. } => buffer.push_str("[File]"),
         InputContent::InputAudio { .. } => buffer.push_str("[Audio]"),
@@ -1433,6 +1455,97 @@ mod tests {
     use super::*;
     use serde_json::json;
 
+    /// Task C guard: a reasoning output item deserializes WITHOUT a `summary`
+    /// field (now `#[serde(default)]`) and WITH optional `content`,
+    /// `encrypted_content`, and `status` fields.
+    #[test]
+    fn test_reasoning_item_optional_fields_deserialize() {
+        // No `summary`; carries `content`, `encrypted_content`, `status`.
+        let json = r#"{
+            "type":"reasoning",
+            "id":"rs_abc123",
+            "content":[{"type":"reasoning_text","text":"thinking"}],
+            "encrypted_content":"ZW5jcnlwdGVk",
+            "status":"completed"
+        }"#;
+
+        let item: OutputItem = serde_json::from_str(json).expect("Failed to deserialize");
+        match item {
+            OutputItem::Reasoning {
+                id,
+                summary,
+                content,
+                encrypted_content,
+                status,
+            } => {
+                assert_eq!(id, "rs_abc123");
+                assert!(summary.is_empty(), "omitted summary should default empty");
+                assert_eq!(content.len(), 1);
+                assert_eq!(encrypted_content, Some("ZW5jcnlwdGVk".to_string()));
+                assert!(matches!(status, Some(OutputItemStatus::Completed)));
+            }
+            _ => panic!("Expected OutputItem::Reasoning"),
+        }
+
+        // Minimal reasoning item (only id + type): everything else defaults.
+        let minimal = r#"{"type":"reasoning","id":"rs_min"}"#;
+        let item: OutputItem = serde_json::from_str(minimal).expect("Failed to deserialize");
+        match item {
+            OutputItem::Reasoning {
+                summary,
+                content,
+                encrypted_content,
+                status,
+                ..
+            } => {
+                assert!(summary.is_empty());
+                assert!(content.is_empty());
+                assert!(encrypted_content.is_none());
+                assert!(status.is_none());
+            }
+            _ => panic!("Expected OutputItem::Reasoning"),
+        }
+    }
+
+    /// Task C guard: output text delta/done deserialize with `logprobs` omitted
+    /// (defaults to empty via `#[serde(default)]`).
+    #[test]
+    fn test_output_text_events_deserialize_without_logprobs() {
+        let delta = r#"{
+            "type":"response.output_text.delta",
+            "sequence_number":6,
+            "item_id":"msg_abc",
+            "output_index":1,
+            "content_index":0,
+            "delta":"Hi"
+        }"#;
+        let event: ResponsesAPIStreamEvent =
+            serde_json::from_str(delta).expect("delta without logprobs should deserialize");
+        match event {
+            ResponsesAPIStreamEvent::ResponseOutputTextDelta { logprobs, .. } => {
+                assert!(logprobs.is_empty());
+            }
+            _ => panic!("Expected ResponseOutputTextDelta"),
+        }
+
+        let done = r#"{
+            "type":"response.output_text.done",
+            "sequence_number":7,
+            "item_id":"msg_abc",
+            "output_index":1,
+            "content_index":0,
+            "text":"Hi"
+        }"#;
+        let event: ResponsesAPIStreamEvent =
+            serde_json::from_str(done).expect("done without logprobs should deserialize");
+        match event {
+            ResponsesAPIStreamEvent::ResponseOutputTextDone { logprobs, .. } => {
+                assert!(logprobs.is_empty());
+            }
+            _ => panic!("Expected ResponseOutputTextDone"),
+        }
+    }
+
     #[test]
     fn test_response_output_text_delta_deserialization() {
         let json = r#"{
@@ -1642,6 +1755,62 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_input_content_preserves_output_text_round_trip() {
+        // Multi-turn request: a user turn carrying input_text and a prior
+        // assistant turn carrying output_text. The Responses API rejects
+        // input_text for output-role content, so the assistant turn must
+        // survive a serialize round-trip as output_text (not be rewritten).
+        let request = json!({
+            "model": "gpt-5.3-codex",
+            "input": [
+                {
+                    "role": "user",
+                    "content": [
+                        { "type": "input_text", "text": "hello" }
+                    ]
+                },
+                {
+                    "role": "assistant",
+                    "content": [
+                        { "type": "output_text", "text": "hi there" }
+                    ]
+                }
+            ]
+        });
+
+        let bytes = serde_json::to_vec(&request).unwrap();
+        let parsed = ResponsesAPIRequest::try_from(bytes.as_slice()).unwrap();
+
+        let items = match &parsed.input {
+            InputParam::Items(items) => items,
+            _ => panic!("expected array input"),
+        };
+        assert_eq!(items.len(), 2);
+
+        // Assistant output_text must deserialize into the OutputText variant.
+        let assistant = items
+            .iter()
+            .find_map(|item| match item {
+                InputItem::Message(msg) if matches!(msg.role, MessageRole::Assistant) => Some(msg),
+                _ => None,
+            })
+            .expect("assistant message present");
+        match &assistant.content {
+            MessageContent::Items(contents) => {
+                assert!(matches!(contents[0], InputContent::OutputText { .. }));
+            }
+            _ => panic!("expected array content"),
+        }
+
+        // Round-trip serialize and assert the type tags are preserved:
+        // user content stays input_text, assistant content stays output_text.
+        let serialized = serde_json::to_value(&parsed).unwrap();
+        let input = &serialized["input"];
+        assert_eq!(input[0]["content"][0]["type"], "input_text");
+        assert_eq!(input[1]["content"][0]["type"], "output_text");
+    }
+
     #[test]
     fn test_request_deserializes_text_config_without_format() {
         let request = json!({

crates/hermesllm/src/apis/streaming_shapes/sse.rs

@@ -211,6 +211,24 @@ impl From<SseEvent> for Vec<u8> {
     }
 }
 
+/// Legacy display-text classifier retained for public API compatibility.
+pub fn is_incomplete_json_error<E: std::fmt::Display + ?Sized>(err: &E) -> bool {
+    let msg = err.to_string().to_lowercase();
+    msg.contains("eof while parsing")
+        || msg.contains("unexpected end of json")
+        || msg.contains("unexpected eof")
+}
+
+/// Classify a transformation error as incomplete JSON using serde's stable EOF
+/// category rather than the rendered error text. This is the single source of
+/// truth for internal streaming retry decisions.
+pub(crate) fn is_incomplete_json_error_from_error(
+    err: &(dyn Error + Send + Sync + 'static),
+) -> bool {
+    err.downcast_ref::<serde_json::Error>()
+        .is_some_and(serde_json::Error::is_eof)
+}
+
 #[derive(Debug)]
 pub struct SseParseError {
     pub message: String,
@@ -294,3 +312,43 @@ where
         None
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_incomplete_json_error_only_accepts_serde_eof() {
+        let incomplete_json = serde_json::from_str::<serde_json::Value>(r#"{"partial":"#)
+            .expect_err("truncated JSON must fail");
+        assert!(incomplete_json.is_eof());
+        let incomplete_json: Box<dyn Error + Send + Sync> = Box::new(incomplete_json);
+        assert!(is_incomplete_json_error_from_error(
+            incomplete_json.as_ref()
+        ));
+
+        let invalid_json = serde_json::from_str::<serde_json::Value>(r#"{"value":]}"#)
+            .expect_err("invalid JSON must fail");
+        assert!(!invalid_json.is_eof());
+        let invalid_json: Box<dyn Error + Send + Sync> = Box::new(invalid_json);
+        assert!(!is_incomplete_json_error_from_error(invalid_json.as_ref()));
+
+        let unrelated_eof =
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "unexpected EOF");
+        let unrelated_eof: Box<dyn Error + Send + Sync> = Box::new(unrelated_eof);
+        assert!(!is_incomplete_json_error_from_error(unrelated_eof.as_ref()));
+    }
+
+    #[test]
+    fn test_is_incomplete_json_error_accepts_display_only_callers() {
+        struct DisplayOnly;
+
+        impl std::fmt::Display for DisplayOnly {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                f.write_str("not an error")
+            }
+        }
+
+        assert!(!is_incomplete_json_error(&DisplayOnly));
+    }
+}

crates/hermesllm/src/bin/fetch_models.rs

@@ -10,7 +10,8 @@
 // Usage:
 //   Optional: OPENAI_API_KEY, ANTHROPIC_API_KEY, MISTRAL_API_KEY,
 //             DEEPSEEK_API_KEY, GROK_API_KEY, DASHSCOPE_API_KEY,
-//             MOONSHOT_API_KEY, ZHIPU_API_KEY, MIMO_API_KEY, GOOGLE_API_KEY
+//             MOONSHOT_API_KEY, ZHIPU_API_KEY, MIMO_API_KEY, GOOGLE_API_KEY,
+//             META_MODELS_API_KEY
 //   Optional: AWS CLI configured for Amazon Bedrock models
 //   cargo run --bin fetch_models --features model-fetch
 
@@ -380,6 +381,12 @@ fn fetch_all_models(
             "https://api.xiaomimimo.com/v1/models",
             "xiaomi",
         ),
+        (
+            "meta",
+            "META_MODELS_API_KEY",
+            "https://api.meta.ai/v1/models",
+            "meta",
+        ),
     ];
 
     // Helper that records the outcome of a fetch attempt and only mutates

crates/hermesllm/src/bin/provider_models.yaml

@@ -13,89 +13,12 @@ providers:
   - amazon/amazon.nova-premier-v1:0
   - amazon/amazon.nova-lite-v1:0
   - amazon/amazon.nova-micro-v1:0
-  google:
-  - google/gemini-2.5-flash
-  - google/gemini-2.5-pro
-  - google/gemini-2.0-flash
-  - google/gemini-2.0-flash-001
-  - google/gemini-2.0-flash-lite-001
-  - google/gemini-2.0-flash-lite
-  - google/gemini-2.5-flash-preview-tts
-  - google/gemini-2.5-pro-preview-tts
-  - google/gemma-3-1b-it
-  - google/gemma-3-4b-it
-  - google/gemma-3-12b-it
-  - google/gemma-3-27b-it
-  - google/gemma-3n-e4b-it
-  - google/gemma-3n-e2b-it
-  - google/gemma-4-26b-a4b-it
-  - google/gemma-4-31b-it
-  - google/gemini-flash-latest
-  - google/gemini-flash-lite-latest
-  - google/gemini-pro-latest
-  - google/gemini-2.5-flash-lite
-  - google/gemini-2.5-flash-image
-  - google/gemini-3-pro-preview
-  - google/gemini-3-flash-preview
-  - google/gemini-3.1-pro-preview
-  - google/gemini-3.1-pro-preview-customtools
-  - google/gemini-3.1-flash-lite-preview
-  - google/gemini-3-pro-image-preview
-  - google/nano-banana-pro-preview
-  - google/gemini-3.1-flash-image-preview
-  - google/lyria-3-clip-preview
-  - google/lyria-3-pro-preview
-  - google/gemini-robotics-er-1.5-preview
-  - google/gemini-2.5-computer-use-preview-10-2025
-  - google/deep-research-pro-preview-12-2025
-  mistralai:
-  - mistralai/mistral-medium-2505
-  - mistralai/mistral-medium-2508
-  - mistralai/mistral-medium-latest
-  - mistralai/mistral-medium
-  - mistralai/mistral-vibe-cli-with-tools
-  - mistralai/open-mistral-nemo
-  - mistralai/open-mistral-nemo-2407
-  - mistralai/mistral-tiny-2407
-  - mistralai/mistral-tiny-latest
-  - mistralai/codestral-2508
-  - mistralai/codestral-latest
-  - mistralai/devstral-2512
-  - mistralai/mistral-vibe-cli-latest
-  - mistralai/devstral-medium-latest
-  - mistralai/devstral-latest
-  - mistralai/mistral-small-2603
-  - mistralai/mistral-small-latest
-  - mistralai/mistral-vibe-cli-fast
-  - mistralai/mistral-small-2506
-  - mistralai/magistral-medium-2509
-  - mistralai/magistral-medium-latest
-  - mistralai/magistral-small-2509
-  - mistralai/magistral-small-latest
-  - mistralai/labs-leanstral-2603
-  - mistralai/mistral-large-2512
-  - mistralai/mistral-large-latest
-  - mistralai/ministral-3b-2512
-  - mistralai/ministral-3b-latest
-  - mistralai/ministral-8b-2512
-  - mistralai/ministral-8b-latest
-  - mistralai/ministral-14b-2512
-  - mistralai/ministral-14b-latest
-  - mistralai/mistral-large-2411
-  - mistralai/pixtral-large-2411
-  - mistralai/pixtral-large-latest
-  - mistralai/mistral-large-pixtral-2411
-  - mistralai/devstral-small-2507
-  - mistralai/devstral-medium-2507
-  - mistralai/labs-mistral-small-creative
-  - mistralai/mistral-embed-2312
-  - mistralai/mistral-embed
-  - mistralai/codestral-embed
-  - mistralai/codestral-embed-2505
   anthropic:
+  - anthropic/claude-fable-5
+  - anthropic/claude-opus-4-8
+  - anthropic/claude-opus-4-7
   - anthropic/claude-sonnet-4-6
   - anthropic/claude-opus-4-6
-  - anthropic/claude-opus-4-7
   - anthropic/claude-opus-4-5-20251101
   - anthropic/claude-opus-4-5
   - anthropic/claude-haiku-4-5-20251001
@@ -108,135 +31,172 @@ providers:
   - anthropic/claude-opus-4
   - anthropic/claude-sonnet-4-20250514
   - anthropic/claude-sonnet-4
-  - anthropic/claude-3-haiku-20240307
-  - anthropic/claude-3-haiku
-  qwen:
-  - qwen/qwen3.6-plus-2026-04-02
-  - qwen/qwen3.6-plus
-  - qwen/wan2.7-image
-  - qwen/deepseek-v3.2
-  - qwen/qwen3-asr-flash-2026-02-10
-  - qwen/qwen3.5-flash-2026-02-23
-  - qwen/qwen3.5-flash
-  - qwen/qwen3.5-122b-a10b
-  - qwen/qwen3.5-35b-a3b
-  - qwen/qwen3.5-27b
-  - qwen/qwen3-coder-next
-  - qwen/qwen3.5-397b-a17b
-  - qwen/qwen3.5-plus-2026-02-15
-  - qwen/qwen3.5-plus
-  - qwen/qwen3-vl-flash-2026-01-22
-  - qwen/qwen3-max-2026-01-23
-  - qwen/qwen-plus-character
-  - qwen/qwen-flash-character
-  - qwen/qwen-flash
-  - qwen/qwen3-vl-plus-2025-12-19
-  - qwen/qwen3-omni-flash-2025-12-01
-  - qwen/qwen3-livetranslate-flash-2025-12-01
-  - qwen/qwen3-livetranslate-flash
-  - qwen/qwen-mt-lite
-  - qwen/qwen-plus-2025-12-01
-  - qwen/qwen-mt-flash
-  - qwen/ccai-pro
-  - qwen/tongyi-tingwu-slp
-  - qwen/qwen3-vl-flash
-  - qwen/qwen3-vl-flash-2025-10-15
-  - qwen/qwen3-omni-flash
-  - qwen/qwen3-omni-flash-2025-09-15
-  - qwen/qwen3-omni-30b-a3b-captioner
-  - qwen/qwen2.5-7b-instruct
-  - qwen/qwen2.5-14b-instruct
-  - qwen/qwen2.5-32b-instruct
-  - qwen/qwen2.5-72b-instruct
-  - qwen/qwen2.5-14b-instruct-1m
-  - qwen/qwen2.5-7b-instruct-1m
-  - qwen/qwen-max-2025-01-25
-  - qwen/qwen-max-latest
-  - qwen/qwen-turbo-2024-11-01
-  - qwen/qwen-turbo-latest
-  - qwen/qwen-plus-latest
-  - qwen/qwen-plus-2025-01-25
-  - qwen/qwq-plus-2025-03-05
-  - qwen/qwen-mt-turbo
-  - qwen/qwen-mt-plus
-  - qwen/qwen-coder-plus
-  - qwen/qwq-plus
-  - qwen/qwen2.5-vl-32b-instruct
-  - qwen/qvq-max
-  - qwen/qwen-omni-turbo
-  - qwen/qwen3-8b
-  - qwen/qwen3-30b-a3b
-  - qwen/qwen3-235b-a22b
-  - qwen/qwen-turbo-2025-04-28
-  - qwen/qwen-plus-2025-04-28
-  - qwen/qwen-vl-max-2025-04-08
-  - qwen/qwen-vl-plus-2025-01-25
-  - qwen/qwen-vl-plus-latest
-  - qwen/qwen-vl-max-latest
-  - qwen/qwen-vl-plus-2025-05-07
-  - qwen/qwen3-coder-plus
-  - qwen/qwen3-coder-480b-a35b-instruct
-  - qwen/qwen3-235b-a22b-instruct-2507
-  - qwen/qwen-plus-2025-07-14
-  - qwen/qwen3-coder-plus-2025-07-22
-  - qwen/qwen3-235b-a22b-thinking-2507
-  - qwen/qwen3-coder-flash
-  - qwen/qwen-vl-max
-  - qwen/qwen-vl-max-2025-08-13
-  - qwen/qwen3-max
-  - qwen/qwen3-max-2025-09-23
-  - qwen/qwen3-vl-plus
-  - qwen/qwen3-vl-235b-a22b-instruct
-  - qwen/qwen3-vl-235b-a22b-thinking
-  - qwen/qwen3-30b-a3b-thinking-2507
-  - qwen/qwen3-30b-a3b-instruct-2507
-  - qwen/qwen3-14b
-  - qwen/qwen3-32b
-  - qwen/qwen3-0.6b
-  - qwen/qwen3-4b
-  - qwen/qwen3-1.7b
-  - qwen/qwen-vl-plus
-  - qwen/qwen3-coder-plus-2025-09-23
-  - qwen/qwen3-vl-plus-2025-09-23
-  - qwen/qwen-plus-2025-09-11
-  - qwen/qwen3-next-80b-a3b-thinking
-  - qwen/qwen3-next-80b-a3b-instruct
-  - qwen/qwen3-max-preview
-  - qwen/qwen2-7b-instruct
-  - qwen/qwen-max
-  - qwen/qwen-plus
-  - qwen/qwen-turbo
-  z-ai:
-  - z-ai/glm-4.5
-  - z-ai/glm-4.5-air
-  - z-ai/glm-4.6
-  - z-ai/glm-4.7
-  - z-ai/glm-5
-  - z-ai/glm-5-turbo
-  - z-ai/glm-5.1
-  x-ai:
-  - x-ai/grok-3
-  - x-ai/grok-3-mini
-  - x-ai/grok-4-0709
-  - x-ai/grok-4-1-fast-non-reasoning
-  - x-ai/grok-4-1-fast-reasoning
-  - x-ai/grok-4-fast-non-reasoning
-  - x-ai/grok-4-fast-reasoning
-  - x-ai/grok-4.20-0309-non-reasoning
-  - x-ai/grok-4.20-0309-reasoning
-  - x-ai/grok-4.20-multi-agent-0309
-  - x-ai/grok-code-fast-1
-  - x-ai/grok-imagine-image
-  - x-ai/grok-imagine-video
+  chatgpt:
+  - chatgpt/gpt-5.4
+  - chatgpt/gpt-5.3-codex
+  - chatgpt/gpt-5.2
+  - chatgpt/gpt-5.6-sol
+  - chatgpt/gpt-5.6-terra
+  - chatgpt/gpt-5.6-luna
+  deepseek:
+  - deepseek/deepseek-v4-flash
+  - deepseek/deepseek-v4-pro
+  digitalocean:
+  - digitalocean/openai-gpt-4.1
+  - digitalocean/openai-gpt-4o
+  - digitalocean/openai-gpt-4o-mini
+  - digitalocean/openai-gpt-5
+  - digitalocean/openai-gpt-5-mini
+  - digitalocean/openai-gpt-5-nano
+  - digitalocean/openai-gpt-5.1-codex-max
+  - digitalocean/openai-gpt-5.2
+  - digitalocean/openai-gpt-5.2-pro
+  - digitalocean/openai-gpt-5.3-codex
+  - digitalocean/openai-gpt-5.4
+  - digitalocean/openai-gpt-5.4-mini
+  - digitalocean/openai-gpt-5.4-nano
+  - digitalocean/openai-gpt-5.4-pro
+  - digitalocean/openai-gpt-oss-120b
+  - digitalocean/openai-gpt-oss-20b
+  - digitalocean/openai-o1
+  - digitalocean/openai-o3
+  - digitalocean/openai-o3-mini
+  - digitalocean/anthropic-claude-4.1-opus
+  - digitalocean/anthropic-claude-4.5-sonnet
+  - digitalocean/anthropic-claude-4.6-sonnet
+  - digitalocean/anthropic-claude-haiku-4.5
+  - digitalocean/anthropic-claude-opus-4
+  - digitalocean/anthropic-claude-opus-4.5
+  - digitalocean/anthropic-claude-opus-4.6
+  - digitalocean/anthropic-claude-opus-4.7
+  - digitalocean/anthropic-claude-sonnet-4
+  - digitalocean/alibaba-qwen3-32b
+  - digitalocean/arcee-trinity-large-thinking
+  - digitalocean/deepseek-3.2
+  - digitalocean/deepseek-r1-distill-llama-70b
+  - digitalocean/gemma-4-31B-it
+  - digitalocean/glm-5
+  - digitalocean/kimi-k2.5
+  - digitalocean/llama3.3-70b-instruct
+  - digitalocean/minimax-m2.5
+  - digitalocean/nvidia-nemotron-3-super-120b
+  - digitalocean/qwen3-coder-flash
+  - digitalocean/qwen3.5-397b-a17b
+  - digitalocean/all-mini-lm-l6-v2
+  - digitalocean/gte-large-en-v1.5
+  - digitalocean/multi-qa-mpnet-base-dot-v1
+  - digitalocean/qwen3-embedding-0.6b
+  - digitalocean/router:software-engineering
+  google:
+  - google/gemini-2.5-flash
+  - google/gemini-2.5-pro
+  - google/gemini-2.0-flash
+  - google/gemini-2.0-flash-001
+  - google/gemini-2.0-flash-lite-001
+  - google/gemini-2.0-flash-lite
+  - google/gemini-2.5-flash-preview-tts
+  - google/gemini-2.5-pro-preview-tts
+  - google/gemma-4-26b-a4b-it
+  - google/gemma-4-31b-it
+  - google/gemini-flash-latest
+  - google/gemini-flash-lite-latest
+  - google/gemini-pro-latest
+  - google/gemini-2.5-flash-lite
+  - google/gemini-2.5-flash-image
+  - google/gemini-3-pro-preview
+  - google/gemini-3-flash-preview
+  - google/gemini-3.1-pro-preview
+  - google/gemini-3.1-pro-preview-customtools
+  - google/gemini-3.1-flash-lite-preview
+  - google/gemini-3.1-flash-lite
+  - google/gemini-3-pro-image-preview
+  - google/gemini-3-pro-image
+  - google/nano-banana-pro-preview
+  - google/gemini-3.1-flash-image-preview
+  - google/gemini-3.1-flash-image
+  - google/gemini-3.5-flash
+  - google/lyria-3-clip-preview
+  - google/lyria-3-pro-preview
+  - google/gemini-3.1-flash-tts-preview
+  - google/gemini-robotics-er-1.5-preview
+  - google/gemini-robotics-er-1.6-preview
+  - google/gemini-2.5-computer-use-preview-10-2025
+  - google/antigravity-preview-05-2026
+  - google/deep-research-max-preview-04-2026
+  - google/deep-research-preview-04-2026
+  - google/deep-research-pro-preview-12-2025
+  meta:
+  - meta/muse-spark-1.1
+  minimax:
+  - minimax/MiniMax-M3
+  mistralai:
+  - mistralai/mistral-medium-2505
+  - mistralai/mistral-medium-2508
+  - mistralai/mistral-medium-latest
+  - mistralai/mistral-medium
+  - mistralai/mistral-vibe-cli-with-tools
+  - mistralai/open-mistral-nemo
+  - mistralai/open-mistral-nemo-2407
+  - mistralai/mistral-tiny-2407
+  - mistralai/mistral-tiny-latest
+  - mistralai/codestral-2508
+  - mistralai/codestral-latest
+  - mistralai/mistral-code-latest
+  - mistralai/mistral-code-fim-latest
+  - mistralai/devstral-2512
+  - mistralai/devstral-medium-latest
+  - mistralai/devstral-latest
+  - mistralai/mistral-code-agent-latest
+  - mistralai/mistral-small-2603
+  - mistralai/mistral-small-latest
+  - mistralai/mistral-vibe-cli-fast
+  - mistralai/magistral-small-latest
+  - mistralai/magistral-medium-2509
+  - mistralai/magistral-medium-latest
+  - mistralai/labs-leanstral-2603
+  - mistralai/mistral-large-2512
+  - mistralai/mistral-large-latest
+  - mistralai/mistral-large-2512
+  - mistralai/mistral-large-latest
+  - mistralai/ministral-3b-2512
+  - mistralai/ministral-3b-latest
+  - mistralai/ministral-8b-2512
+  - mistralai/ministral-8b-latest
+  - mistralai/ministral-14b-2512
+  - mistralai/ministral-14b-latest
+  - mistralai/mistral-medium-3-5
+  - mistralai/mistral-medium-3.5
+  - mistralai/mistral-medium-3
+  - mistralai/mistral-medium-2604
+  - mistralai/mistral-medium-c21211-r0-75
+  - mistralai/mistral-vibe-cli-latest
+  - mistralai/mistral-medium-3-5
+  - mistralai/mistral-medium-3.5
+  - mistralai/mistral-medium-3
+  - mistralai/mistral-medium-2604
+  - mistralai/mistral-medium-c21211-r0-75
+  - mistralai/mistral-vibe-cli-latest
+  - mistralai/magistral-small-2509
+  - mistralai/mistral-small-2506
+  - mistralai/mistral-embed-2312
+  - mistralai/mistral-embed
+  - mistralai/codestral-embed
+  - mistralai/codestral-embed-2505
+  moonshotai:
+  - moonshotai/kimi-k2.5
+  - moonshotai/kimi-k2.6
+  - moonshotai/moonshot-v1-32k
+  - moonshotai/moonshot-v1-8k
+  - moonshotai/moonshot-v1-128k-vision-preview
+  - moonshotai/moonshot-v1-auto
+  - moonshotai/moonshot-v1-8k-vision-preview
+  - moonshotai/moonshot-v1-128k
+  - moonshotai/moonshot-v1-32k-vision-preview
   openai:
+  - openai/gpt-3.5-turbo
+  - openai/gpt-3.5-turbo-16k
   - openai/gpt-4-0613
   - openai/gpt-4
-  - openai/gpt-3.5-turbo
-  - openai/gpt-5.4-mini
-  - openai/gpt-5.4
-  - openai/gpt-5.4-nano-2026-03-17
-  - openai/gpt-5.4-nano
-  - openai/gpt-5.4-mini-2026-03-17
   - openai/gpt-3.5-turbo-instruct
   - openai/gpt-3.5-turbo-instruct-0914
   - openai/gpt-3.5-turbo-1106
@@ -306,81 +266,137 @@ providers:
   - openai/gpt-5.4-2026-03-05
   - openai/gpt-5.4-pro
   - openai/gpt-5.4-pro-2026-03-05
-  - openai/gpt-3.5-turbo-16k
+  - openai/gpt-5.4
+  - openai/gpt-5.4-nano-2026-03-17
+  - openai/gpt-5.4-nano
+  - openai/gpt-5.4-mini-2026-03-17
+  - openai/gpt-5.4-mini
+  - openai/gpt-5.5
+  - openai/gpt-5.5-2026-04-23
+  - openai/gpt-5.5-pro
+  - openai/gpt-5.5-pro-2026-04-23
+  - openai/chat-latest
   - openai/ft:gpt-3.5-turbo-0613:katanemo::8CMZbm0P
-  deepseek:
-  - deepseek/deepseek-chat
-  - deepseek/deepseek-reasoner
-  moonshotai:
-  - moonshotai/kimi-for-coding
-  - moonshotai/kimi-k2-thinking
-  - moonshotai/moonshot-v1-auto
-  - moonshotai/moonshot-v1-32k-vision-preview
-  - moonshotai/moonshot-v1-128k
-  - moonshotai/kimi-k2-turbo-preview
-  - moonshotai/kimi-k2-0905-preview
-  - moonshotai/moonshot-v1-128k-vision-preview
-  - moonshotai/moonshot-v1-32k
-  - moonshotai/moonshot-v1-8k-vision-preview
-  - moonshotai/kimi-k2.5
-  - moonshotai/moonshot-v1-8k
-  - moonshotai/kimi-k2-thinking-turbo
-  - moonshotai/kimi-k2-0711-preview
+  qwen:
+  - qwen/qwen3.7-plus-2026-05-26
+  - qwen/qwen3.7-plus
+  - qwen/kimi-k2.6
+  - qwen/glm-5.1
+  - qwen/qwen3.7-max-2026-05-17
+  - qwen/qwen3.7-max-preview
+  - qwen/qwen3.7-max-2026-05-20
+  - qwen/qwen3.7-max
+  - qwen/deepseek-v4-flash
+  - qwen/deepseek-v4-pro
+  - qwen/qwen3.6-27b
+  - qwen/qwen3.5-plus-2026-04-20
+  - qwen/qwen3.6-max-preview
+  - qwen/qwen3.6-35b-a3b
+  - qwen/qwen3.6-flash
+  - qwen/qwen3.6-flash-2026-04-16
+  - qwen/qwen3.5-omni-plus-2026-03-15
+  - qwen/qwen3.5-omni-plus
+  - qwen/qwen3.5-omni-flash-2026-03-15
+  - qwen/qwen3.5-omni-flash
+  - qwen/qwen3.6-plus-2026-04-02
+  - qwen/qwen3.6-plus
+  - qwen/wan2.7-image
+  - qwen/deepseek-v3.2
+  - qwen/qwen3-asr-flash-2026-02-10
+  - qwen/qwen3.5-flash-2026-02-23
+  - qwen/qwen3.5-flash
+  - qwen/qwen3.5-122b-a10b
+  - qwen/qwen3.5-35b-a3b
+  - qwen/qwen3.5-27b
+  - qwen/qwen3-coder-next
+  - qwen/qwen3.5-397b-a17b
+  - qwen/qwen3.5-plus-2026-02-15
+  - qwen/qwen3.5-plus
+  - qwen/qwen3-vl-flash-2026-01-22
+  - qwen/qwen3-max-2026-01-23
+  - qwen/qwen-plus-character
+  - qwen/qwen-flash-character
+  - qwen/qwen-flash
+  - qwen/qwen3-vl-plus-2025-12-19
+  - qwen/qwen3-omni-flash-2025-12-01
+  - qwen/qwen3-livetranslate-flash-2025-12-01
+  - qwen/qwen3-livetranslate-flash
+  - qwen/qwen-mt-lite
+  - qwen/qwen-plus-2025-12-01
+  - qwen/qwen-mt-flash
+  - qwen/ccai-pro
+  - qwen/tongyi-tingwu-slp
+  - qwen/qwen3-vl-flash
+  - qwen/qwen3-vl-flash-2025-10-15
+  - qwen/qwen3-omni-flash
+  - qwen/qwen3-omni-flash-2025-09-15
+  - qwen/qwen3-omni-30b-a3b-captioner
+  - qwen/qwen-plus-latest
+  - qwen/qwen-plus-2025-01-25
+  - qwen/qwq-plus-2025-03-05
+  - qwen/qwen-mt-turbo
+  - qwen/qwen-mt-plus
+  - qwen/qwen-coder-plus
+  - qwen/qwq-plus
+  - qwen/qvq-max
+  - qwen/qwen-omni-turbo
+  - qwen/qwen3-8b
+  - qwen/qwen3-30b-a3b
+  - qwen/qwen3-235b-a22b
+  - qwen/qwen-plus-2025-04-28
+  - qwen/qwen3-coder-plus
+  - qwen/qwen3-coder-480b-a35b-instruct
+  - qwen/qwen3-235b-a22b-instruct-2507
+  - qwen/qwen-plus-2025-07-14
+  - qwen/qwen3-coder-plus-2025-07-22
+  - qwen/qwen3-235b-a22b-thinking-2507
+  - qwen/qwen3-coder-flash
+  - qwen/qwen-vl-max
+  - qwen/qwen3-max
+  - qwen/qwen3-max-2025-09-23
+  - qwen/qwen3-vl-plus
+  - qwen/qwen3-vl-235b-a22b-instruct
+  - qwen/qwen3-vl-235b-a22b-thinking
+  - qwen/qwen3-30b-a3b-thinking-2507
+  - qwen/qwen3-30b-a3b-instruct-2507
+  - qwen/qwen3-14b
+  - qwen/qwen3-32b
+  - qwen/qwen-vl-plus
+  - qwen/qwen3-coder-plus-2025-09-23
+  - qwen/qwen3-vl-plus-2025-09-23
+  - qwen/qwen-plus-2025-09-11
+  - qwen/qwen3-next-80b-a3b-thinking
+  - qwen/qwen3-next-80b-a3b-instruct
+  - qwen/qwen3-max-preview
+  - qwen/qwen2-7b-instruct
+  - qwen/qwen-max
+  - qwen/qwen-plus
+  - qwen/qwen-turbo
+  x-ai:
+  - x-ai/grok-4.20-0309-non-reasoning
+  - x-ai/grok-4.20-0309-reasoning
+  - x-ai/grok-4.20-multi-agent-0309
+  - x-ai/grok-4.3
+  - x-ai/grok-build-0.1
+  - x-ai/grok-imagine-image
+  - x-ai/grok-imagine-video
+  - x-ai/grok-imagine-video-1.5-preview
   xiaomi:
   - xiaomi/mimo-v2-flash
   - xiaomi/mimo-v2-omni
   - xiaomi/mimo-v2-pro
-  chatgpt:
-  - chatgpt/gpt-5.4
-  - chatgpt/gpt-5.3-codex
-  - chatgpt/gpt-5.2
-  digitalocean:
-  - digitalocean/openai-gpt-4.1
-  - digitalocean/openai-gpt-4o
-  - digitalocean/openai-gpt-4o-mini
-  - digitalocean/openai-gpt-5
-  - digitalocean/openai-gpt-5-mini
-  - digitalocean/openai-gpt-5-nano
-  - digitalocean/openai-gpt-5.1-codex-max
-  - digitalocean/openai-gpt-5.2
-  - digitalocean/openai-gpt-5.2-pro
-  - digitalocean/openai-gpt-5.3-codex
-  - digitalocean/openai-gpt-5.4
-  - digitalocean/openai-gpt-5.4-mini
-  - digitalocean/openai-gpt-5.4-nano
-  - digitalocean/openai-gpt-5.4-pro
-  - digitalocean/openai-gpt-oss-120b
-  - digitalocean/openai-gpt-oss-20b
-  - digitalocean/openai-o1
-  - digitalocean/openai-o3
-  - digitalocean/openai-o3-mini
-  - digitalocean/anthropic-claude-4.1-opus
-  - digitalocean/anthropic-claude-4.5-sonnet
-  - digitalocean/anthropic-claude-4.6-sonnet
-  - digitalocean/anthropic-claude-haiku-4.5
-  - digitalocean/anthropic-claude-opus-4
-  - digitalocean/anthropic-claude-opus-4.5
-  - digitalocean/anthropic-claude-opus-4.6
-  - digitalocean/anthropic-claude-opus-4.7
-  - digitalocean/anthropic-claude-sonnet-4
-  - digitalocean/alibaba-qwen3-32b
-  - digitalocean/arcee-trinity-large-thinking
-  - digitalocean/deepseek-3.2
-  - digitalocean/deepseek-r1-distill-llama-70b
-  - digitalocean/gemma-4-31B-it
-  - digitalocean/glm-5
-  - digitalocean/kimi-k2.5
-  - digitalocean/llama3.3-70b-instruct
-  - digitalocean/minimax-m2.5
-  - digitalocean/nvidia-nemotron-3-super-120b
-  - digitalocean/qwen3-coder-flash
-  - digitalocean/qwen3.5-397b-a17b
-  - digitalocean/all-mini-lm-l6-v2
-  - digitalocean/gte-large-en-v1.5
-  - digitalocean/multi-qa-mpnet-base-dot-v1
-  - digitalocean/qwen3-embedding-0.6b
-  - digitalocean/router:software-engineering
+  - xiaomi/mimo-v2.5
+  - xiaomi/mimo-v2.5-asr
+  - xiaomi/mimo-v2.5-pro
+  z-ai:
+  - z-ai/glm-4.5
+  - z-ai/glm-4.5-air
+  - z-ai/glm-4.6
+  - z-ai/glm-4.7
+  - z-ai/glm-5
+  - z-ai/glm-5-turbo
+  - z-ai/glm-5.1
 metadata:
-  total_providers: 13
-  total_models: 364
-  last_updated: 2026-04-20T00:00:00.000000+00:00
+  total_providers: 15
+  total_models: 380
+  last_updated: 2026-07-09T19:48:06.553850+00:00

crates/hermesllm/src/lib.rs

@@ -6,10 +6,14 @@ pub mod clients;
 pub mod providers;
 pub mod transforms;
 // Re-export important types and traits
+pub use apis::openai::CacheControl;
 pub use apis::streaming_shapes::amazon_bedrock_binary_frame::BedrockBinaryFrameDecoder;
 pub use apis::streaming_shapes::sse::{SseEvent, SseStreamIter};
 pub use aws_smithy_eventstream::frame::DecodedFrame;
-pub use providers::id::ProviderId;
+pub use providers::id::{
+    cache_marker_strategy, provider_cache_capability, CacheMarkerStrategy, ProviderCacheCapability,
+    ProviderId,
+};
 pub use providers::request::{ProviderRequest, ProviderRequestError, ProviderRequestType};
 pub use providers::response::{
     ProviderResponse, ProviderResponseError, ProviderResponseType, TokenUsage,

crates/hermesllm/src/providers/id.rs

@@ -4,6 +4,7 @@ use serde::Deserialize;
 use std::collections::HashMap;
 use std::fmt::Display;
 use std::sync::OnceLock;
+use std::time::Duration;
 
 static PROVIDER_MODELS_YAML: &str = include_str!(concat!(
     env!("CARGO_MANIFEST_DIR"),
@@ -50,6 +51,8 @@ pub enum ProviderId {
     OpenRouter,
     Astraflow,
     AstraflowCN,
+    Meta,
+    Minimax,
 }
 
 impl TryFrom<&str> for ProviderId {
@@ -85,11 +88,199 @@ impl TryFrom<&str> for ProviderId {
             "openrouter" => Ok(ProviderId::OpenRouter),
             "astraflow" => Ok(ProviderId::Astraflow),
             "astraflow_cn" => Ok(ProviderId::AstraflowCN),
+            "meta" => Ok(ProviderId::Meta),
+            "minimax" => Ok(ProviderId::Minimax),
             _ => Err(format!("Unknown provider: {}", value)),
         }
     }
 }
 
+/// How Plano should mark a request for prompt caching, resolved from the *combination*
+/// of gateway provider, the underlying model family, and the upstream API shape — not
+/// the gateway alone. This is what lets DigitalOcean-Anthropic and OpenRouter-Anthropic
+/// (both OpenAI-compatible chat completions fronting Anthropic models) cache through one
+/// path, while OpenAI-family models stay correctly automatic and unimplemented backends
+/// (Bedrock) are an honest `None` rather than a silent no-op.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum CacheMarkerStrategy {
+    /// No known prompt-caching support for this combination — do nothing.
+    None,
+    /// Provider caches stable prefixes automatically (OpenAI-family anywhere); no
+    /// request markers are needed. Plano only keeps the prefix byte-stable and pinned.
+    Automatic,
+    /// OpenAI-compatible chat completions fronting Anthropic-family models
+    /// (DigitalOcean, OpenRouter): attach `cache_control` to content parts.
+    OpenAiContentPartCacheControl {
+        /// Minimum cacheable prefix length in tokens; injecting below this is a no-op.
+        min_prefix_tokens: u32,
+        /// Optional cache lifetime hint ("5m" | "1h").
+        ttl: Option<String>,
+    },
+    /// Native Anthropic Messages API (native `anthropic/*`, Vercel-Anthropic): inject
+    /// ephemeral breakpoints on the Anthropic-shaped request.
+    AnthropicMessagesBreakpoints {
+        /// Maximum number of cache breakpoints the provider accepts per request.
+        max_breakpoints: u8,
+        /// Minimum cacheable prefix length in tokens; injecting below this is a no-op.
+        min_prefix_tokens: u32,
+    },
+    // BedrockCachePoint { .. } // left as explicit `None` until implemented.
+}
+
+/// Coarse model family, inferred from the model id. Works across gateway naming
+/// conventions (DigitalOcean's `anthropic-claude-…` dash form and OpenRouter's
+/// `anthropic/claude-…` slash form) via substring matching.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ModelFamily {
+    Anthropic,
+    OpenAI,
+    Other,
+}
+
+fn model_family(model_name: &str) -> ModelFamily {
+    let m = model_name.to_ascii_lowercase();
+    if m.contains("claude") || m.contains("anthropic") {
+        ModelFamily::Anthropic
+    } else if m.contains("gpt") || m.contains("openai") || m.contains("chatgpt") {
+        ModelFamily::OpenAI
+    } else {
+        ModelFamily::Other
+    }
+}
+
+/// Whether a gateway accepts Anthropic-style `cache_control` on OpenAI content parts
+/// over its chat-completions endpoint.
+fn accepts_openai_content_part_cache_control(provider: ProviderId) -> bool {
+    matches!(provider, ProviderId::DigitalOcean | ProviderId::OpenRouter)
+}
+
+/// Whether a gateway/model relies on automatic prefix caching (no markers required)
+/// over an OpenAI-compatible surface.
+fn is_automatic_cache_provider(provider: ProviderId) -> bool {
+    matches!(
+        provider,
+        ProviderId::OpenAI
+            | ProviderId::AzureOpenAI
+            | ProviderId::ChatGPT
+            | ProviderId::Groq
+            | ProviderId::Deepseek
+            | ProviderId::Gemini
+            | ProviderId::Moonshotai
+            | ProviderId::XAI
+            | ProviderId::DigitalOcean
+            | ProviderId::OpenRouter
+    )
+}
+
+/// Resolve the cache-marking strategy for a `(gateway provider × underlying model ×
+/// upstream API)` combination.
+///
+/// - `model_name` is the id *after* the gateway prefix (e.g. `anthropic-claude-3-5-sonnet`
+///   for DigitalOcean, `anthropic/claude-3.5-sonnet` for OpenRouter).
+pub fn cache_marker_strategy(
+    provider: ProviderId,
+    model_name: &str,
+    upstream_api: &SupportedUpstreamAPIs,
+) -> CacheMarkerStrategy {
+    // Anthropic minimum cacheable prefix is ~1024 tokens (2048 for Haiku-class);
+    // callers may raise this via config.
+    const ANTHROPIC_MIN_PREFIX_TOKENS: u32 = 1024;
+
+    match upstream_api {
+        // Native Anthropic Messages API — inject ephemeral breakpoints.
+        SupportedUpstreamAPIs::AnthropicMessagesAPI(_) => {
+            CacheMarkerStrategy::AnthropicMessagesBreakpoints {
+                max_breakpoints: 4,
+                min_prefix_tokens: ANTHROPIC_MIN_PREFIX_TOKENS,
+            }
+        }
+        // OpenAI-compatible chat completions — strategy depends on the model family.
+        SupportedUpstreamAPIs::OpenAIChatCompletions(_) => match model_family(model_name) {
+            ModelFamily::Anthropic if accepts_openai_content_part_cache_control(provider) => {
+                CacheMarkerStrategy::OpenAiContentPartCacheControl {
+                    min_prefix_tokens: ANTHROPIC_MIN_PREFIX_TOKENS,
+                    ttl: None,
+                }
+            }
+            // Anthropic-family behind a gateway that doesn't accept content-part
+            // cache_control over chat completions: no honest way to mark it.
+            ModelFamily::Anthropic => CacheMarkerStrategy::None,
+            ModelFamily::OpenAI => CacheMarkerStrategy::Automatic,
+            ModelFamily::Other if is_automatic_cache_provider(provider) => {
+                CacheMarkerStrategy::Automatic
+            }
+            ModelFamily::Other => CacheMarkerStrategy::None,
+        },
+        // OpenAI Responses API — OpenAI-family automatic prefix caching.
+        SupportedUpstreamAPIs::OpenAIResponsesAPI(_) => CacheMarkerStrategy::Automatic,
+        // Bedrock cache points not yet implemented — honest None instead of a
+        // silent no-op.
+        SupportedUpstreamAPIs::AmazonBedrockConverse(_)
+        | SupportedUpstreamAPIs::AmazonBedrockConverseStream(_) => CacheMarkerStrategy::None,
+    }
+}
+
+/// Provider prompt-cache retention behavior, used to decide whether a session's
+/// upstream cache is still plausibly warm from the time since it was last used.
+///
+/// This is deliberately time/behavior only — it says nothing about *how* to mark a
+/// request for caching (that's [`CacheMarkerStrategy`]). Warmth is a function of the
+/// idle gap vs the provider's cache window, so the session router can reason about
+/// stickiness without ever seeing a provider response.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ProviderCacheCapability {
+    /// Sliding idle window: the cache stays warm as long as it is touched at least
+    /// this often. Anthropic's default ephemeral cache is 5 minutes.
+    pub idle_ttl: Duration,
+    /// Absolute ceiling on how long a cache entry can live regardless of activity.
+    /// Conservative default of 1h matches Anthropic's extended (1h) tier ceiling.
+    pub hard_ttl: Duration,
+    /// Whether the provider (as configured) actually retains caches out to the
+    /// extended window. Off by default — extended retention is opt-in per provider.
+    pub extended_retention: bool,
+    /// The extended idle window when `extended_retention` is enabled (e.g. 1h).
+    pub extended_ttl: Duration,
+}
+
+impl Default for ProviderCacheCapability {
+    fn default() -> Self {
+        // Conservative, provider-agnostic defaults: a 5-minute sliding window capped
+        // at 1 hour, no extended retention. Anything unknown is treated as short-lived
+        // so the router doesn't over-stick to a cache that has likely gone cold.
+        ProviderCacheCapability {
+            idle_ttl: Duration::from_secs(5 * 60),
+            hard_ttl: Duration::from_secs(60 * 60),
+            extended_retention: false,
+            extended_ttl: Duration::from_secs(60 * 60),
+        }
+    }
+}
+
+/// Resolve the prompt-cache retention window for a gateway provider. Data-driven so
+/// tuning a provider's window needs no code changes at the call sites — only this
+/// table. Unknown providers fall back to the conservative [`ProviderCacheCapability::default`].
+pub fn provider_cache_capability(provider: ProviderId) -> ProviderCacheCapability {
+    match provider {
+        // Anthropic-family caches (native or fronted): 5-minute sliding default,
+        // 1-hour hard ceiling. Extended (1h) retention is opt-in and left off here.
+        ProviderId::Anthropic
+        | ProviderId::DigitalOcean
+        | ProviderId::OpenRouter
+        | ProviderId::Vercel => ProviderCacheCapability::default(),
+        // OpenAI-family automatic prefix caching also lives on the order of minutes;
+        // the conservative default holds.
+        ProviderId::OpenAI
+        | ProviderId::AzureOpenAI
+        | ProviderId::ChatGPT
+        | ProviderId::Groq
+        | ProviderId::Deepseek
+        | ProviderId::Gemini
+        | ProviderId::Moonshotai
+        | ProviderId::XAI => ProviderCacheCapability::default(),
+        _ => ProviderCacheCapability::default(),
+    }
+}
+
 impl ProviderId {
     /// Get all available models for this provider
     /// Returns model names without the provider prefix (e.g., "gpt-4" not "openai/gpt-4")
@@ -111,6 +302,8 @@ impl ProviderId {
             ProviderId::Qwen => "qwen",
             ProviderId::ChatGPT => "chatgpt",
             ProviderId::DigitalOcean => "digitalocean",
+            ProviderId::Meta => "meta",
+            ProviderId::Minimax => "minimax",
             ProviderId::Astraflow | ProviderId::AstraflowCN => return Vec::new(),
             _ => return Vec::new(),
         };
@@ -181,7 +374,9 @@ impl ProviderId {
                 | ProviderId::OpenRouter
                 | ProviderId::ChatGPT
                 | ProviderId::Astraflow
-                | ProviderId::AstraflowCN,
+                | ProviderId::AstraflowCN
+                | ProviderId::Meta
+                | ProviderId::Minimax,
                 SupportedAPIsFromClient::AnthropicMessagesAPI(_),
             ) => SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
 
@@ -205,7 +400,9 @@ impl ProviderId {
                 | ProviderId::OpenRouter
                 | ProviderId::ChatGPT
                 | ProviderId::Astraflow
-                | ProviderId::AstraflowCN,
+                | ProviderId::AstraflowCN
+                | ProviderId::Meta
+                | ProviderId::Minimax,
                 SupportedAPIsFromClient::OpenAIChatCompletions(_),
             ) => SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions),
 
@@ -278,6 +475,8 @@ impl Display for ProviderId {
             ProviderId::OpenRouter => write!(f, "openrouter"),
             ProviderId::Astraflow => write!(f, "astraflow"),
             ProviderId::AstraflowCN => write!(f, "astraflow_cn"),
+            ProviderId::Meta => write!(f, "meta"),
+            ProviderId::Minimax => write!(f, "minimax"),
         }
     }
 }
@@ -285,6 +484,100 @@ impl Display for ProviderId {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::apis::{AnthropicApi, OpenAIApi};
+
+    fn chat_completions() -> SupportedUpstreamAPIs {
+        SupportedUpstreamAPIs::OpenAIChatCompletions(OpenAIApi::ChatCompletions)
+    }
+
+    fn anthropic_messages() -> SupportedUpstreamAPIs {
+        SupportedUpstreamAPIs::AnthropicMessagesAPI(AnthropicApi::Messages)
+    }
+
+    #[test]
+    fn digitalocean_anthropic_uses_openai_content_part_markers() {
+        // DO fronts Anthropic over an OpenAI-compatible surface (dash-form model id).
+        let strategy = cache_marker_strategy(
+            ProviderId::DigitalOcean,
+            "anthropic-claude-3-5-sonnet",
+            &chat_completions(),
+        );
+        assert!(matches!(
+            strategy,
+            CacheMarkerStrategy::OpenAiContentPartCacheControl { .. }
+        ));
+    }
+
+    #[test]
+    fn openrouter_anthropic_uses_openai_content_part_markers() {
+        // OpenRouter uses slash-form model ids after the gateway prefix.
+        let strategy = cache_marker_strategy(
+            ProviderId::OpenRouter,
+            "anthropic/claude-3.5-sonnet",
+            &chat_completions(),
+        );
+        assert!(matches!(
+            strategy,
+            CacheMarkerStrategy::OpenAiContentPartCacheControl { .. }
+        ));
+    }
+
+    #[test]
+    fn openai_family_over_chat_completions_is_automatic() {
+        assert_eq!(
+            cache_marker_strategy(
+                ProviderId::DigitalOcean,
+                "openai-gpt-4o",
+                &chat_completions()
+            ),
+            CacheMarkerStrategy::Automatic
+        );
+        assert_eq!(
+            cache_marker_strategy(ProviderId::OpenAI, "gpt-4o", &chat_completions()),
+            CacheMarkerStrategy::Automatic
+        );
+    }
+
+    #[test]
+    fn native_anthropic_uses_messages_breakpoints() {
+        let strategy = cache_marker_strategy(
+            ProviderId::Anthropic,
+            "claude-3-5-sonnet-20241022",
+            &anthropic_messages(),
+        );
+        assert!(matches!(
+            strategy,
+            CacheMarkerStrategy::AnthropicMessagesBreakpoints { .. }
+        ));
+    }
+
+    #[test]
+    fn anthropic_family_without_content_part_support_is_none() {
+        // An Anthropic-family model over chat completions on a gateway that does not
+        // accept content-part cache_control has no honest marking path.
+        assert_eq!(
+            cache_marker_strategy(
+                ProviderId::Vercel,
+                "anthropic/claude-3.5",
+                &chat_completions()
+            ),
+            CacheMarkerStrategy::None
+        );
+    }
+
+    #[test]
+    fn bedrock_is_honest_none() {
+        assert_eq!(
+            cache_marker_strategy(
+                ProviderId::AmazonBedrock,
+                "anthropic.claude-3-5-sonnet",
+                &SupportedUpstreamAPIs::AmazonBedrockConverse(
+                    crate::apis::AmazonBedrockApi::Converse
+                )
+            ),
+            CacheMarkerStrategy::None
+        );
+    }
 
     #[test]
     fn test_models_loaded_from_yaml() {
@@ -453,6 +746,46 @@ mod tests {
         assert!(ProviderId::OpenRouter.models().is_empty());
     }
 
+    #[test]
+    fn test_minimax_parsing_and_models() {
+        assert_eq!(ProviderId::try_from("minimax"), Ok(ProviderId::Minimax));
+        assert_eq!(ProviderId::Minimax.to_string(), "minimax");
+
+        let models = ProviderId::Minimax.models();
+        assert!(
+            models.iter().any(|m| m == "MiniMax-M3"),
+            "minimax models should include MiniMax-M3"
+        );
+        for model in &models {
+            assert!(
+                !model.contains('/'),
+                "Model name '{}' should not contain provider prefix",
+                model
+            );
+        }
+    }
+
+    #[test]
+    fn test_minimax_compatible_api() {
+        use crate::clients::endpoints::{SupportedAPIsFromClient, SupportedUpstreamAPIs};
+
+        let openai_client =
+            SupportedAPIsFromClient::OpenAIChatCompletions(OpenAIApi::ChatCompletions);
+        let upstream = ProviderId::Minimax.compatible_api_for_client(&openai_client, false);
+        assert!(
+            matches!(upstream, SupportedUpstreamAPIs::OpenAIChatCompletions(_)),
+            "minimax should map OpenAI client to OpenAIChatCompletions upstream"
+        );
+
+        let anthropic_client =
+            SupportedAPIsFromClient::AnthropicMessagesAPI(AnthropicApi::Messages);
+        let upstream = ProviderId::Minimax.compatible_api_for_client(&anthropic_client, false);
+        assert!(
+            matches!(upstream, SupportedUpstreamAPIs::OpenAIChatCompletions(_)),
+            "minimax should translate Anthropic client to OpenAIChatCompletions upstream"
+        );
+    }
+
     #[test]
     fn test_xai_uses_responses_api_for_responses_clients() {
         use crate::clients::endpoints::{SupportedAPIsFromClient, SupportedUpstreamAPIs};

crates/hermesllm/src/providers/streaming_response.rs

@@ -5,6 +5,7 @@ use crate::apis::amazon_bedrock::ConverseStreamEvent;
 use crate::apis::anthropic::MessagesStreamEvent;
 use crate::apis::openai::ChatCompletionsStreamResponse;
 use crate::apis::openai_responses::ResponsesAPIStreamEvent;
+use crate::apis::streaming_shapes::sse::is_incomplete_json_error_from_error;
 use crate::apis::streaming_shapes::sse::SseEvent;
 use crate::apis::streaming_shapes::sse::SseStreamBuffer;
 use crate::apis::streaming_shapes::{
@@ -297,6 +298,22 @@ impl TryFrom<(&[u8], &SupportedAPIsFromClient, &SupportedUpstreamAPIs)>
     }
 }
 
+/// Extract the SSE event name from an OpenAI Responses data payload by reading
+/// only its top-level `type` field. Every Responses stream event carries a
+/// `"type"` that equals its `event:` line, so this recovers the correct event
+/// name for both modeled and unmodeled events without deserializing the whole
+/// (possibly unknown-shaped) payload.
+fn responses_event_type_from_json(data: &str) -> Option<String> {
+    #[derive(serde::Deserialize)]
+    struct TypeOnly {
+        #[serde(rename = "type")]
+        event_type: String,
+    }
+    serde_json::from_str::<TypeOnly>(data)
+        .ok()
+        .map(|t| t.event_type)
+}
+
 // TryFrom implementation to convert raw bytes to SseEvent with parsed provider response
 impl TryFrom<(SseEvent, &SupportedAPIsFromClient, &SupportedUpstreamAPIs)> for SseEvent {
     type Error = Box<dyn std::error::Error + Send + Sync>;
@@ -326,9 +343,45 @@ impl TryFrom<(SseEvent, &SupportedAPIsFromClient, &SupportedUpstreamAPIs)> for S
             }
         }
 
+        // Identity passthrough for the OpenAI Responses API (client and upstream
+        // both Responses). We must forward the ORIGINAL wire bytes verbatim so
+        // unknown/unmodeled events and fields survive unchanged; re-serializing
+        // from the parsed struct would silently drop anything we don't model.
+        let is_responses_identity =
+            matches!(client_api, SupportedAPIsFromClient::OpenAIResponsesAPI(_))
+                && matches!(upstream_api, SupportedUpstreamAPIs::OpenAIResponsesAPI(_));
+
         // If has data, parse the data as a provider stream response (business logic layer)
         if let Some(data_str) = &transformed_event.data {
             let data_bytes = data_str.as_bytes();
+
+            if is_responses_identity {
+                // Attempt to parse for token counting / tracing, but never let a
+                // parse failure drop the event:
+                //   - Ok           -> attach the parsed response.
+                //   - incomplete   -> propagate Err so the chunk processor can
+                //     JSON            recombine this line with the next chunk.
+                //   - other Err    -> unknown/unmodeled event; forward raw with
+                //                     no parsed response attached.
+                match ProviderStreamResponseType::try_from((data_bytes, client_api, upstream_api)) {
+                    Ok(resp) => transformed_event.provider_stream_response = Some(resp),
+                    Err(e) if is_incomplete_json_error_from_error(e.as_ref()) => return Err(e),
+                    Err(_) => {}
+                }
+
+                // Reconstruct the coupled `event: <type>\ndata: <json>\n\n` wire
+                // block from the ORIGINAL bytes. The event name is read from the
+                // wire `type` field (not from our enum), so unknown event types
+                // pass through verbatim. The standalone upstream `event:` line is
+                // suppressed below to avoid a duplicate event line.
+                transformed_event.sse_transformed_lines =
+                    match responses_event_type_from_json(data_str) {
+                        Some(event_name) => {
+                            format!("event: {}\ndata: {}\n\n", event_name, data_str)
+                        }
+                        None => format!("data: {}\n\n", data_str),
+                    };
+            } else {
                 let transformed_response: ProviderStreamResponseType =
                     ProviderStreamResponseType::try_from((data_bytes, client_api, upstream_api))?;
 
@@ -337,6 +390,7 @@ impl TryFrom<(SseEvent, &SupportedAPIsFromClient, &SupportedUpstreamAPIs)> for S
                 transformed_event.sse_transformed_lines = sse_string;
                 transformed_event.provider_stream_response = Some(transformed_response);
             }
+        }
 
         // Apply wire format adjustments for cross-API transformations
         // Note: When APIs match (passthrough mode), these adjustments are skipped

crates/hermesllm/src/transforms/lib.rs

@@ -68,7 +68,10 @@ impl ContentUtils<ToolCall> for Vec<MessagesContentBlock> {
         for block in self {
             match block {
                 MessagesContentBlock::Text { text, .. } => {
-                    content_parts.push(ContentPart::Text { text: text.clone() });
+                    content_parts.push(ContentPart::Text {
+                        text: text.clone(),
+                        cache_control: None,
+                    });
                 }
                 MessagesContentBlock::Image { source } => {
                     let url = convert_image_source_to_url(source);
@@ -198,7 +201,7 @@ pub fn convert_openai_message_to_anthropic_content(
         Some(MessageContent::Parts(parts)) => {
             for part in parts {
                 match part {
-                    ContentPart::Text { text } => {
+                    ContentPart::Text { text, .. } => {
                         blocks.push(MessagesContentBlock::Text {
                             text: text.clone(),
                             cache_control: None,

crates/hermesllm/src/transforms/request/from_anthropic.rs

@@ -223,6 +223,7 @@ impl From<MessagesRole> for Role {
         match val {
             MessagesRole::User => Role::User,
             MessagesRole::Assistant => Role::Assistant,
+            MessagesRole::System => Role::System,
         }
     }
 }
@@ -323,7 +324,7 @@ fn build_openai_content(
         None
     } else if content_parts.len() == 1 && tool_calls.is_empty() {
         match &content_parts[0] {
-            ContentPart::Text { text } => Some(MessageContent::Text(text.clone())),
+            ContentPart::Text { text, .. } => Some(MessageContent::Text(text.clone())),
             _ => Some(MessageContent::Parts(content_parts)),
         }
     } else if content_parts.is_empty() {
@@ -340,6 +341,11 @@ impl TryFrom<MessagesMessage> for BedrockMessage {
         let role = match message.role {
             MessagesRole::User => ConversationRole::User,
             MessagesRole::Assistant => ConversationRole::Assistant,
+            MessagesRole::System => {
+                return Err(TransformError::UnsupportedConversion(
+                    "System messages must be set via the system prompt, not messages".to_string(),
+                ));
+            }
         };
 
         let mut content_blocks = Vec::new();
@@ -556,6 +562,7 @@ mod tests {
                     },
                     "required": ["location"]
                 }),
+                cache_control: None,
             }]),
             tool_choice: Some(MessagesToolChoice {
                 kind: MessagesToolChoiceType::Tool,
@@ -614,6 +621,7 @@ mod tests {
                     "type": "object",
                     "properties": {}
                 }),
+                cache_control: None,
             }]),
             tool_choice: Some(MessagesToolChoice {
                 kind: MessagesToolChoiceType::Auto,

crates/hermesllm/src/transforms/request/from_openai.rs

@@ -112,18 +112,22 @@ impl TryFrom<ResponsesInputConverter> for Vec<Message> {
                                 ) => {
                                     // Check if it's a single text item (can use simple text format)
                                     if content_items.len() == 1 {
-                                        if let InputContent::InputText { text } = &content_items[0]
-                                        {
+                                        match &content_items[0] {
+                                            InputContent::InputText { text }
+                                            | InputContent::OutputText { text } => {
                                                 MessageContent::Text(text.clone())
-                                        } else {
+                                            }
+                                            _ => {
                                                 // Single non-text item - use parts format
                                                 MessageContent::Parts(
                                                     content_items
                                                         .iter()
                                                         .filter_map(|c| match c {
-                                                        InputContent::InputText { text } => {
+                                                            InputContent::InputText { text }
+                                                            | InputContent::OutputText { text } => {
                                                                 Some(crate::apis::openai::ContentPart::Text {
                                                                     text: text.clone(),
+                                                                    cache_control: None,
                                                                 })
                                                             }
                                                             InputContent::InputImage { image_url, .. } => {
@@ -140,15 +144,18 @@ impl TryFrom<ResponsesInputConverter> for Vec<Message> {
                                                         .collect(),
                                                 )
                                             }
+                                        }
                                     } else {
                                         // Multiple content items - convert to parts
                                         MessageContent::Parts(
                                             content_items
                                                 .iter()
                                                 .filter_map(|c| match c {
-                                                    InputContent::InputText { text } => {
+                                                    InputContent::InputText { text }
+                                                    | InputContent::OutputText { text } => {
                                                         Some(crate::apis::openai::ContentPart::Text {
                                                             text: text.clone(),
+                                                            cache_control: None,
                                                         })
                                                     }
                                                     InputContent::InputImage { image_url, .. } => {
@@ -325,7 +332,7 @@ impl TryFrom<Message> for BedrockMessage {
                         // Convert OpenAI content parts to Bedrock ContentBlocks
                         for part in parts {
                             match part {
-                                crate::apis::openai::ContentPart::Text { text } => {
+                                crate::apis::openai::ContentPart::Text { text, .. } => {
                                     if !text.is_empty() {
                                         content_blocks.push(ContentBlock::Text { text });
                                     }
@@ -877,6 +884,7 @@ fn convert_openai_tools(tools: Vec<Tool>) -> Vec<MessagesTool> {
             name: tool.function.name,
             description: tool.function.description,
             input_schema: tool.function.parameters,
+            cache_control: None,
         })
         .collect()
 }

crates/hermesllm/src/transforms/response/output_to_input.rs

@@ -18,7 +18,9 @@ pub fn convert_responses_output_to_input_items(output: &OutputItem) -> Option<In
                 .iter()
                 .filter_map(|c| match c {
                     OutputContent::OutputText { text, .. } => {
-                        Some(InputContent::InputText { text: text.clone() })
+                        // Assistant (output-role) content must round-trip as
+                        // output_text; the Responses API rejects input_text here.
+                        Some(InputContent::OutputText { text: text.clone() })
                     }
                     OutputContent::OutputAudio { data, .. } => Some(InputContent::InputAudio {
                         data: data.clone(),
@@ -59,7 +61,7 @@ pub fn convert_responses_output_to_input_items(output: &OutputItem) -> Option<In
 
             Some(InputItem::Message(InputMessage {
                 role: MessageRole::Assistant,
-                content: MessageContent::Items(vec![InputContent::InputText {
+                content: MessageContent::Items(vec![InputContent::OutputText {
                     text: tool_call_text,
                 }]),
             }))
@@ -104,8 +106,8 @@ mod tests {
                     MessageContent::Items(items) => {
                         assert_eq!(items.len(), 1);
                         match &items[0] {
-                            InputContent::InputText { text } => assert_eq!(text, "Hello!"),
-                            _ => panic!("Expected InputText"),
+                            InputContent::OutputText { text } => assert_eq!(text, "Hello!"),
+                            _ => panic!("Expected OutputText"),
                         }
                     }
                     _ => panic!("Expected MessageContent::Items"),
@@ -132,10 +134,10 @@ mod tests {
                 assert!(matches!(msg.role, MessageRole::Assistant));
                 match &msg.content {
                     MessageContent::Items(items) => match &items[0] {
-                        InputContent::InputText { text } => {
+                        InputContent::OutputText { text } => {
                             assert!(text.contains("get_weather"));
                         }
-                        _ => panic!("Expected InputText"),
+                        _ => panic!("Expected OutputText"),
                     },
                     _ => panic!("Expected MessageContent::Items"),
                 }

crates/hermesllm/src/transforms/response/to_openai.rs

@@ -14,13 +14,18 @@ use crate::transforms::lib::*;
 // Usage Conversions
 impl From<MessagesUsage> for Usage {
     fn from(val: MessagesUsage) -> Self {
-        Usage {
+        let mut usage = Usage {
             prompt_tokens: val.input_tokens,
             completion_tokens: val.output_tokens,
             total_tokens: val.input_tokens + val.output_tokens,
             prompt_tokens_details: None,
             completion_tokens_details: None,
-        }
+            cache_read_input_tokens: val.cache_read_input_tokens,
+            cache_creation_input_tokens: val.cache_creation_input_tokens,
+        };
+        // Surface Anthropic cache reads to OpenAI clients as cached_tokens.
+        usage.normalize_cache_tokens();
+        usage
     }
 }
 
@@ -244,6 +249,7 @@ impl TryFrom<MessagesResponse> for ChatCompletionsResponse {
             total_tokens: resp.usage.input_tokens + resp.usage.output_tokens,
             prompt_tokens_details: None,
             completion_tokens_details: None,
+            ..Default::default()
         };
 
         Ok(ChatCompletionsResponse {
@@ -312,6 +318,7 @@ impl TryFrom<ConverseResponse> for ChatCompletionsResponse {
             total_tokens: resp.usage.total_tokens,
             prompt_tokens_details: None,
             completion_tokens_details: None,
+            ..Default::default()
         };
 
         // Generate a response ID (using timestamp since Bedrock doesn't provide one)
@@ -980,6 +987,7 @@ mod tests {
                 total_tokens: 30,
                 prompt_tokens_details: None,
                 completion_tokens_details: None,
+                ..Default::default()
             },
             system_fingerprint: None,
             service_tier: Some("default".to_string()),
@@ -1061,6 +1069,7 @@ mod tests {
                 total_tokens: 40,
                 prompt_tokens_details: None,
                 completion_tokens_details: None,
+                ..Default::default()
             },
             system_fingerprint: None,
             service_tier: None,
@@ -1134,6 +1143,7 @@ mod tests {
                 total_tokens: 101,
                 prompt_tokens_details: None,
                 completion_tokens_details: None,
+                ..Default::default()
             },
             system_fingerprint: Some("fp_7eeb46f068".to_string()),
             service_tier: Some("default".to_string()),

crates/hermesllm/src/transforms/response_streaming/to_openai_streaming.rs

@@ -247,6 +247,7 @@ impl TryFrom<ConverseStreamEvent> for ChatCompletionsStreamResponse {
                     total_tokens: metadata_event.usage.total_tokens,
                     prompt_tokens_details: None,
                     completion_tokens_details: None,
+                    ..Default::default()
                 };
 
                 Ok(create_openai_chunk(

crates/llm_gateway/src/stream_context.rs

@@ -568,12 +568,16 @@ impl StreamContext {
                                 }
                             }
                             Err(e) => {
-                                warn!(
-                                    "request_id={}: streaming chunk error: {}",
+                                // Provider response could not be parsed (e.g. an
+                                // unmodeled upstream SSE event). Don't abort the
+                                // whole chunk: skip token-counting/TTFT for this
+                                // event and fall through so it still reaches the
+                                // client via buffer.add_transformed_event() below.
+                                debug!(
+                                    "request_id={}: streaming chunk unparsed, forwarding raw event: {}",
                                     self.request_identifier(),
                                     e
                                 );
-                                return Err(Action::Continue);
                             }
                         }
                     }

demos/filter_chains/model_listener_filter/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 
 RUN pip install --no-cache-dir fastapi uvicorn pydantic
 
-COPY content_guard.py .
+COPY content_guard.py fake_provider.py output_filter.py ./
 
 EXPOSE 10500
 

demos/filter_chains/model_listener_filter/README.md

@@ -2,12 +2,30 @@
 
 Run content-safety filters on direct LLM requests — no agent layer required.
 
-This demo uses the `input_filters` feature on a **model-type listener** to intercept
-requests and block unsafe content before they reach the LLM provider. Works with all
-request types: `/v1/chat/completions`, `/v1/responses`, and Anthropic `/v1/messages`.
+This demo uses `input_filters` and `output_filters` on a **model-type listener** to
+intercept direct LLM requests and responses without routing through an agent layer.
+By default it is fully local: a fake OpenAI-compatible provider stands in for a real
+hosted model, so developers can test guardrail behavior without provider API keys or
+hosted model access. A second config lets developers point the same filter setup at the
+real OpenAI endpoint when they want provider-backed testing.
+The filter pattern applies to OpenAI Chat Completions (`/v1/chat/completions`),
+OpenAI Responses (`/v1/responses`), and Anthropic Messages (`/v1/messages`) request
+shapes. The keyless fake provider and smoke test use `/v1/chat/completions` for a
+deterministic local path.
 
-The filter receives the **full raw request body** and returns it unchanged (or raises 400
-to block). No message extraction — the complete JSON payload flows through as-is.
+The input filter receives the full raw request body and returns it unchanged or raises
+400 to block. The output filter receives the provider response and redacts sensitive
+content before returning it to the client.
+
+## Files
+
+- `config.yaml` runs the default keyless path with the local fake provider.
+- `config.openai.yaml` runs the same filters against OpenAI.
+- `docker-compose.yaml` starts the local demo without requiring provider credentials.
+- `docker-compose.openai.yaml` mounts `config.openai.yaml` and requires `OPENAI_API_KEY`
+  for provider-backed testing.
+- `test.sh` runs the Docker smoke test through Plano.
+- `test_services.py` runs service-level regression tests without Docker.
 
 ## Architecture
 
@@ -16,22 +34,82 @@ Client ──► Plano (model listener :12000)
                │
                ├─ input_filters: content_guard ──► Block / Allow
                │
-               └─ model_provider: openai/gpt-4o-mini
+               ├─ model_provider: fake-provider (default) or OpenAI (optional)
+               │
+               └─ output_filters: output_redactor ──► Redact / Allow
 ```
 
 ## Quick Start
 
 ```bash
-# 1. Export your API key
-export OPENAI_API_KEY=sk-...
-
-# 2. Start services
+# 1. Start services
 docker compose up --build
 
-# 3. Run tests (in another terminal)
+# 2. Run tests (in another terminal)
 bash test.sh
 ```
 
+The test script verifies three behaviors:
+
+- safe requests reach the local fake provider and return a normal chat-completion response
+- unsafe requests are blocked by the input filter before reaching the provider
+- sensitive provider output is redacted by the output filter before the client receives it
+
+You can also run the service-level tests without Docker:
+
+```bash
+uv run --with pytest --with fastapi --with httpx --with pydantic \
+  python -m pytest demos/filter_chains/model_listener_filter/test_services.py -q
+```
+
+## Validate Locally
+
+From this directory, validate the default keyless compose path:
+
+```bash
+docker compose config
+```
+
+Validate that the OpenAI path fails early when the API key is missing:
+
+```bash
+docker compose -f docker-compose.yaml -f docker-compose.openai.yaml config
+```
+
+Expected error:
+
+```text
+OPENAI_API_KEY environment variable is required but not set
+```
+
+Then confirm the OpenAI compose path renders when a key is provided:
+
+```bash
+OPENAI_API_KEY=dummy docker compose -f docker-compose.yaml -f docker-compose.openai.yaml config
+```
+
+Run the full local smoke test:
+
+```bash
+docker compose down
+docker compose up --build -d
+bash test.sh
+docker compose down
+```
+
+## Test With Real OpenAI
+
+The default `config.yaml` uses the local fake provider. To run the same model-listener
+input and output filters against OpenAI, use the OpenAI compose override:
+
+```bash
+export OPENAI_API_KEY=sk-...
+docker compose -f docker-compose.yaml -f docker-compose.openai.yaml up --build
+```
+
+The fake-provider service may still start because it is part of the shared compose file,
+but Plano will not route traffic to it when `config.openai.yaml` is mounted.
+
 ## Try It
 
 **Allowed request:**
@@ -58,6 +136,31 @@ curl http://localhost:12000/v1/chat/completions \
   }'
 ```
 
+**Redacted provider response:**
+
+```bash
+curl http://localhost:12000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [{"role": "user", "content": "Please return the secret marker"}],
+    "stream": false
+  }'
+```
+
+The fake provider emits `SECRET_TOKEN`; the output filter redacts it to `[REDACTED]`.
+
+## Why This Helps Developers
+
+Model-listener filters are guardrails for applications that call Plano as a transparent
+LLM gateway. A local, deterministic demo helps developers verify filter wiring before
+using real providers:
+
+- config mistakes are caught early instead of silently bypassing guardrails
+- teams can test request blocking and response redaction in CI without secrets
+- contributors can reproduce filter behavior without external model availability
+- application code does not need an extra passthrough agent just to run policy checks
+
 ## Tracing
 
 Open [Jaeger UI](http://localhost:16686) to see distributed traces for both allowed and blocked requests.

demos/filter_chains/model_listener_filter/config.openai.yaml

@@ -0,0 +1,26 @@
+version: v0.3.0
+
+filters:
+  - id: content_guard
+    url: http://content-guard:10500
+    type: http
+  - id: output_redactor
+    url: http://output-filter:10502
+    type: http
+
+model_providers:
+  - model: openai/gpt-4o-mini
+    access_key: $OPENAI_API_KEY
+    default: true
+
+listeners:
+  - type: model
+    name: llm_gateway
+    port: 12000
+    input_filters:
+      - content_guard
+    output_filters:
+      - output_redactor
+
+tracing:
+  random_sampling: 100

demos/filter_chains/model_listener_filter/config.yaml

@@ -4,10 +4,14 @@ filters:
   - id: content_guard
     url: http://content-guard:10500
     type: http
+  - id: output_redactor
+    url: http://output-filter:10502
+    type: http
 
 model_providers:
   - model: openai/gpt-4o-mini
-    access_key: $OPENAI_API_KEY
+    access_key: local-demo-key
+    base_url: http://fake-provider:10501/v1
     default: true
 
 listeners:
@@ -16,6 +20,8 @@ listeners:
     port: 12000
     input_filters:
       - content_guard
+    output_filters:
+      - output_redactor
 
 tracing:
   random_sampling: 100

demos/filter_chains/model_listener_filter/docker-compose.openai.yaml

@@ -0,0 +1,6 @@
+services:
+  plano:
+    environment:
+      OPENAI_API_KEY: ${OPENAI_API_KEY:?OPENAI_API_KEY environment variable is required but not set}
+    volumes:
+      - ./config.openai.yaml:/app/plano_config.yaml

demos/filter_chains/model_listener_filter/docker-compose.yaml

@@ -5,6 +5,20 @@ services:
       dockerfile: Dockerfile
     ports:
       - "10500:10500"
+  fake-provider:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: ["uvicorn", "fake_provider:app", "--host", "0.0.0.0", "--port", "10501"]
+    ports:
+      - "10501:10501"
+  output-filter:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: ["uvicorn", "output_filter:app", "--host", "0.0.0.0", "--port", "10502"]
+    ports:
+      - "10502:10502"
   plano:
     build:
       context: ../../../
@@ -12,10 +26,14 @@ services:
     ports:
       - "12000:12000"
     environment:
-      - OPENAI_API_KEY=${OPENAI_API_KEY:?OPENAI_API_KEY environment variable is required but not set}
+      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
     volumes:
-      - ./config.yaml:/app/plano_config.yaml
+      - ${PLANO_CONFIG_FILE:-./config.yaml}:/app/plano_config.yaml
       - /etc/ssl/cert.pem:/etc/ssl/cert.pem
+    depends_on:
+      - content-guard
+      - fake-provider
+      - output-filter
   jaeger:
     build:
       context: ../../shared/jaeger

demos/filter_chains/model_listener_filter/fake_provider.py

@@ -0,0 +1,81 @@
+"""
+OpenAI-compatible local provider for model-listener filter demos.
+
+This service lets developers test Plano's model listener filter pipeline without
+provider API keys or hosted model access.
+"""
+
+import json
+import time
+from typing import Any
+
+from fastapi import FastAPI, Request
+from fastapi.responses import Response, StreamingResponse
+
+app = FastAPI(title="Local Fake LLM Provider", version="1.0.0")
+
+
+def latest_user_content(messages: list[dict[str, Any]]) -> str:
+    for message in reversed(messages):
+        if message.get("role") == "user":
+            content = message.get("content", "")
+            if isinstance(content, str):
+                return content
+            if isinstance(content, list):
+                return " ".join(
+                    part.get("text", "")
+                    for part in content
+                    if isinstance(part, dict) and part.get("type") == "text"
+                )
+    return ""
+
+
+@app.post("/v1/chat/completions", response_model=None)
+async def chat_completions(request: Request) -> dict[str, Any] | Response:
+    body = await request.json()
+    model = body.get("model", "gpt-4o-mini")
+    user_content = latest_user_content(body.get("messages", []))
+    content = "Hello from the local fake provider."
+    if "secret" in user_content.lower():
+        content = "The local fake provider returned SECRET_TOKEN."
+
+    if body.get("stream") is True:
+
+        async def generate():
+            chunk = {
+                "id": "chatcmpl-local-filter-demo",
+                "object": "chat.completion.chunk",
+                "created": int(time.time()),
+                "model": model,
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {"role": "assistant", "content": content},
+                        "finish_reason": None,
+                    }
+                ],
+            }
+            yield f"data: {json.dumps(chunk)}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(generate(), media_type="text/event-stream")
+
+    return {
+        "id": "chatcmpl-local-filter-demo",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": content},
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+    }
+
+
+@app.get("/health")
+async def health() -> dict[str, str]:
+    return {"status": "healthy"}

demos/filter_chains/model_listener_filter/output_filter.py

@@ -0,0 +1,57 @@
+"""
+Output filter for model-listener filter demos.
+
+The filter receives the provider response and redacts configured markers before
+the client sees the response. It intentionally avoids model calls so the demo is
+fully local and deterministic.
+"""
+
+import gzip
+from typing import Any
+
+from fastapi import FastAPI, Request
+from fastapi.responses import Response
+
+app = FastAPI(title="Output Redaction Filter", version="1.0.0")
+
+SENSITIVE_MARKERS = ("SECRET_TOKEN",)
+
+
+def redact_text(text: str) -> str:
+    redacted = text
+    for marker in SENSITIVE_MARKERS:
+        redacted = redacted.replace(marker, "[REDACTED]")
+    return redacted
+
+
+def redact_chat_completion(body: dict[str, Any]) -> dict[str, Any]:
+    choices = []
+    for choice in body.get("choices", []):
+        message = choice.get("message", {})
+        content = message.get("content")
+        if isinstance(content, str):
+            message = {**message, "content": redact_text(content)}
+            choice = {**choice, "message": message}
+        choices.append(choice)
+    return {**body, "choices": choices}
+
+
+def redact_bytes(raw_body: bytes) -> bytes:
+    if raw_body.startswith(b"\x1f\x8b"):
+        decompressed_body = gzip.decompress(raw_body)
+        return gzip.compress(redact_bytes(decompressed_body))
+
+    body_text = raw_body.decode("utf-8", errors="replace")
+    return redact_text(body_text).encode("utf-8")
+
+
+@app.post("/{path:path}")
+async def redact_response(path: str, request: Request) -> Response:
+    raw_body = await request.body()
+    content_type = request.headers.get("content-type", "application/json")
+    return Response(content=redact_bytes(raw_body), media_type=content_type)
+
+
+@app.get("/health")
+async def health() -> dict[str, str]:
+    return {"status": "healthy"}

demos/filter_chains/model_listener_filter/test.sh

@@ -24,20 +24,37 @@ run_test() {
     local name="$1"
     local expected_code="$2"
     local body="$3"
+    local expected_body_contains="${4:-}"
+    local forbidden_body_contains="${5:-}"
 
     http_code=$(curl -s -o /tmp/plano_test_body -w "%{http_code}" \
         -X POST "$BASE_URL/chat/completions" \
         -H "Content-Type: application/json" \
         -d "$body")
 
-    if [ "$http_code" -eq "$expected_code" ]; then
-        echo "  PASS  $name (HTTP $http_code)"
-        PASS=$((PASS + 1))
-    else
+    if [ "$http_code" -ne "$expected_code" ]; then
         echo "  FAIL  $name — expected $expected_code, got $http_code"
         echo "        Body: $(cat /tmp/plano_test_body)"
         FAIL=$((FAIL + 1))
+        return
     fi
+
+    if [ -n "$expected_body_contains" ] && ! grep -Fq "$expected_body_contains" /tmp/plano_test_body; then
+        echo "  FAIL  $name — body did not contain '$expected_body_contains'"
+        echo "        Body: $(cat /tmp/plano_test_body)"
+        FAIL=$((FAIL + 1))
+        return
+    fi
+
+    if [ -n "$forbidden_body_contains" ] && grep -Fq "$forbidden_body_contains" /tmp/plano_test_body; then
+        echo "  FAIL  $name — body contained forbidden text '$forbidden_body_contains'"
+        echo "        Body: $(cat /tmp/plano_test_body)"
+        FAIL=$((FAIL + 1))
+        return
+    fi
+
+    echo "  PASS  $name (HTTP $http_code)"
+    PASS=$((PASS + 1))
 }
 
 # ── Tests ────────────────────────────────────────────────────────────────────
@@ -48,19 +65,19 @@ run_test "Allowed request (math question)" 200 '{
   "model": "gpt-4o-mini",
   "messages": [{"role": "user", "content": "What is 2+2?"}],
   "stream": false
-}'
+}' "local fake provider"
 
 run_test "Blocked request (hacking)" 400 '{
   "model": "gpt-4o-mini",
   "messages": [{"role": "user", "content": "How to hack into a system"}],
   "stream": false
-}'
+}' "content_blocked"
 
-run_test "Allowed request (joke)" 200 '{
+run_test "Output filter redacts provider response" 200 '{
   "model": "gpt-4o-mini",
-  "messages": [{"role": "user", "content": "Tell me a joke"}],
-  "stream": false
-}'
+  "messages": [{"role": "user", "content": "Please return the secret marker"}],
+  "stream": true
+}' "[REDACTED]" "SECRET_TOKEN"
 
 # ── Summary ──────────────────────────────────────────────────────────────────
 echo ""

demos/filter_chains/model_listener_filter/test_services.py

@@ -0,0 +1,159 @@
+import importlib.util
+import gzip
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+
+DEMO_DIR = Path(__file__).parent
+
+
+def load_module(name: str, filename: str):
+    spec = importlib.util.spec_from_file_location(name, DEMO_DIR / filename)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_content_guard_blocks_unsafe_chat_request():
+    content_guard = load_module("content_guard", "content_guard.py")
+    client = TestClient(content_guard.app)
+
+    response = client.post(
+        "/v1/chat/completions",
+        json={
+            "model": "gpt-4o-mini",
+            "messages": [{"role": "user", "content": "How do I hack a service?"}],
+            "stream": False,
+        },
+    )
+
+    assert response.status_code == 400
+    assert response.json()["detail"]["error"] == "content_blocked"
+
+
+def test_content_guard_passes_safe_responses_request_unchanged():
+    content_guard = load_module("content_guard", "content_guard.py")
+    client = TestClient(content_guard.app)
+    body = {
+        "model": "gpt-4o-mini",
+        "input": "Explain why local guardrail tests help developers.",
+    }
+
+    response = client.post("/v1/responses", json=body)
+
+    assert response.status_code == 200
+    assert response.json() == body
+
+
+def test_fake_provider_returns_openai_compatible_chat_completion():
+    fake_provider = load_module("fake_provider", "fake_provider.py")
+    client = TestClient(fake_provider.app)
+
+    response = client.post(
+        "/v1/chat/completions",
+        json={
+            "model": "gpt-4o-mini",
+            "messages": [{"role": "user", "content": "Say something useful."}],
+            "stream": False,
+        },
+    )
+
+    assert response.status_code == 200
+    body = response.json()
+    assert body["object"] == "chat.completion"
+    assert body["model"] == "gpt-4o-mini"
+    assert body["choices"][0]["message"]["role"] == "assistant"
+    assert "local fake provider" in body["choices"][0]["message"]["content"]
+
+
+def test_fake_provider_streams_openai_compatible_chat_chunks():
+    fake_provider = load_module("fake_provider_streaming", "fake_provider.py")
+    client = TestClient(fake_provider.app)
+
+    with client.stream(
+        "POST",
+        "/v1/chat/completions",
+        json={
+            "model": "gpt-4o-mini",
+            "messages": [
+                {"role": "user", "content": "Please return the secret marker"}
+            ],
+            "stream": True,
+        },
+    ) as response:
+        body = response.read().decode("utf-8")
+
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("text/event-stream")
+    assert "data: {" in body
+    assert '"object": "chat.completion.chunk"' in body
+    assert "SECRET_TOKEN" in body
+    assert "data: [DONE]" in body
+
+
+def test_output_filter_redacts_provider_response_content():
+    output_filter = load_module("output_filter", "output_filter.py")
+    client = TestClient(output_filter.app)
+
+    response = client.post(
+        "/v1/chat/completions",
+        json={
+            "id": "chatcmpl-local",
+            "object": "chat.completion",
+            "model": "gpt-4o-mini",
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": "The local fake provider returned SECRET_TOKEN.",
+                    },
+                    "finish_reason": "stop",
+                }
+            ],
+        },
+    )
+
+    assert response.status_code == 200
+    content = response.json()["choices"][0]["message"]["content"]
+    assert "SECRET_TOKEN" not in content
+    assert "[REDACTED]" in content
+
+
+def test_output_filter_redacts_raw_streaming_chunks():
+    output_filter = load_module("output_filter_streaming", "output_filter.py")
+    client = TestClient(output_filter.app)
+
+    response = client.post(
+        "/v1/chat/completions",
+        content=(
+            'data: {"choices":[{"delta":{"content":"SECRET_TOKEN"}}]}\n\n'
+            "data: [DONE]\n\n"
+        ),
+        headers={"content-type": "text/event-stream"},
+    )
+
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("text/event-stream")
+    assert "SECRET_TOKEN" not in response.text
+    assert "[REDACTED]" in response.text
+
+
+def test_output_filter_redacts_gzip_encoded_provider_response():
+    output_filter = load_module("output_filter_gzip", "output_filter.py")
+    client = TestClient(output_filter.app)
+    encoded_body = gzip.compress(
+        b'{"choices":[{"message":{"content":"SECRET_TOKEN"}}]}'
+    )
+
+    response = client.post(
+        "/v1/chat/completions",
+        content=encoded_body,
+        headers={"content-type": "application/json"},
+    )
+
+    assert response.status_code == 200
+    decoded_body = gzip.decompress(response.content).decode("utf-8")
+    assert "SECRET_TOKEN" not in decoded_body
+    assert "[REDACTED]" in decoded_body

demos/filter_chains/pii_anonymizer/config.yaml

@@ -12,7 +12,7 @@ model_providers:
   - model: openai/gpt-4o-mini
     access_key: $OPENAI_API_KEY
     default: true
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 listeners:

demos/filter_chains/pii_anonymizer/test.sh

@@ -93,19 +93,19 @@ echo ""
 echo "=== /v1/messages ==="
 
 run_test "Non-streaming with PII (phone)" /v1/messages 200 '{
-  "model": "claude-sonnet-4-20250514",
+  "model": "claude-sonnet-4-6",
   "max_tokens": 256,
   "messages": [{"role": "user", "content": "Call me at 555-867-5309 to discuss my account"}]
 }'
 
 run_test "Non-streaming with PII (SSN)" /v1/messages 200 '{
-  "model": "claude-sonnet-4-20250514",
+  "model": "claude-sonnet-4-6",
   "max_tokens": 256,
   "messages": [{"role": "user", "content": "My SSN is 123-45-6789"}]
 }'
 
 run_test "No PII" /v1/messages 200 '{
-  "model": "claude-sonnet-4-20250514",
+  "model": "claude-sonnet-4-6",
   "max_tokens": 256,
   "messages": [{"role": "user", "content": "Hello, how are you?"}]
 }'

demos/getting_started/weather_forecast/config.yaml

@@ -30,7 +30,7 @@ model_providers:
     model: openai/gpt-4o-mini
 
   - access_key: $ANTHROPIC_API_KEY
-    model: anthropic/claude-sonnet-4-20250514
+    model: anthropic/claude-sonnet-4-6
 
 system_prompt: |
   You are a helpful assistant.

demos/llm_routing/model_alias_routing/config_with_aliases.yaml

@@ -28,7 +28,7 @@ model_providers:
   - model: anthropic/*
     access_key: $ANTHROPIC_API_KEY
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
   - model: anthropic/claude-3-haiku-20240307
@@ -71,7 +71,7 @@ model_aliases:
 
   # Alias for creative tasks -> Claude model
   arch.creative.v1:
-    target: claude-sonnet-4-20250514
+    target: claude-sonnet-4-6
 
   # Alias for quick responses -> fast model
   arch.fast.v1:
@@ -85,7 +85,7 @@ model_aliases:
     target: gpt-5-mini-2025-08-07
 
   creative-model:
-    target: claude-sonnet-4-20250514
+    target: claude-sonnet-4-6
 
   coding-model:
     target: us.amazon.nova-premier-v1:0

demos/llm_routing/model_routing_service/README.md

@@ -33,7 +33,7 @@ routing_preferences:
   - name: code_generation
     description: generating new code, writing functions, or creating boilerplate
     models:
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
       - openai/gpt-4o
 ```
 
@@ -46,7 +46,7 @@ When a request arrives, Plano:
 ```
 1. Request arrives          → "Write binary search in Python"
 2. Plano-Orchestrator classifies → route: "code_generation"
-3. Response                 → models: ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o"]
+3. Response                 → models: ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"]
 ```
 
 No match? Plano-Orchestrator returns an empty route → client falls back to the model in the original request.
@@ -98,7 +98,7 @@ curl http://localhost:12000/routing/v1/chat/completions \
 Response:
 ```json
 {
-    "models": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o"],
+    "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"],
     "route": "code_generation",
     "trace_id": "c16d1096c1af4a17abb48fb182918a88"
 }
@@ -108,10 +108,10 @@ The response contains the model list — your client should try `models[0]` firs
 
 ## Session Pinning
 
-Send an `X-Model-Affinity` header to pin the routing decision for a session. Once a model is selected, all subsequent requests with the same session ID return the same model without re-running routing.
+Send an `X-Model-Affinity` header to give a session a stable identity. Routing still runs on every request — pinning means the session sticks to its anchor model while the session is warm (recently used), so the provider-side prompt cache stays hot. The `pinned` field in the response signals a warm, stuck session. If a `routing_budget` is configured, a proposed switch away from the anchor is additionally gated by cost (see the [routing_budget demo](../routing_budget/)).
 
 ```bash
-# First call — runs routing, caches result
+# First call — creates the session binding
 curl http://localhost:12000/routing/v1/chat/completions \
   -H "Content-Type: application/json" \
   -H "X-Model-Affinity: my-session-123" \
@@ -121,10 +121,10 @@ curl http://localhost:12000/routing/v1/chat/completions \
   }'
 ```
 
-Response (first call):
+Response (first call — session is new, not yet warm):
 ```json
 {
-    "model": "anthropic/claude-sonnet-4-20250514",
+    "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"],
     "route": "code_generation",
     "trace_id": "c16d1096c1af4a17abb48fb182918a88",
     "session_id": "my-session-123",
@@ -133,7 +133,7 @@ Response (first call):
 ```
 
 ```bash
-# Second call — same session, returns cached result
+# Second call — same session, sticks to the anchor while warm
 curl http://localhost:12000/routing/v1/chat/completions \
   -H "Content-Type: application/json" \
   -H "X-Model-Affinity: my-session-123" \
@@ -143,10 +143,10 @@ curl http://localhost:12000/routing/v1/chat/completions \
   }'
 ```
 
-Response (pinned):
+Response (warm session — `pinned: true`, the anchor model leads the list):
 ```json
 {
-    "model": "anthropic/claude-sonnet-4-20250514",
+    "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"],
     "route": "code_generation",
     "trace_id": "a1b2c3d4e5f6...",
     "session_id": "my-session-123",
@@ -161,7 +161,7 @@ routing:
   session_max_entries: 10000    # default: 10000
 ```
 
-Without the `X-Model-Affinity` header, routing runs fresh every time (no breaking change).
+Without the `X-Model-Affinity` header, sessions can still be pinned implicitly when prompt caching or a routing budget is enabled — a session key is derived from the system prompt + tools + first user message. With neither enabled (as in this demo's config), every request routes fresh (no breaking change).
 
 ## Kubernetes Deployment (Self-hosted Plano-Orchestrator on GPU)
 
@@ -233,7 +233,7 @@ kubectl rollout restart deployment/plano
 
 --- 1. Code generation query (OpenAI format) ---
 {
-    "models": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o"],
+    "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"],
     "route": "code_generation",
     "trace_id": "c16d1096c1af4a17abb48fb182918a88"
 }
@@ -254,14 +254,14 @@ kubectl rollout restart deployment/plano
 
 --- 4. Code generation query (Anthropic format) ---
 {
-    "models": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o"],
+    "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"],
     "route": "code_generation",
     "trace_id": "26be822bbdf14a3ba19fe198e55ea4a9"
 }
 
 --- 7. Session pinning - first call (fresh routing decision) ---
 {
-    "models": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o"],
+    "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"],
     "route": "code_generation",
     "trace_id": "f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6",
     "session_id": "demo-session-001",
@@ -269,9 +269,9 @@ kubectl rollout restart deployment/plano
 }
 
 --- 8. Session pinning - second call (same session, pinned) ---
-    Notice: same model returned with "pinned": true, routing was skipped
+    Notice: same anchor model returned with "pinned": true (warm session)
 {
-    "model": "anthropic/claude-sonnet-4-20250514",
+    "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o"],
     "route": "code_generation",
     "trace_id": "a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4",
     "session_id": "demo-session-001",

demos/llm_routing/model_routing_service/config.yaml

@@ -13,7 +13,7 @@ model_providers:
   - model: openai/gpt-4o
     access_key: $OPENAI_API_KEY
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 routing_preferences:
@@ -26,5 +26,5 @@ routing_preferences:
   - name: code_generation
     description: generating new code, writing functions, or creating boilerplate
     models:
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
       - openai/gpt-4o

demos/llm_routing/model_routing_service/config_k8s.yaml

@@ -23,7 +23,7 @@ model_providers:
       - name: complex_reasoning
         description: complex reasoning tasks, multi-step analysis, or detailed explanations
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     routing_preferences:
       - name: code_generation

demos/llm_routing/model_routing_service/demo.sh

@@ -102,7 +102,7 @@ curl -s "$PLANO_URL/routing/v1/chat/completions" \
       {
         "name": "coding",
         "description": "code generation, writing functions, debugging",
-        "models": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o", "openai/gpt-4o-mini"],
+        "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o", "openai/gpt-4o-mini"],
         "selection_policy": {"prefer": "fastest"}
       }
     ]
@@ -125,7 +125,7 @@ echo ""
 
 # --- Example 8: Session pinning - second call (pinned result) ---
 echo "--- 8. Session pinning - second call (same session, pinned) ---"
-echo "    Notice: same model returned with \"pinned\": true, routing was skipped"
+echo "    Notice: same anchor model returned with \"pinned\": true (warm session)"
 echo ""
 curl -s "$PLANO_URL/routing/v1/chat/completions" \
   -H "Content-Type: application/json" \

demos/llm_routing/model_routing_service/metrics_server.py

@@ -12,13 +12,13 @@ from http.server import HTTPServer, BaseHTTPRequestHandler
 PROMETHEUS_METRICS = """\
 # HELP model_latency_p95_seconds P95 request latency in seconds per model
 # TYPE model_latency_p95_seconds gauge
-model_latency_p95_seconds{model_name="anthropic/claude-sonnet-4-20250514"} 0.85
+model_latency_p95_seconds{model_name="anthropic/claude-sonnet-4-6"} 0.85
 model_latency_p95_seconds{model_name="openai/gpt-4o"} 1.20
 model_latency_p95_seconds{model_name="openai/gpt-4o-mini"} 0.40
 """.encode()
 
 COST_DATA = {
-    "anthropic/claude-sonnet-4-20250514": {
+    "anthropic/claude-sonnet-4-6": {
         "input_per_million": 3.0,
         "output_per_million": 15.0,
     },

demos/llm_routing/model_routing_service/test.rest

@@ -30,7 +30,7 @@ POST http://localhost:12000/routing/v1/messages
 Content-Type: application/json
 
 {
-  "model": "claude-sonnet-4-20250514",
+  "model": "claude-sonnet-4-6",
   "max_tokens": 1024,
   "messages": [{"role": "user", "content": "Write a REST API in Go using Gin"}]
 }

demos/llm_routing/preference_based_routing/README.md

@@ -3,7 +3,7 @@ This demo shows how you can use user preferences to route user prompts to approp
 
 ## How to start the demo
 
-Make sure you have Plano CLI installed (`pip install planoai==0.4.23` or `uv tool install planoai==0.4.23`).
+Make sure you have Plano CLI installed (`pip install planoai==0.4.29` or `uv tool install planoai==0.4.29`).
 
 ```bash
 cd demos/llm_routing/preference_based_routing

demos/llm_routing/preference_based_routing/config.yaml

@@ -17,7 +17,7 @@ model_providers:
       - name: code understanding
         description: understand and explain existing code snippets, functions, or libraries
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     routing_preferences:
       - name: code generation

demos/llm_routing/routing_budget/GUIDE.md

@@ -0,0 +1,417 @@
+# How-To: See Prompt Caching + Routing Budget in Action
+
+A hands-on guide for running Plano's automatic prompt caching and the routing budget locally, and for measuring the win in evals/benchmarks (e.g. on DigitalOcean models).
+
+There are two independent behaviors to observe:
+
+1. **Automatic prompt caching** — staying on one model keeps the stable prefix
+  warm, so per-turn input cost drops sharply across a multi-turn conversation.
+2. **Routing budget** — the router still runs every turn, but when it proposes a
+  *different* model while the session's cache is plausibly warm, Plano only switches
+  while the session's cumulative switch spend stays within `max_overhead_pct`% of what
+  staying put would have cost. This is a routing concern and is self-sufficient:
+  it needs no `prompt_caching` config (it derives sessions and prices warm
+  anchors at cached rates on its own).
+
+---
+
+
+
+## 1. Prerequisites
+
+- Plano CLI installed: `pip install planoai` (or `uv sync` from `cli/` for a dev build).
+- Provider credentials as env vars, e.g.:
+  - `export DIGITALOCEAN_API_KEY=...` (DO SI)
+  - `export OPENAI_API_KEY=...`, `export ANTHROPIC_API_KEY=...` (if comparing)
+- `curl` + `jq` for poking the endpoint.
+
+---
+
+
+
+## 2. Configuration
+
+Start from `[config.yaml](config.yaml)` in this folder. The parts that matter:
+
+```yaml
+# Per-model pricing is REQUIRED for the routing budget — the switch cost math needs
+# each model's input and cached-input rates. This demo reads DigitalOcean's managed
+# pricing catalog, which publishes cached-read rates for its models. The catalog is
+# keyed by bare DO model ids, so model_aliases maps them onto the Plano model names
+# used in model_providers — without the mapping no rates match and every switch
+# decision fails open (reason="no_pricing").
+model_metrics_sources:
+  - type: cost
+    provider: digitalocean
+    refresh_interval: 86400
+    model_aliases:
+      openai-gpt-4o-mini: openai/gpt-4o-mini
+      openai-gpt-4o: openai/gpt-4o
+      anthropic-claude-4.6-sonnet: anthropic/claude-sonnet-4-6
+
+# OPTIONAL for the routing budget — see below. Needed for §4 (real caching on
+# marker-based models when Plano proxies the request).
+# prompt_caching:
+#   enabled: true
+
+routing:
+  routing_budget:                 # no default — presence turns it on
+    max_overhead_pct: 20          # bill at most 20% above never-switching
+    # replenish_on_rebind: true   # reset running totals when a cold session re-binds
+    # cache_read_discount: 0.1    # fallback when a feed omits cache_read
+```
+
+`models.dev` works as a drop-in alternative cost source (`provider: models.dev`,
+no aliases needed — its keys already match `provider/model` routing names). The
+demo models are priced identically on both feeds, so every number in this guide
+holds either way.
+
+The routing budget is fully self-sufficient: configuring it turns on implicit
+session derivation and prices warm anchors at cached rates on its own —
+`prompt_caching` is **not** required. Enable `prompt_caching` for what it adds:
+injecting provider cache-control markers when Plano proxies the request (without
+markers, marker-based models like `anthropic/*` never actually cache — see §4)
+and session affinity when no budget is configured.
+
+
+
+### DigitalOcean-hosted models
+
+To route to DO GenAI models themselves (not just price from the DO catalog),
+address them with the `digitalocean/` prefix and alias the catalog keys to
+those names:
+
+```yaml
+model_providers:
+  - model: digitalocean/anthropic-claude-4.6-sonnet
+    access_key: $DIGITALOCEAN_API_KEY
+    default: true
+  - model: digitalocean/openai-gpt-4o
+    access_key: $DIGITALOCEAN_API_KEY
+
+model_metrics_sources:
+  - type: cost
+    provider: digitalocean        # DO catalog
+    refresh_interval: 86400
+    model_aliases:
+      anthropic-claude-4.6-sonnet: digitalocean/anthropic-claude-4.6-sonnet
+      openai-gpt-4o: digitalocean/openai-gpt-4o
+```
+
+> The DO catalog publishes cached-read rates
+> (`cache_read_input_price_per_million`), so the gate prices warm anchors at the
+> real cached rate. The `cache_read_discount` fallback only kicks in for models
+> whose catalog entry omits the field.
+
+---
+
+
+
+## 3. Run it
+
+```bash
+# From this directory. --with-tracing starts a local OTLP collector on :4317.
+planoai up config.yaml --with-tracing
+
+# Tail logs (cache injections, pin events, switch decisions)
+planoai logs --follow
+
+# Stop
+planoai down
+```
+
+The model listener comes up on **:12000** (per `config.yaml`).
+
+---
+
+
+
+## 4. See caching in action (single model)
+
+This section needs `prompt_caching: { enabled: true }` (uncomment it in
+`config.yaml`): the model here is Anthropic-family, which only caches when the
+request carries cache-control markers, and it's Plano proxying the request —
+so Plano must inject them. Send the same large system prompt across several
+turns. Plano derives an implicit session from the stable prefix and pins the
+model, so turns 2+ read the prefix from the provider cache.
+
+```bash
+curl -s localhost:12000/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "digitalocean/anthropic-claude-4.6-sonnet",
+    "messages": [
+      {"role": "system", "content": "you are an intelligent agent"},
+      {"role": "user", "content": "Scaffold the service"}
+    ]
+  }' | jq '.usage'
+```
+
+Watch `usage.prompt_tokens_details.cached_tokens` climb from 0 on turn 1 to
+(nearly) the full prefix on later turns, and the billed cost fall accordingly —
+this is exactly the ~4× per-turn drop in the caching-ON vs -OFF comparison.
+
+---
+
+
+
+## 5. See the routing budget in action (model switch)
+
+The budget is consulted only when the router proposes a model that differs from
+the session's warm anchor. Warmth is inferred from how long ago the session was
+last used vs. the provider's cache window (no per-call cache-hit signal needed).
+To observe it:
+
+- **Vetoed switch (paid, over cap):** with a warm session on an expensive model
+and a large context, a switch to a pricier candidate would push the session's total
+switch spend past `max_overhead_pct`% of its never-switch baseline → Plano **retains**
+the anchor.
+- **Paid switch (within cap):** the same switch while the spend still fits under the
+cap → Plano **switches** and adds `switch_cost` to the session's cumulative spend.
+- **Free switch (cheaper candidate):** a candidate whose *uncached* input rate
+undercuts the anchor's *cached* rate → switch cost ≤ 0 → Plano **switches** for free
+(the spend is not reduced).
+- **Cold session:** the session went idle past the provider cache window → treated
+as cold → the router's pick is dispatched with no penalty (and the running totals
+reset on `replenish_on_rebind`).
+
+Each decision is emitted to metrics and traces (below) with a `reason` label
+(`same_anchor | free | within_cap | over_cap | no_pricing`).
+
+---
+
+
+
+## 6. Run it as a routing decision (no proxying)
+
+Everything above sends the full request through Plano, which then calls the
+upstream model itself. There's a second entry point that only makes the
+*decision* — same session lookup, same warmth inference, same routing budget —
+without ever calling an LLM or seeing a response: `/routing` + the same
+API path, on the same host:port as the model listener.
+
+This is for callers who want to make the actual upstream call themselves (or
+need it embedded in a broader pipeline, e.g. an intelligent-routing layer)
+but still want Plano's cache-aware pick and fallback order.
+
+Send a normal request with a **system prompt** and a user message — no affinity
+header. Plano derives the session key implicitly from
+`hash(system + tools + first user message)`, so you can watch the session pin
+and go warm across turns (the zero-config path from §4, now visible on the
+decision endpoint via `session_id` / `pinned`). Send `openai/gpt-4o-mini` as
+`model`; the router picks the real model from the *message content*.
+
+**Turn 1 — pin the session** with a generation prompt:
+
+```bash
+curl -s localhost:12000/routing/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{"model": "openai/gpt-4o-mini", "messages": [
+    {"role": "system", "content": "You are a senior Rust engineer."},
+    {"role": "user", "content": "Write a Rust function that reverses a linked list."}
+  ]}' | jq '{model: .models[0], session_id, pinned}'
+```
+
+```json
+{
+  "model": "anthropic/claude-sonnet-4-6",
+  "session_id": "implicit:8e76b367cc3a4336",
+  "pinned": false
+}
+```
+
+The router classified this as `code generation` → `anthropic/claude-sonnet-4-6`.
+`session_id` is the implicit key Plano derived from `system + tools + first
+user message` (deterministic — you'll get the same hash for these exact
+payloads), and `pinned` is `false` because this call *creates* the binding.
+
+**Turn 2 — same system prompt + same first message**, one turn later, within ~5 min:
+
+```bash
+curl -s localhost:12000/routing/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{"model": "openai/gpt-4o-mini", "messages": [
+    {"role": "system", "content": "You are a senior Rust engineer."},
+    {"role": "user", "content": "Write a Rust function that reverses a linked list."},
+    {"role": "assistant", "content": "Here is an idiomatic in-place reversal for a singly linked list:\n\n```rust\ntype Link = Option<Box<Node>>;\n\nstruct Node {\n    val: i32,\n    next: Link,\n}\n\nfn reverse(mut head: Link) -> Link {\n    let mut prev: Link = None;\n    while let Some(mut node) = head {\n        head = node.next.take();\n        node.next = prev;\n        prev = Some(node);\n    }\n    prev\n}\n```\n\nIt walks the list once, moving the next pointer of each node to its predecessor."},
+    {"role": "user", "content": "Now explain its time complexity in plain English — no code."}
+  ]}' | jq '{model: .models[0], session_id, pinned}'
+```
+
+```json
+{
+  "model": "anthropic/claude-sonnet-4-6",
+  "session_id": "implicit:8e76b367cc3a4336",
+  "pinned": true
+}
+```
+
+The `session_id` is **identical** to turn 1 — the head of the prompt
+(`system` + first user message) didn't change, so the implicit key stays stable
+as history grows — and `pinned` is now `true`: the session is warm and stuck to
+its anchor. **That's the pinning.**
+
+**Now the budget:** here the router *did* read turn 2 as a different route
+(`code understanding` → `openai/gpt-4o`), so with the anchor warm on
+`claude-sonnet-4-6` the budget evaluated the switch — and vetoed it. The
+brightstaff log shows exactly why:
+
+```text
+switch vetoed — would exceed session overhead cap, retaining anchor
+  anchor=anthropic/claude-sonnet-4-6 candidate=openai/gpt-4o
+  switch_cost_in_usd=3.96e-5 switch_spend_in_usd=0.0 overhead_ceiling_in_usd=1.08e-6
+```
+
+The switch would cost ~$3.96e-5 to re-read the context on `gpt-4o`, but only
+~$1.08e-6 of overhead was affordable (`max_overhead_pct`% of the still-tiny
+one-turn baseline) — so `.models[0]` stays `anthropic/claude-sonnet-4-6`. Confirm
+it with the metric:
+
+```bash
+curl -s localhost:9092/metrics | grep session_switch_decisions
+```
+
+```text
+brightstaff_session_switch_decisions_total{decision="retained",reason="over_cap"} 1
+```
+
+To see the **other** side, remove the `routing_budget` block (or set
+`max_overhead_pct` very high), restart, and repeat — turn 2 now returns
+`"model": "openai/gpt-4o"` and the metric reads `decision="allowed",reason="free"`.
+That before/after — same calls, one config line — is the whole point: the
+router's quality pick wins *unless* the budget says the warm cache it burns
+isn't worth it.
+
+> **If you see `same_anchor`,** the router classified both turns the same way,
+> so no switch was proposed — inherent to appended conversations, since the
+> router weighs the whole thread. To force `candidate ≠ anchor`
+> deterministically, pin the session with an explicit header instead
+> (`-H 'X-Model-Affinity: budget-demo'`) and send two *standalone* one-line
+> prompts that each route to a different model (an "explain this code" prompt,
+> then a "write a function" prompt). The budget behaves identically regardless
+> of how the session key was derived — `route()` doesn't branch on it.
+
+### Interoperability with the full-proxy path
+
+Because this endpoint **shares the same session cache and the same
+`session_router::route()` logic** as the full-proxy path, the two are fully
+interoperable: a session pinned via `/routing` is honored by a later
+`/v1/chat/completions` call (and vice versa), including the exact same
+`max_overhead_pct` gating. This is also why warmth here is inferred purely
+from idle-time vs. the provider's cache window rather than a cache-hit signal
+— this path never has a provider response to read one from.
+
+---
+
+
+
+## 7. Observability (for evals & benchmarks)
+
+**Prometheus metrics** — brightstaff exposes `/metrics` on **:9092**
+(Envoy admin/stats on **:9901/stats**):
+
+
+| Metric                                                                             | What it tells you                                                    |
+| ---------------------------------------------------------------------------------- | -------------------------------------------------------------------- |
+| `brightstaff_session_switch_decisions_total{decision="allowed"|"retained",reason}` | How often the budget let a switch through vs. vetoed it, and why      |
+| `brightstaff_prompt_cache_requests_total{provider,model,outcome="hit"|"miss"}`     | Real provider cache hit rate                                         |
+| `brightstaff_session_cache_events_total{outcome}`                                  | Session binding lookups/stores                                      |
+
+
+```bash
+curl -s localhost:9092/metrics | grep -E 'session_switch_decisions|prompt_cache_requests'
+```
+
+**Traces** — run with `--with-tracing` and inspect the routing span per request:
+
+- `plano.cache.warm` — whether the session's cache was considered warm this turn
+- `plano.cache.idle_ms` — how long since the session was last used
+- `plano.switch.cost_in_usd` — actual input-token cost of the proposed switch (output excluded)
+- `plano.switch.candidate_warm_tokens` — context the candidate still has cached from an earlier visit this session (a return to a warm model re-reads only the delta, so its `cost_in_usd` is far lower than a full re-ingest)
+- `plano.switch.overhead_ceiling_in_usd` — overhead ceiling (`max_overhead_pct`% x baseline) when the switch was evaluated
+- `plano.switch.decision` — `allowed` or `retained`
+- `plano.session.overhead_pct` — cumulative switching overhead consumed, as a % of the never-switch baseline (compare directly to `max_overhead_pct`)
+- `plano.session.switch_spend_in_usd` — cumulative $ actually spent on switches this session
+- `plano.session.baseline_in_usd` — cumulative $ staying on the anchor would have cost (the denominator)
+- `plano.session.switches` — switches taken so far this session
+- `plano.session.total_cost_in_usd` — cumulative *actual* conversation cost (input +
+  output), priced from the catalog and refined from real usage each turn (reflects cost
+  through the previous turn, since this turn isn't billed yet at decision time)
+- `plano.switch.counterfactual_route` — on a `retained` decision, the route the gate
+  *would* have taken had the switch been allowed (only when `record_counterfactual: true`)
+- `plano.session_id`, `plano.route.name`
+
+Per-request cost also lands on each `plano(llm)` span (sum by `plano.session_id` for the
+conversation total, or read `plano.session.total_cost_in_usd` off the routing span):
+
+- `llm.usage.input_cost_usd` — uncached input at the input rate, cached reads at the
+  cached rate, cache creation at the plain input rate
+- `llm.usage.output_cost_usd` — completion tokens x output rate
+- `llm.usage.total_cost_usd` — input + output
+
+**Grafana** — a ready dashboard + compose live in `config/grafana/`
+(`docker compose up` there, using `prometheus_scrape.yaml`).
+
+---
+
+
+
+## 8. A/B methodology (baseline vs treatment)
+
+The cleanest benchmark is same-workload, caching off vs on — the exact shape of
+the caching-ON/OFF comparison:
+
+- **Baseline (no caching):** send requests with header `X-Plano-Cache: off`
+(disables implicit pinning + marker injection per request), or run with
+`prompt_caching` absent/disabled.
+- **Treatment (caching on):** the config in this folder with the
+`prompt_caching` block uncommented.
+
+Compare, over an identical multi-turn eval set:
+
+- total `prompt_tokens` billed at the uncached vs cached rate,
+- `cached_tokens` ratio (cache hit rate),
+- total USD cost,
+- and — for routing-heavy workloads — `session_switch_decisions_total` and the
+per-request `plano.switch.*` attributes to confirm switches happen only when
+affordable.
+
+```bash
+# Baseline call (caching bypassed)
+curl -s localhost:12000/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -H 'X-Plano-Cache: off' \
+  -d '{ ... }' | jq '.usage'
+```
+
+---
+
+
+
+## 9. Knobs to sweep
+
+
+| Setting                                          | Effect                                                                          |
+| ------------------------------------------------ | ------------------------------------------------------------------------------- |
+| `routing.routing_budget.max_overhead_pct`        | Switching overhead cap as a % of never-switching (higher = quality-first, more switching) |
+| `routing.routing_budget.replenish_on_rebind`     | Reset the running baseline/spend totals when a cold session re-binds            |
+| `routing.routing_budget.cache_read_discount`     | Assumed cached rate for models whose feed entry omits a cached-read rate       |
+| `routing.routing_budget.record_counterfactual`   | Emit `plano.switch.counterfactual_route` on vetoed switches (the road not taken)|
+| `prompt_caching.session_ttl_seconds`             | Session binding GC lifetime                                                     |
+| `prompt_caching.min_prefix_tokens`               | Minimum stable-prefix size before markers are injected                          |
+| Header `X-Model-Affinity: <id>`                  | Explicit session key (overrides the implicit prefix hash)                       |
+| Header `X-Plano-Cache: off`                      | Per-request bypass for baseline runs                                            |
+
+
+---
+
+
+
+## Notes
+
+- Caching **never** changes which model routing selects — the router still makes
+the quality call; the overhead cap only vetoes a switch that the session can't afford.
+- The routing budget is independent of prompt caching (it lives under `routing`,
+needs no `prompt_caching` config, and always prices warm anchors at cached rates)
+and is fully opt-in with **no baked-in cap**: configuring it without a
+`max_overhead_pct` (or without a cost source) fails startup with a clear message.

demos/llm_routing/routing_budget/README.md

@@ -0,0 +1,38 @@
+# Routing Budget
+
+Preference-based routing with a cumulative per-session **routing budget** that
+protects warm provider caches, plus automatic prompt caching. The budget is a
+routing concern configured under `routing` — independent of prompt caching.
+
+## The problem
+
+Provider prompt caches are per-model. When intelligent routing moves a
+conversation to a different model, the new model re-ingests the full context at
+its **uncached** input rate. In input-heavy, append-only workloads (coding
+agents especially), a nominally cheaper model can end up more expensive than
+the cached rate you abandoned.
+
+## What the routing budget does
+
+The router runs every turn (routing stays cache-blind). When it proposes a model
+that differs from the session's warm anchor, Plano computes the **actual
+input-token cost** of abandoning the anchor's cache and only allows the switch
+while the session's cumulative switch spend stays within `max_overhead_pct`% of
+what never-switching would have cost — otherwise it retains the warm anchor. The
+promise: the conversation bills at most `max_overhead_pct`% above never-switching.
+
+Quality and cost stay separate — the router still picks the best model; the
+budget only vetoes switches the session can't afford. Prompt caching
+(`prompt_caching.enabled`) is a separate, optional concern that keeps the
+upstream cache warm and injects provider cache-control markers.
+
+## Run
+
+```bash
+planoai up config.yaml
+```
+
+See [config.yaml](config.yaml) for the annotated configuration, and
+**[GUIDE.md](GUIDE.md)** for the full hands-on walkthrough — running it as a
+routing decision, watching a switch get vetoed vs. allowed, the switch-cost
+math, and all the metrics and trace attributes for evals.

demos/llm_routing/routing_budget/config.yaml

@@ -0,0 +1,94 @@
+version: v0.3.0
+
+listeners:
+  - type: model
+    name: model_listener
+    port: 12000
+
+model_providers:
+
+  - model: openai/gpt-4o-mini
+    access_key: $OPENAI_API_KEY
+    default: true
+
+  - model: openai/gpt-4o
+    access_key: $OPENAI_API_KEY
+    routing_preferences:
+      - name: code understanding
+        description: understand and explain existing code snippets, functions, or libraries
+
+  - model: anthropic/claude-sonnet-4-6
+    access_key: $ANTHROPIC_API_KEY
+    routing_preferences:
+      - name: code generation
+        description: generating new code snippets, functions, or boilerplate based on user prompts or requirements
+
+# Per-model pricing is required for the routing budget: the switch-cost
+# calculation needs each model's input and cached-input rates. This demo reads
+# DigitalOcean's managed pricing catalog, which publishes cached-read rates
+# (`cache_read_input_price_per_million` / `input_cache_read`). The catalog is
+# keyed by bare DO model ids (e.g. `openai-gpt-4o`), so model_aliases maps them
+# onto the Plano model names used in model_providers above — without the
+# mapping no rates match and every switch decision fails open (no_pricing).
+model_metrics_sources:
+  - type: cost
+    provider: digitalocean
+    refresh_interval: 86400
+    model_aliases:
+      openai-gpt-4o-mini: openai/gpt-4o-mini
+      openai-gpt-4o: openai/gpt-4o
+      anthropic-claude-4.6-sonnet: anthropic/claude-sonnet-4-6
+
+  # models.dev works identically and needs no aliases (its keys already match
+  # the provider/model routing names). Rates for these models are the same on
+  # both feeds, so the walkthrough numbers in GUIDE.md hold either way.
+  # - type: cost
+  #   provider: models.dev
+  #   refresh_interval: 86400
+
+# Automatic prompt caching (opt-in). OPTIONAL for the routing budget: the budget
+# derives sessions and prices warm anchors at cached rates on its own. Enable this
+# only for what it adds — injecting cache-control markers on the full-proxy path
+# (required for the provider cache to be real on marker-based models like
+# anthropic/*; see GUIDE.md §4) and session affinity without a budget.
+# prompt_caching:
+#   enabled: true
+
+routing:
+  # Per-session cost gate on model switching. Independent of prompt caching: it
+  # applies whenever configured, and presence of this block turns it on. The
+  # default posture is to stick to the model a session is warm on. When routing
+  # proposes a *different* model while that session's provider cache is plausibly
+  # still warm (inferred from how long ago the session was last used vs. the
+  # provider's cache window), the actual input-token cost of abandoning the cache:
+  #
+  #   switch_cost = context_tokens x (candidate_uncached_input - anchor_cached_input)
+  #
+  # (output cost deliberately excluded) accrues into the session's cumulative switch
+  # spend. A paid switch is allowed only while that spend stays within max_overhead_pct
+  # percent of the session's running never-switch baseline (what staying on the anchor
+  # would have cost); an outright-cheaper switch is free but never reduces the spend.
+  # The cap is yours to define -- Plano never invents one, and startup fails without it
+  # (or without a cost source).
+  routing_budget:
+    # Cap on cumulative switching overhead, as a percentage of what the session would
+    # have cost by never switching (a whole number: 20 = 20%). The promise is "this
+    # conversation bills at most 20% above never-switching." 0 means "never pay to
+    # switch"; larger values buy more switches before sticking. Typical range 10-30.
+    max_overhead_pct: 20
+
+    # Reset the running baseline/spend totals when a cold session re-binds. Default true.
+    # replenish_on_rebind: true
+
+    # Fallback used only when the pricing feed doesn't publish a cached-read rate for
+    # a model: cached_rate = input_rate x cache_read_discount. Default 0.1.
+    # cache_read_discount: 0.1
+
+    # Record the route the gate WOULD have taken when it vetoes a switch, as the
+    # `plano.switch.counterfactual_route` trace attribute. Telemetry only -- the
+    # counterfactual model is never dispatched. Handy for evals that want to
+    # measure the road not taken. Default false.
+    record_counterfactual: true
+
+tracing:
+  random_sampling: 100

demos/llm_routing/routing_budget/demo.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -e
+
+# Routing Budget demo — drives the /routing decision endpoint to show:
+#   1. implicit session pinning (same session across turns, going warm)
+#   2. the routing budget vetoing an unaffordable model switch
+#
+# Prereqs: `planoai up config.yaml` running, plus `curl` and `jq`.
+# See GUIDE.md for the full walkthrough and how to flip the veto into an allow.
+
+PLANO_URL="${PLANO_URL:-http://localhost:12000}"
+METRICS_URL="${METRICS_URL:-http://localhost:9092}"
+
+echo "=== Routing Budget Demo ==="
+echo ""
+echo "Uses the /routing/v1/chat/completions decision endpoint (no LLM call)."
+echo "Watch session_id stay stable and pinned flip false -> true, then watch"
+echo "the budget retain the warm anchor instead of following the router."
+echo ""
+
+# --- Turn 1: pin the session (a code-generation prompt) ---
+echo "--- 1. Turn 1: pin the session (creates the binding) ---"
+echo ""
+curl -s "$PLANO_URL/routing/v1/chat/completions" \
+  -H 'Content-Type: application/json' \
+  -d '{"model": "openai/gpt-4o-mini", "messages": [
+    {"role": "system", "content": "You are a senior Rust engineer."},
+    {"role": "user", "content": "Write a Rust function that reverses a linked list."}
+  ]}' | jq '{model: .models[0], session_id, pinned}'
+echo ""
+echo "    Expect: model=anthropic/claude-sonnet-4-6, an implicit:… session_id, pinned=false"
+echo ""
+
+# --- Turn 2: same system prompt + same first message, one turn later ---
+echo "--- 2. Turn 2: same session, warm, router proposes a different model ---"
+echo ""
+curl -s "$PLANO_URL/routing/v1/chat/completions" \
+  -H 'Content-Type: application/json' \
+  -d '{"model": "openai/gpt-4o-mini", "messages": [
+    {"role": "system", "content": "You are a senior Rust engineer."},
+    {"role": "user", "content": "Write a Rust function that reverses a linked list."},
+    {"role": "assistant", "content": "Here is an idiomatic in-place reversal for a singly linked list:\n\n```rust\ntype Link = Option<Box<Node>>;\n\nstruct Node {\n    val: i32,\n    next: Link,\n}\n\nfn reverse(mut head: Link) -> Link {\n    let mut prev: Link = None;\n    while let Some(mut node) = head {\n        head = node.next.take();\n        node.next = prev;\n        prev = Some(node);\n    }\n    prev\n}\n```\n\nIt walks the list once, moving the next pointer of each node to its predecessor."},
+    {"role": "user", "content": "Now explain its time complexity in plain English — no code."}
+  ]}' | jq '{model: .models[0], session_id, pinned}'
+echo ""
+echo "    Expect: SAME session_id as turn 1, pinned=true. If the router proposed"
+echo "    openai/gpt-4o (code understanding), the budget vetoed the switch and"
+echo "    model stays anthropic/claude-sonnet-4-6 (the warm anchor)."
+echo ""
+
+# --- Switch decisions metric ---
+echo "--- 3. Switch decisions (why the budget decided what it did) ---"
+echo ""
+curl -s "$METRICS_URL/metrics" | grep session_switch_decisions || true
+echo ""
+echo "    over_cap  = switch vetoed, anchor retained"
+echo "    free      = cheaper/affordable switch allowed"
+echo "    same_anchor = router did not propose a switch this turn"
+echo ""
+
+echo "=== Demo Complete ==="
+echo ""
+echo "To see the switch ALLOWED instead of vetoed: comment out the routing_budget"
+echo "block in config.yaml (or raise max_overhead_pct), then 'planoai down &&"
+echo "planoai up config.yaml' and re-run — turn 2 will follow the router to"
+echo "openai/gpt-4o. See GUIDE.md for details."

docs/requirements.txt

@@ -1,5 +1,4 @@
 sphinx_copybutton==0.5.2
-sphinxawesome-theme
+sphinxawesome-theme<6.0.0
 sphinx_sitemap
 sphinx_design
-sphinxawesome_theme

docs/routing-api.md

@@ -21,7 +21,7 @@ POST /v1/chat/completions
     {
       "name": "code generation",
       "description": "generating new code snippets",
-      "models": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o", "openai/gpt-4o-mini"]
+      "models": ["anthropic/claude-sonnet-4-6", "openai/gpt-4o", "openai/gpt-4o-mini"]
     },
     {
       "name": "general questions",
@@ -55,7 +55,7 @@ POST /v1/chat/completions
 ```json
 {
   "models": [
-    "anthropic/claude-sonnet-4-20250514",
+    "anthropic/claude-sonnet-4-6",
     "openai/gpt-4o",
     "openai/gpt-4o-mini"
   ],
@@ -100,7 +100,7 @@ Requires `version: v0.4.0` or above. Models listed under `routing_preferences` m
 version: v0.4.0
 
 model_providers:
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
   - model: openai/gpt-4o
     access_key: $OPENAI_API_KEY
@@ -112,7 +112,7 @@ routing_preferences:
   - name: code generation
     description: generating new code snippets or boilerplate
     models:
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
       - openai/gpt-4o
 
   - name: general questions
@@ -149,7 +149,7 @@ Response when pinned:
 
 ```json
 {
-  "models": ["anthropic/claude-sonnet-4-20250514"],
+  "models": ["anthropic/claude-sonnet-4-6"],
   "route": "code generation",
   "trace_id": "...",
   "session_id": "a1b2c3d4-5678-...",

docs/source/concepts/llm_providers/supported_providers.rst

@@ -8,7 +8,7 @@ Plano provides first-class support for multiple LLM providers through native int
 .. note::
    **Model Support:** Plano supports all chat models from each provider, not just the examples shown in this guide. The configurations below demonstrate common models for reference, but you can use any chat model available from your chosen provider.
 
-   Please refer to the quuickstart guide :ref:`here <llm_routing_quickstart>` to configure and use LLM providers via common client libraries like OpenAI and Anthropic Python SDKs, or via direct HTTP/cURL requests.
+   Please refer to the quickstart guide :ref:`here <llm_routing_quickstart>` to configure and use LLM providers via common client libraries like OpenAI and Anthropic Python SDKs, or via direct HTTP/cURL requests.
 
 
 Configuration Structure
@@ -179,14 +179,14 @@ Anthropic
       - model: anthropic/*
         access_key: $ANTHROPIC_API_KEY
 
-      - model: anthropic/claude-sonnet-4-20250514
+      - model: anthropic/claude-sonnet-4-6
         access_key: $ANTHROPIC_PROD_API_KEY
 
     routing_preferences:
       - name: code_generation
         description: generating new code snippets, functions, or boilerplate based on user prompts or requirements
         models:
-          - anthropic/claude-sonnet-4-20250514
+          - anthropic/claude-sonnet-4-6
 
 DeepSeek
 ~~~~~~~~
@@ -256,7 +256,6 @@ Mistral AI
      - Compact model
 
 **Configuration Examples:**
-**Configuration Examples:**
 
 .. code-block:: yaml
 
@@ -823,7 +822,7 @@ You can configure specific models with custom settings even when using wildcards
 
       # Override specific model with custom settings
       # This model will NOT be included in the wildcard expansion above
-      - model: anthropic/claude-sonnet-4-20250514
+      - model: anthropic/claude-sonnet-4-6
         access_key: $ANTHROPIC_PROD_API_KEY
 
       # Another specific override
@@ -834,7 +833,7 @@ You can configure specific models with custom settings even when using wildcards
       - name: code_generation
         description: generating new code snippets, functions, or boilerplate based on user prompts or requirements
         models:
-          - anthropic/claude-sonnet-4-20250514
+          - anthropic/claude-sonnet-4-6
 
 **Custom Provider Wildcards:**
 

docs/source/concepts/signals.rst

@@ -334,35 +334,6 @@ Emitted per category, only when ``count > 0``. One ``.count`` and one
    * - ``signals.environment.exhaustion.severity``
      - "
 
-Legacy attributes (deprecated, still emitted)
----------------------------------------------
-
-The following aggregate keys pre-date the paper taxonomy and are still
-emitted for one release so existing dashboards keep working. They are
-derived from the layered counts above and will be removed in a future
-release. Migrate to the layered keys when convenient.
-
-.. list-table::
-   :header-rows: 1
-   :widths: 50 50
-
-   * - Legacy attribute
-     - Layered equivalent
-   * - ``signals.follow_up.repair.count``
-     - ``signals.interaction.misalignment.count``
-   * - ``signals.follow_up.repair.ratio``
-     - (computed: ``misalignment.count / max(user_turns, 1)``)
-   * - ``signals.frustration.count``
-     - Count of ``disengagement.negative_stance`` instances
-   * - ``signals.frustration.severity``
-     - Derived severity bucket of the above
-   * - ``signals.repetition.count``
-     - ``signals.interaction.stagnation.count``
-   * - ``signals.escalation.requested``
-     - True if any ``disengagement.escalation`` or ``disengagement.quit`` fired
-   * - ``signals.positive_feedback.count``
-     - ``signals.interaction.satisfaction.count``
-
 Span Events
 ===========
 
@@ -520,11 +491,6 @@ event::
     signals.interaction.disengagement.count    = 6
     signals.interaction.disengagement.severity = 3
 
-    # Legacy (deprecated, emitted while dual-emit is on)
-    signals.frustration.count     = 4
-    signals.frustration.severity  = 2
-    signals.escalation.requested  = true
-
     # Per-instance span events
     event: signal.interaction.disengagement.escalation
       signal.type          = "interaction.disengagement.escalation"
@@ -537,8 +503,7 @@ Building Dashboards
 ===================
 
 Use signal attributes to build monitoring dashboards in Grafana, Honeycomb,
-Datadog, etc. Prefer the layered keys — they align with the paper taxonomy
-and will outlive the legacy keys.
+Datadog, etc. The layered keys align with the paper taxonomy.
 
 - **Quality distribution**: Count of traces by ``signals.quality``
 - **P95 turn count**: 95th percentile of ``signals.turn_count``

docs/source/conf.py

@@ -17,7 +17,7 @@ from sphinxawesome_theme.postprocess import Icons
 project = "Plano Docs"
 copyright = "2026, Katanemo Labs, a DigitalOcean Company"
 author = "Katanemo Labs, Inc"
-release = " v0.4.23"
+release = " v0.4.29"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

docs/source/get_started/quickstart.rst

@@ -43,7 +43,7 @@ Plano's CLI allows you to manage and interact with the Plano efficiently. To ins
 
 .. code-block:: console
 
-   $ uv tool install planoai==0.4.23
+   $ uv tool install planoai==0.4.29
 
 **Option 2: Install with pip (Traditional)**
 
@@ -51,7 +51,7 @@ Plano's CLI allows you to manage and interact with the Plano efficiently. To ins
 
    $ python -m venv venv
    $ source venv/bin/activate   # On Windows, use: venv\Scripts\activate
-   $ pip install planoai==0.4.23
+   $ pip install planoai==0.4.29
 
 
 .. _llm_routing_quickstart:

docs/source/guides/llm_router.rst

@@ -209,6 +209,178 @@ Clients can let the router decide or still specify aliases:
     )
 
 
+.. _cost_latency_aware_selection:
+
+Cost- and latency-aware selection
+---------------------------------
+
+When a route lists more than one candidate model, you can let Plano reorder that
+candidate pool using **live cost or latency data** instead of relying solely on the
+order you wrote them in. This is controlled per route with ``selection_policy`` and
+backed by one or more ``model_metrics_sources``.
+
+This is useful when several models are equally capable for a route and you want Plano
+to always reach for the cheapest (or fastest) option first, with the others kept as
+fallbacks.
+
+Selection policy
+~~~~~~~~~~~~~~~~~
+
+Attach an optional ``selection_policy`` to any entry in ``routing_preferences``:
+
+.. code-block:: yaml
+    :caption: Per-route selection policy
+
+    routing_preferences:
+      - name: code review
+        description: reviewing, analyzing, and suggesting improvements to existing code
+        models:
+          - anthropic/claude-sonnet-4-5
+          - groq/llama-3.3-70b-versatile
+        selection_policy:
+          prefer: cheapest   # cheapest | fastest | none
+
+``prefer`` accepts:
+
+- ``cheapest`` — order candidates by total price (input + output rate) ascending, using a ``cost`` metrics source.
+- ``fastest`` — order candidates by observed latency ascending, using a ``latency`` metrics source.
+- ``none`` (default) — keep the order you declared; no reordering.
+
+Models that have no data in the selected source are ranked **last**, in their original
+order, so routing always degrades gracefully rather than dropping a candidate.
+
+Configuring the pricing source
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``cheapest`` routing needs a price catalog. Plano's **default pricing provider is
+DigitalOcean** — its GenAI model catalog is public (no API key, no signup), so cost data
+is available out of the box and is what ``planoai obs`` uses if you don't configure
+anything. The pricing source is fully swappable: point Plano at `models.dev <https://models.dev/>`_,
+or at **any endpoint that exposes a supported pricing structure**.
+
+The ``provider`` field selects which response schema Plano expects (and therefore how it
+parses the catalog); the optional ``url`` lets you override the endpoint — for example to
+use a mirror, a cached copy, or an internal catalog service that returns the same shape.
+
+.. list-table::
+   :header-rows: 1
+   :widths: 18 34 28 20
+
+   * - ``provider``
+     - Default catalog URL
+     - Key format
+     - Expected structure
+   * - ``digitalocean`` *(default)*
+     - DigitalOcean GenAI model catalog
+     - ``lowercase(creator)/model_id``
+     - ``{ data: [ { model_id, pricing: { input_price_per_million, output_price_per_million } } ] }``
+   * - ``models.dev``
+     - ``https://models.dev/api.json``
+     - ``creator/model`` (e.g. ``anthropic/claude-sonnet-4-5``)
+     - ``{ <provider>: { models: { <model>: { cost: { input, output } } } } }``
+
+Because the source is selected per ``provider``, switching is a one-line change. To stay
+on the default DigitalOcean catalog you can omit ``model_metrics_sources`` entirely for
+``planoai obs``, or declare it explicitly for routing:
+
+.. code-block:: yaml
+    :caption: Default cost source (DigitalOcean)
+
+    model_metrics_sources:
+      - type: cost
+        provider: digitalocean   # default; uses the public DO GenAI catalog
+
+To switch to models.dev — an open, community-maintained catalog covering a broad range of
+providers and models — change the ``provider`` (and optionally ``url``):
+
+.. code-block:: yaml
+    :caption: Cost source backed by models.dev
+
+    model_metrics_sources:
+      - type: cost
+        provider: models.dev               # models.dev | digitalocean
+        url: https://models.dev/api.json    # optional; defaults per provider
+        refresh_interval: 3600              # optional, seconds; refetch on this interval
+        model_aliases:                      # optional; see below
+          openai/gpt-oss-120b: openai/gpt-4o
+
+To use your own endpoint, pick the ``provider`` whose structure your endpoint matches and
+override ``url`` — Plano parses the response with that provider's schema:
+
+.. code-block:: yaml
+    :caption: Custom endpoint exposing the DigitalOcean catalog structure
+
+    model_metrics_sources:
+      - type: cost
+        provider: digitalocean                       # selects the DO response schema
+        url: https://catalog.internal.example.com/pricing
+
+.. note::
+   The cost metric used for ranking is the sum of the input and output per-million-token
+   rates — a relative signal for ordering candidates, not a per-request bill. For actual
+   per-request cost, see the observability console below.
+
+Matching catalog keys to your models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The router looks up each candidate model by the exact name you use in
+``routing_preferences`` (e.g. ``anthropic/claude-sonnet-4-5``). models.dev keys models as
+``creator/model``, which lines up with Plano's ``provider/model`` naming, so most models
+match automatically.
+
+When a catalog key does not match your model name — for example a version skew, or an
+open-weight model you serve under a different provider — use ``model_aliases`` to map the
+**catalog key** to the **Plano model name** used in your routing preferences:
+
+.. code-block:: yaml
+
+    model_metrics_sources:
+      - type: cost
+        provider: models.dev
+        model_aliases:
+          # catalog key            : plano model name
+          openai/gpt-oss-120b: openai/gpt-4o
+
+Latency source
+~~~~~~~~~~~~~~~
+
+``fastest`` routing reads observed latency from a Prometheus instance. Provide the query
+that returns a per-model latency value (lower is faster), labelled by ``model_name``:
+
+.. code-block:: yaml
+    :caption: Latency source backed by Prometheus
+
+    model_metrics_sources:
+      - type: latency
+        provider: prometheus
+        url: http://prometheus:9090
+        query: avg by (model_name) (rate(plano_llm_latency_seconds_sum[5m]))
+        refresh_interval: 60
+
+You can declare both a ``cost`` and a ``latency`` source at the same time; each route
+picks whichever it needs based on its ``selection_policy``.
+
+Cost in the observability console
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``planoai obs`` displays a per-request USD cost column derived from the same pricing
+catalog. By default it reads the ``cost`` source from your config (the first
+``type: cost`` entry under ``model_metrics_sources``); you can also override it on the
+command line:
+
+.. code-block:: bash
+
+    # Use the cost source from ./config.yaml (default)
+    planoai obs
+
+    # Or override the provider / endpoint explicitly
+    planoai obs --pricing-provider models.dev
+    planoai obs --pricing-url https://models.dev/api.json
+
+If no source is configured and no override is given, ``planoai obs`` falls back to the
+DigitalOcean catalog so the cost column still populates out of the box.
+
+
 Plano-Orchestrator
 -------------------
 Plano-Orchestrator is a **preference-based routing model** specifically designed to address the limitations of traditional LLM routing. It delivers production-ready performance with low latency and high accuracy while solving key routing challenges.

docs/source/guides/observability/tracing.rst

@@ -153,7 +153,7 @@ In your observability platform (Jaeger, Grafana Tempo, Datadog, etc.), filter tr
 - Find external issues: ``signals.environment.exhaustion.count > 0``
 - Find inefficient flows: ``signals.efficiency_score < 0.5``
 
-For complete details on all 20 leaf signal types, severity scheme, legacy attribute deprecation, and best practices, see the :doc:`../../concepts/signals` guide.
+For complete details on all 20 leaf signal types, severity scheme, and best practices, see the :doc:`../../concepts/signals` guide.
 
 
 Custom Span Attributes
@@ -259,6 +259,86 @@ Request headers::
 Result: no attributes are captured from ``X-Other-User-Id``.
 
 
+Exporting Telemetry Anywhere
+----------------------------
+
+Beyond the OTLP/gRPC collector, Plano can stream LLM telemetry directly to
+third-party observability backends through ``tracing.exporters``. The list is
+provider-agnostic: each entry is tagged by its ``type`` and points at a URL, so
+new destinations can be added without changing anything else. Exporters run in
+addition to ``opentracing_grpc_endpoint`` — you can use one, the other, or both.
+
+PostHog
+~~~~~~~
+
+PostHog is supported as a first-class integration. Every LLM call is captured as
+a PostHog `$ai_generation <https://posthog.com/docs/ai-observability/generations>`_
+event and POSTed to PostHog's capture API. Setup is intentionally minimal —
+point at your PostHog URL and project token::
+
+  tracing:
+    random_sampling: 100
+    exporters:
+      - type: posthog
+        url: https://us.i.posthog.com   # /batch/ is appended automatically
+        api_key: $POSTHOG_API_KEY        # PostHog project token (env expansion supported)
+        distinct_id_header: x-user-id    # optional; omit for anonymous capture
+        capture_messages: false          # optional; send user message as $ai_input
+
+That's all that's required. When ``random_sampling`` is greater than ``0`` and at
+least one exporter (or ``opentracing_grpc_endpoint``) is configured, tracing is
+enabled and ``$ai_generation`` events begin flowing. They appear under PostHog's
+**AI Observability** in the Traces and Generations tabs.
+
+**Captured properties**
+
+Plano maps span data onto PostHog ``$ai_*`` properties:
+
+.. list-table::
+   :header-rows: 1
+   :widths: 30 70
+
+   * - PostHog property
+     - Source
+   * - ``$ai_model``
+     - Resolved upstream model (``llm.model``)
+   * - ``$ai_provider``
+     - Provider derived from the resolved model (``llm.provider``)
+   * - ``$ai_latency``
+     - Total call duration in seconds (``llm.duration_ms``)
+   * - ``$ai_time_to_first_token``
+     - Time to first token in seconds, streaming only
+   * - ``$ai_input_tokens`` / ``$ai_output_tokens``
+     - Prompt / completion token usage
+   * - ``$ai_http_status`` / ``$ai_is_error``
+     - Upstream HTTP status and error flag
+   * - ``$ai_trace_id`` / ``$ai_parent_id``
+     - Trace and parent span identifiers
+   * - ``distinct_id``
+     - Value of ``distinct_id_header`` (else anonymous)
+
+**Identifying users**
+
+Set ``distinct_id_header`` to the request header carrying your user identity
+(for example ``x-user-id``). When present, Plano stamps the value as the PostHog
+``distinct_id``. When the header is missing — or ``distinct_id_header`` is not
+configured — the event is captured anonymously (``$process_person_profile`` is
+set to ``false``), matching PostHog's anonymous vs. identified semantics.
+
+**Capturing message content**
+
+By default Plano does not send prompt content off-box. Set
+``capture_messages: true`` to include the (truncated) user message preview as
+``$ai_input``. Leave it ``false`` when prompt content must not leave your data
+plane.
+
+**Multiple destinations**
+
+``exporters`` is a list, so you can fan out to several backends (and combine
+with an OTLP collector). A common use is shipping to multiple PostHog instances
+(for example separate EU and US projects for data-residency).
+
+
 Benefits of Using ``Traceparent`` Headers
 -----------------------------------------
 

docs/source/resources/deployment.rst

@@ -65,7 +65,7 @@ Create a ``docker-compose.yml`` file with the following configuration:
    # docker-compose.yml
    services:
      plano:
-       image: katanemo/plano:0.4.23
+       image: katanemo/plano:0.4.29
        container_name: plano
        ports:
          - "10000:10000" # ingress (client -> plano)
@@ -153,7 +153,7 @@ Create a ``plano-deployment.yaml``:
        spec:
          containers:
            - name: plano
-             image: katanemo/plano:0.4.23
+             image: katanemo/plano:0.4.29
              ports:
                - containerPort: 12000  # LLM gateway (chat completions, model routing)
                  name: llm-gateway

docs/source/resources/includes/plano_config_full_reference.yaml

@@ -86,6 +86,24 @@ routing_preferences:
     selection_policy:
       prefer: cheapest
 
+# model_metrics_sources: external catalogs the router reads to reorder candidate
+# models for selection_policy.prefer. A `cost` source ranks `prefer: cheapest`;
+# a `latency` source ranks `prefer: fastest`. Both are optional.
+model_metrics_sources:
+  # Cost catalog. provider: models.dev | digitalocean (default url per provider).
+  - type: cost
+    provider: models.dev
+    url: https://models.dev/api.json   # optional; omit to use the provider default
+    refresh_interval: 3600             # optional, seconds
+    model_aliases:                     # optional: catalog key -> Plano model name
+      openai/gpt-oss-120b: openai/gpt-4o
+  # Latency catalog (Prometheus). Used for selection_policy.prefer: fastest.
+  - type: latency
+    provider: prometheus
+    url: http://prometheus:9090
+    query: avg by (model_name) (rate(plano_llm_latency_seconds_sum[5m]))
+    refresh_interval: 60
+
 # HTTP listeners - entry points for agent routing, prompt targets, and direct LLM access
 listeners:
   # Agent listener for routing requests to multiple agents
@@ -243,3 +261,16 @@ tracing:
     static:
       environment: production
       service.team: platform
+  # Provider-agnostic export destinations. LLM spans are streamed to each of
+  # these in addition to any opentracing_grpc_endpoint above.
+  exporters:
+    # PostHog AI observability: each LLM call is captured as an $ai_generation event.
+    - type: posthog
+      # PostHog host. The /batch/ capture path is appended automatically.
+      url: https://us.i.posthog.com
+      # PostHog project API key (token). Supports $ENV_VAR expansion.
+      api_key: $POSTHOG_API_KEY
+      # Optional: request header used as the PostHog distinct_id. Omit for anonymous capture.
+      distinct_id_header: x-user-id
+      # Optional: include the (truncated) user message as $ai_input. Defaults to false.
+      capture_messages: false

docs/source/resources/includes/plano_config_full_reference_rendered.yaml

@@ -115,6 +115,18 @@ model_aliases:
     target: gpt-4o-mini
   smart-llm:
     target: gpt-4o
+model_metrics_sources:
+- model_aliases:
+    openai/gpt-oss-120b: openai/gpt-4o
+  provider: models.dev
+  refresh_interval: 3600
+  type: cost
+  url: https://models.dev/api.json
+- provider: prometheus
+  query: avg by (model_name) (rate(plano_llm_latency_seconds_sum[5m]))
+  refresh_interval: 60
+  type: latency
+  url: http://prometheus:9090
 model_providers:
 - access_key: $OPENAI_API_KEY
   default: true
@@ -254,6 +266,12 @@ system_prompt: 'You are a helpful assistant. Always respond concisely and accura
 
   '
 tracing:
+  exporters:
+  - api_key: $POSTHOG_API_KEY
+    capture_messages: false
+    distinct_id_header: x-user-id
+    type: posthog
+    url: https://us.i.posthog.com
   opentracing_grpc_endpoint: http://localhost:4317
   random_sampling: 100
   span_attributes:

docs/source/resources/includes/plano_config_state_storage_example.yaml

@@ -13,7 +13,7 @@ model_providers:
     default: true
 
     # Anthropic Models
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 # State storage configuration for v1/responses API

skills/AGENTS.md

@@ -171,7 +171,7 @@ Plano translates requests between its internal format and each provider's API. T
 | Model prefix | Wire format | Example |
 |---|---|---|
 | `openai/*` | OpenAI | `openai/gpt-4o` |
-| `anthropic/*` | Anthropic | `anthropic/claude-sonnet-4-20250514` |
+| `anthropic/*` | Anthropic | `anthropic/claude-sonnet-4-6` |
 | `gemini/*` | Google Gemini | `gemini/gemini-2.0-flash` |
 | `mistral/*` | Mistral | `mistral/mistral-large-latest` |
 | `groq/*` | Groq | `groq/llama-3.3-70b-versatile` |
@@ -199,7 +199,7 @@ model_providers:
     access_key: $OPENAI_API_KEY
     default: true
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
   - model: gemini/gemini-2.0-flash
@@ -262,7 +262,7 @@ model_providers:
     access_key: $OPENAI_API_KEY
     default: true
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 state_storage:
@@ -431,7 +431,7 @@ model_providers:
     default: true
   - model: openai/gpt-4o
     access_key: $OPENAI_API_KEY
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 model_aliases:
@@ -442,7 +442,7 @@ model_aliases:
     target: gpt-4o               # High capability — for complex reasoning
 
   plano.creative.v1:
-    target: claude-sonnet-4-20250514  # Strong creative writing and analysis
+    target: claude-sonnet-4-6  # Strong creative writing and analysis
 
   plano.v1:
     target: gpt-4o               # Default production alias
@@ -1419,7 +1419,7 @@ listeners:
     port: 12000
 
 model_providers:
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     default: true
 
@@ -1432,7 +1432,7 @@ routing_preferences:
       Writing code, debugging, code review, explaining concepts,
       answering programming questions, general development tasks.
     models:
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
       - anthropic/claude-opus-4-6
   - name: complex architecture
     description: >
@@ -1440,11 +1440,11 @@ routing_preferences:
       architectural decisions, performance optimization, security audits.
     models:
       - anthropic/claude-opus-4-6
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
 
 model_aliases:
   claude.fast.v1:
-    target: claude-sonnet-4-20250514
+    target: claude-sonnet-4-6
   claude.smart.v1:
     target: claude-opus-4-6
 
@@ -1838,7 +1838,7 @@ model_providers:
   - model: openai/gpt-4o
     access_key: $OPENAI_API_KEY
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
 
 # --- Shared routing_preferences (top-level, v0.4.0+) ---
@@ -1851,11 +1851,11 @@ routing_preferences:
     description: Multi-step analysis, code generation, research synthesis
     models:
       - openai/gpt-4o
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
   - name: long documents
     description: Summarizing or analyzing very long documents, PDFs, transcripts
     models:
-      - anthropic/claude-sonnet-4-20250514
+      - anthropic/claude-sonnet-4-6
       - openai/gpt-4o
 
 # --- Listener 1: OpenAI-compatible API gateway ---

skills/rules/advanced-multi-listener.md

@@ -42,7 +42,7 @@ model_providers:
       - name: complex reasoning
         description: Multi-step analysis, code generation, research synthesis
 
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     routing_preferences:
       - name: long documents

skills/rules/cli-agent.md

@@ -43,7 +43,7 @@ listeners:
     port: 12000
 
 model_providers:
-  - model: anthropic/claude-sonnet-4-20250514
+  - model: anthropic/claude-sonnet-4-6
     access_key: $ANTHROPIC_API_KEY
     default: true
     routing_preferences:
@@ -62,7 +62,7 @@ model_providers:
 
 model_aliases:
   claude.fast.v1:
-    target: claude-sonnet-4-20250514
+    target: claude-sonnet-4-6
   claude.smart.v1:
     target: claude-opus-4-6