diff --git a/crates/brightstaff/src/router/model_metrics.rs b/crates/brightstaff/src/router/model_metrics.rs index 01050063..2990a046 100644 --- a/crates/brightstaff/src/router/model_metrics.rs +++ b/crates/brightstaff/src/router/model_metrics.rs @@ -24,7 +24,8 @@ pub struct ModelRates { pub input_per_million: f64, pub output_per_million: f64, /// Cached (prompt-cache read) input rate. Present when the feed publishes it - /// (models.dev `cost.cache_read`); absent for feeds that don't (DO catalog). + /// (models.dev `cost.cache_read`, DO catalog `cache_read_input_price_per_million` + /// / `input_cache_read`); absent for models the feed prices without a cache SKU. pub cache_read_per_million: Option, } @@ -268,11 +269,23 @@ struct DoModel { /// DigitalOcean catalog pricing. Despite the `_per_million` field names, the DO /// API returns these as USD **per token** (e.g. gpt-4o input `2.5e-6`), so they /// are scaled to per-million at ingestion to match `ModelRates`' convention. +/// +/// The catalog publishes the cached-read rate under two names (a migration in +/// flight): `cache_read_input_price_per_million` and `input_cache_read`. Entries +/// may carry either or both; both are parsed and the explicit name wins. #[derive(serde::Deserialize)] struct DoPricing { input_price_per_million: Option, output_price_per_million: Option, cache_read_input_price_per_million: Option, + input_cache_read: Option, +} + +impl DoPricing { + fn cache_read(&self) -> Option { + self.cache_read_input_price_per_million + .or(self.input_cache_read) + } } #[derive(serde::Deserialize)] @@ -359,9 +372,7 @@ fn parse_do_pricing( * TOKENS_PER_MILLION, output_per_million: pricing.output_price_per_million.unwrap_or(0.0) * TOKENS_PER_MILLION, - cache_read_per_million: pricing - .cache_read_input_price_per_million - .map(|r| r * TOKENS_PER_MILLION), + cache_read_per_million: pricing.cache_read().map(|r| r * TOKENS_PER_MILLION), }; Some((key, rates)) }) @@ -677,6 +688,58 @@ mod tests { ); } + #[test] + fn test_parse_do_pricing_reads_cache_rate_from_either_field_name() { + // Real catalog shape (Jul 2026): DO publishes the cached-read rate under + // both `cache_read_input_price_per_million` and `input_cache_read` for + // some models, and only one of the two for others. All variants must + // resolve; the explicit name wins when both are present. + let json = r#"{ + "data": [ + { + "model_id": "kimi-k2.5", + "pricing": { + "input_price_per_million": 3.75e-7, + "output_price_per_million": 2.025e-6, + "cache_read_input_price_per_million": 2.03e-7, + "input_cache_read": 2.03e-7 + } + }, + { + "model_id": "glm-5", + "pricing": { + "input_price_per_million": 6e-7, + "output_price_per_million": 2.2e-6, + "input_cache_read": 1.1e-7 + } + }, + { + "model_id": "legacy-model", + "pricing": { + "input_price_per_million": 5e-7, + "output_price_per_million": 1e-6 + } + } + ] + }"#; + let list: DoModelList = serde_json::from_str(json).unwrap(); + let rates = parse_do_pricing(list, &HashMap::new()); + + // Both names present -> parsed (and scaled per-token -> per-million). + let kimi = rates.get("kimi-k2.5").unwrap(); + assert!((kimi.input_per_million - 0.375).abs() < 1e-9); + assert!((kimi.cache_read_per_million.unwrap() - 0.203).abs() < 1e-9); + + // Only the new `input_cache_read` name -> still parsed. + let glm = rates.get("glm-5").unwrap(); + assert!((glm.cache_read_per_million.unwrap() - 0.11).abs() < 1e-9); + + // Neither -> None, so the budget falls back to input x cache_read_discount. + let legacy = rates.get("legacy-model").unwrap(); + assert_eq!(legacy.cache_read_per_million, None); + assert!((legacy.cached_input_rate(0.1) - 0.05).abs() < 1e-9); + } + #[test] fn test_parse_models_dev_pricing_composes_provider_keys() { let json = r#"{ diff --git a/demos/llm_routing/routing_budget/GUIDE.md b/demos/llm_routing/routing_budget/GUIDE.md index 9c038dc2..95e67a39 100644 --- a/demos/llm_routing/routing_budget/GUIDE.md +++ b/demos/llm_routing/routing_budget/GUIDE.md @@ -34,11 +34,19 @@ Start from `[config.yaml](config.yaml)` in this folder. The parts that matter: ```yaml # Per-model pricing is REQUIRED for the routing budget — the switch cost math needs -# each model's input and cached-input rates. +# each model's input and cached-input rates. This demo reads DigitalOcean's managed +# pricing catalog, which publishes cached-read rates for its models. The catalog is +# keyed by bare DO model ids, so model_aliases maps them onto the Plano model names +# used in model_providers — without the mapping no rates match and every switch +# decision fails open (reason="no_pricing"). model_metrics_sources: - type: cost - provider: models.dev # publishes real cache_read rates + provider: digitalocean refresh_interval: 86400 + model_aliases: + openai-gpt-4o-mini: openai/gpt-4o-mini + openai-gpt-4o: openai/gpt-4o + anthropic-claude-4.6-sonnet: anthropic/claude-sonnet-4-6 prompt_caching: enabled: true # automatic caching + session affinity (separate concern) @@ -50,16 +58,21 @@ routing: # cache_read_discount: 0.1 # fallback when a feed omits cache_read ``` +`models.dev` works as a drop-in alternative cost source (`provider: models.dev`, +no aliases needed — its keys already match `provider/model` routing names). The +demo models are priced identically on both feeds, so every number in this guide +holds either way. + The routing budget lives under `routing` and is independent of prompt caching — it applies whether or not `prompt_caching.enabled` is set. -### DigitalOcean variant +### DigitalOcean-hosted models -Address DO GenAI models with the `digitalocean/` prefix and point the cost feed -at the DO catalog (or keep `models.dev`, which publishes cached-read rates the -DO catalog doesn't): +To route to DO GenAI models themselves (not just price from the DO catalog), +address them with the `digitalocean/` prefix and alias the catalog keys to +those names: ```yaml model_providers: @@ -73,11 +86,15 @@ model_metrics_sources: - type: cost provider: digitalocean # DO catalog refresh_interval: 86400 + model_aliases: + anthropic-claude-4.6-sonnet: digitalocean/anthropic-claude-4.6-sonnet + openai-gpt-4o: digitalocean/openai-gpt-4o ``` -> The DO catalog does not publish a cached-read rate, so for DO-only setups the -> gate falls back to `input_rate × cache_read_discount`. For exact cached rates, -> add a `models.dev` cost source instead. +> The DO catalog publishes cached-read rates +> (`cache_read_input_price_per_million`), so the gate prices warm anchors at the +> real cached rate. The `cache_read_discount` fallback only kicks in for models +> whose catalog entry omits the field. --- @@ -367,7 +384,7 @@ curl -s localhost:12000/v1/chat/completions \ | ------------------------------------------------ | ------------------------------------------------------------------------------- | | `routing.routing_budget.max_overhead_pct` | Switching overhead cap as a % of never-switching (higher = quality-first, more switching) | | `routing.routing_budget.replenish_on_rebind` | Reset the running baseline/spend totals when a cold session re-binds | -| `routing.routing_budget.cache_read_discount` | Assumed cached rate when a feed omits `cache_read` (DO fallback) | +| `routing.routing_budget.cache_read_discount` | Assumed cached rate for models whose feed entry omits a cached-read rate | | `routing.routing_budget.record_counterfactual` | Emit `plano.switch.counterfactual_route` on vetoed switches (the road not taken)| | `prompt_caching.session_ttl_seconds` | Session binding GC lifetime | | `prompt_caching.min_prefix_tokens` | Minimum stable-prefix size before markers are injected | diff --git a/demos/llm_routing/routing_budget/config.yaml b/demos/llm_routing/routing_budget/config.yaml index 082b4b4c..b66e3a7b 100644 --- a/demos/llm_routing/routing_budget/config.yaml +++ b/demos/llm_routing/routing_budget/config.yaml @@ -24,11 +24,27 @@ model_providers: description: generating new code snippets, functions, or boilerplate based on user prompts or requirements # Per-model pricing is required for the routing budget: the switch-cost -# calculation needs each model's input and cached-input rates. +# calculation needs each model's input and cached-input rates. This demo reads +# DigitalOcean's managed pricing catalog, which publishes cached-read rates +# (`cache_read_input_price_per_million` / `input_cache_read`). The catalog is +# keyed by bare DO model ids (e.g. `openai-gpt-4o`), so model_aliases maps them +# onto the Plano model names used in model_providers above — without the +# mapping no rates match and every switch decision fails open (no_pricing). model_metrics_sources: - type: cost - provider: models.dev + provider: digitalocean refresh_interval: 86400 + model_aliases: + openai-gpt-4o-mini: openai/gpt-4o-mini + openai-gpt-4o: openai/gpt-4o + anthropic-claude-4.6-sonnet: anthropic/claude-sonnet-4-6 + + # models.dev works identically and needs no aliases (its keys already match + # the provider/model routing names). Rates for these models are the same on + # both feeds, so the walkthrough numbers in GUIDE.md hold either way. + # - type: cost + # provider: models.dev + # refresh_interval: 86400 # Automatic prompt caching (opt-in). Keeps a conversation pinned to the same # model so the upstream provider's prompt cache stays warm across turns, and