From ad92f7fbe7e00e3ca938e13835bcfe868159f955 Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Wed, 22 Jul 2026 13:55:48 -0700 Subject: [PATCH] feat(routing): decouple budget pricing from prompt_caching (#996) * feat(routing): budget prices cache-aware regardless of prompt_caching flag * docs: prompt_caching is optional for the routing budget --- config/plano_config_schema.yaml | 6 +- crates/brightstaff/src/handlers/llm/mod.rs | 1 - .../src/handlers/llm/session_router.rs | 84 ++++--------------- .../src/handlers/routing_service.rs | 1 - crates/brightstaff/src/router/orchestrator.rs | 41 +++------ crates/common/src/configuration.rs | 7 +- demos/llm_routing/routing_budget/GUIDE.md | 40 +++++---- demos/llm_routing/routing_budget/config.yaml | 13 +-- 8 files changed, 69 insertions(+), 124 deletions(-) diff --git a/config/plano_config_schema.yaml b/config/plano_config_schema.yaml index 503c8c4d..0b92c777 100644 --- a/config/plano_config_schema.yaml +++ b/config/plano_config_schema.yaml @@ -547,8 +547,10 @@ properties: routing_budget: type: object description: > - Per-session cost gate on model switching. Independent of prompt caching — it - applies whenever configured (presence of this block turns it on). The default + Per-session cost gate on model switching. Self-sufficient and independent of + prompt_caching — it applies whenever configured (presence of this block turns + it on), derives implicit sessions on its own, and always prices warm anchors + at their cached input rate (provider caches are assumed real). The default posture is to stick to the model a session is warm on. When routing proposes a different model while that session's provider cache is plausibly still warm (inferred from the idle gap vs. the provider's cache window), the actual diff --git a/crates/brightstaff/src/handlers/llm/mod.rs b/crates/brightstaff/src/handlers/llm/mod.rs index 8751b25c..5cccf930 100644 --- a/crates/brightstaff/src/handlers/llm/mod.rs +++ b/crates/brightstaff/src/handlers/llm/mod.rs @@ -408,7 +408,6 @@ async fn llm_chat_inner( context_tokens, candidate_model: &candidate_model, candidate_route: candidate_route.as_deref(), - caching_enabled: prompt_caching.enabled, }, ) .await; diff --git a/crates/brightstaff/src/handlers/llm/session_router.rs b/crates/brightstaff/src/handlers/llm/session_router.rs index 8479358e..651b517d 100644 --- a/crates/brightstaff/src/handlers/llm/session_router.rs +++ b/crates/brightstaff/src/handlers/llm/session_router.rs @@ -13,10 +13,11 @@ //! baseline (what staying on the session's `default_model` would have cost — priced //! independently of the current anchor, which drifts as switches happen). An //! outright-cheaper switch is free but never reduces that spend. The promise: this -//! conversation bills at most `max_overhead_pct`% above never-switching. With prompt -//! caching the anchor is warm, so staying is priced at its cached rate and the switch -//! pays the cache-loss delta; without caching there is no warm cache, so both the -//! baseline and the switch cost are priced at the plain uncached input rate. +//! conversation bills at most `max_overhead_pct`% above never-switching. A warm +//! anchor is priced at its *cached* rate and the switch pays the cache-loss delta: +//! provider caches are assumed real whenever the budget is active (most providers +//! cache automatically), independent of `prompt_caching.enabled`, which only +//! controls Plano's own marker injection and caching-without-budget affinity. //! //! The default posture is to stick. Quality and cost stay separate: the router decides //! whether a switch *improves quality*; the overhead cap decides whether it is *affordable*. @@ -108,11 +109,6 @@ pub struct RouteFacts<'a> { /// The model the quality router picked for this request. pub candidate_model: &'a str, pub candidate_route: Option<&'a str>, - /// Whether automatic prompt caching is enabled for this instance. Selects how the - /// switch-cost gate prices context reads: with caching the anchor is warm (cached - /// rate, plus warm-token credit for A→B→A returns); without it there is no warm - /// cache, so both sides are priced at their plain uncached input rate. - pub caching_enabled: bool, } /// The routing decision plus the session state to carry into the response side. @@ -288,7 +284,6 @@ pub async fn route( context_tokens, &session_default, cfg.cache_read_discount, - facts.caching_enabled, ) .await .unwrap_or(0.0), @@ -309,21 +304,17 @@ pub async fn route( ceiling_opt = Some(ceiling); // Credit any context the candidate still has cached from an earlier visit // this session: a return to a still-warm model re-reads only the tokens - // appended since, not the whole context (the A→B→A case). Only meaningful - // with prompt caching — without it there is no cache to return to. - candidate_warm_tokens = if facts.caching_enabled { - b.history - .iter() - .find(|v| v.model == facts.candidate_model) - .filter(|v| { - now.duration_since(v.last_used).unwrap_or(Duration::MAX) - <= warmth_window(&capability_for_model(facts.candidate_model)) - }) - .map(|v| v.cached_tokens.min(context_tokens)) - .unwrap_or(0) - } else { - 0 - }; + // appended since, not the whole context (the A→B→A case). + candidate_warm_tokens = b + .history + .iter() + .find(|v| v.model == facts.candidate_model) + .filter(|v| { + now.duration_since(v.last_used).unwrap_or(Duration::MAX) + <= warmth_window(&capability_for_model(facts.candidate_model)) + }) + .map(|v| v.cached_tokens.min(context_tokens)) + .unwrap_or(0); match orchestrator .estimate_switch_cost_in_usd( context_tokens, @@ -331,7 +322,6 @@ pub async fn route( facts.candidate_model, candidate_warm_tokens, cfg.cache_read_discount, - facts.caching_enabled, ) .await { @@ -717,7 +707,6 @@ mod tests { context_tokens: 0, candidate_model: candidate, candidate_route: None, - caching_enabled: true, } } @@ -805,52 +794,12 @@ mod tests { context_tokens: 0, candidate_model: "openai/pricey", candidate_route: None, - caching_enabled: true, }; let d = route(&orch, Some(&st), facts).await; assert_eq!(d.model, "openai/pricey"); assert!(!d.warm); } - // With prompt caching OFF, the anchor holds no warm cache: the switch cost is priced - // against both models' *uncached* input rates and the never-switch baseline grows at - // the default model's uncached rate — no cache-loss penalty, no warm-token credit. - #[tokio::test] - async fn caching_off_prices_switch_against_uncached_rates() { - let orch = orch_with_rates(); - // Warm on `expensive` (uncached 3.0/M). Seed baseline $1.00. - seed_warm_binding(&orch, 1.0, 0.0, 30).await; - let st = routing_budget(25.0); - let facts = RouteFacts { - session_id: Some("s1"), - tenant_id: None, - prefix_hash: Some(1), - context_tokens: 0, - candidate_model: "openai/pricey", - candidate_route: None, - caching_enabled: false, - }; - let d = route(&orch, Some(&st), facts).await; - - assert_eq!(d.model, "openai/pricey"); - assert!(d.warm); - assert_eq!(d.switches, 1); - // Baseline grows at the default's *uncached* rate: 100k x 3.0/M = $0.30 -> $1.30. - assert!( - (d.baseline_usd - 1.30).abs() < 1e-6, - "baseline {} != 1.30 (uncached default rate)", - d.baseline_usd - ); - // Switch cost = 100k x (pricey 5.0 - expensive 3.0)/1M = $0.20 (both uncached), - // well under the 25% x $1.30 = $0.325 ceiling. Contrast the caching-on cost of - // 100k x (5.0 - cached 0.3)/1M = $0.47, which would be vetoed here. - assert!( - (d.switch_spend_usd - 0.20).abs() < 1e-6, - "spend {} != 0.20 (uncached-vs-uncached switch cost)", - d.switch_spend_usd - ); - } - #[tokio::test] async fn baseline_priced_on_default_not_anchor() { // A session that started on `expensive` (default) but has since switched to @@ -959,7 +908,6 @@ mod tests { context_tokens: 100_000, candidate_model: candidate, candidate_route: None, - caching_enabled: true, }; // Turn 1 — cold start: no binding yet, the candidate is honored and becomes both diff --git a/crates/brightstaff/src/handlers/routing_service.rs b/crates/brightstaff/src/handlers/routing_service.rs index 689faec5..0fc759f1 100644 --- a/crates/brightstaff/src/handlers/routing_service.rs +++ b/crates/brightstaff/src/handlers/routing_service.rs @@ -239,7 +239,6 @@ async fn routing_decision_inner( context_tokens, candidate_model: &candidate_model, candidate_route: result.route_name.as_deref(), - caching_enabled: prompt_caching.enabled, }, ) .await; diff --git a/crates/brightstaff/src/router/orchestrator.rs b/crates/brightstaff/src/router/orchestrator.rs index e93e65e5..050797d3 100644 --- a/crates/brightstaff/src/router/orchestrator.rs +++ b/crates/brightstaff/src/router/orchestrator.rs @@ -213,13 +213,12 @@ impl OrchestratorService { /// Estimate the input-cost (USD) of switching a session from `anchor_model` /// (the model that handled the latest request) to `candidate_model`. /// - /// When `caching_enabled`, the anchor is warm, so staying re-reads the context at its - /// *cached* rate and `candidate_warm_tokens` (context the candidate still holds from an - /// earlier visit this session) re-read at the candidate's cached rate. When caching is - /// off there is no warm cache on either side: staying re-reads at the anchor's *plain* - /// (uncached) rate and the candidate re-ingests the whole context uncached (no - /// warm-token credit), so the cost collapses to - /// `context_tokens x (candidate_uncached_rate - anchor_uncached_rate) / 1M`. + /// The anchor is warm (this is only called for warm sessions), so staying re-reads the + /// context at its *cached* rate, while `candidate_warm_tokens` (context the candidate + /// still holds from an earlier visit this session) re-read at the candidate's cached + /// rate. Provider caches are assumed real whenever the routing budget is active — + /// most providers cache automatically, with or without Plano's marker injection + /// (`prompt_caching.enabled` is not consulted here). /// /// Fetches per-model rates from the configured cost feed; returns `None` when pricing /// is missing for either side so the caller can fail open (switch freely) rather than @@ -232,22 +231,13 @@ impl OrchestratorService { candidate_model: &str, candidate_warm_tokens: u64, cache_read_discount: f64, - caching_enabled: bool, ) -> Option { let anchor = self.model_rates(anchor_model).await?; let candidate = self.model_rates(candidate_model).await?; - let (anchor_read_rate, candidate_warm_tokens) = if caching_enabled { - ( - anchor.cached_input_rate(cache_read_discount), - candidate_warm_tokens, - ) - } else { - (anchor.input_per_million, 0) - }; Some(switch_cost_in_usd( context_tokens, candidate_warm_tokens, - anchor_read_rate, + anchor.cached_input_rate(cache_read_discount), candidate.input_per_million, candidate.cached_input_rate(cache_read_discount), )) @@ -256,26 +246,19 @@ impl OrchestratorService { /// This turn's contribution to the session's *never-switch* baseline: the USD cost of /// reading `context_tokens` on `model` — the session's `default_model`, i.e. what it /// would have paid by never switching (not the possibly-drifted current anchor). Priced - /// at the cached input rate when `caching_enabled` (the never-switch path stays warm), - /// at the plain uncached rate otherwise (nothing is cached, so every turn re-reads the - /// full context). Summed across turns, this is the denominator the percentage overhead - /// cap is measured against. `None` when the model has no pricing (the caller then can't - /// grow the baseline this turn). + /// at the cached input rate: the never-switch path stays warm, and provider caches are + /// assumed real whenever the routing budget is active. Summed across turns, this is + /// the denominator the percentage overhead cap is measured against. `None` when the + /// model has no pricing (the caller then can't grow the baseline this turn). pub async fn context_read_cost_in_usd( &self, context_tokens: u64, model: &str, cache_read_discount: f64, - caching_enabled: bool, ) -> Option { let rates = self.model_rates(model).await?; let context_millions = context_tokens as f64 / TOKENS_PER_MILLION; - let rate = if caching_enabled { - rates.cached_input_rate(cache_read_discount) - } else { - rates.input_per_million - }; - Some(context_millions * rate) + Some(context_millions * rates.cached_input_rate(cache_read_discount)) } // ---- LLM routing ---- diff --git a/crates/common/src/configuration.rs b/crates/common/src/configuration.rs index 0896caa3..f595bc50 100644 --- a/crates/common/src/configuration.rs +++ b/crates/common/src/configuration.rs @@ -33,9 +33,10 @@ pub struct Routing { pub session_ttl_seconds: Option, pub session_max_entries: Option, pub session_cache: Option, - /// Cost gate on model switching within a session. Independent of prompt caching: - /// this is a routing decision that applies whenever it is configured, whether or - /// not `prompt_caching` is enabled. Presence of this block turns it on. + /// Cost gate on model switching within a session. Self-sufficient and independent + /// of prompt caching: presence of this block turns it on, implicit sessions are + /// derived on its own, and warm anchors are always priced at cached rates — + /// `prompt_caching` only controls marker injection and affinity-without-budget. pub routing_budget: Option, } diff --git a/demos/llm_routing/routing_budget/GUIDE.md b/demos/llm_routing/routing_budget/GUIDE.md index 95e67a39..e530ab12 100644 --- a/demos/llm_routing/routing_budget/GUIDE.md +++ b/demos/llm_routing/routing_budget/GUIDE.md @@ -9,8 +9,9 @@ There are two independent behaviors to observe: 2. **Routing budget** — the router still runs every turn, but when it proposes a *different* model while the session's cache is plausibly warm, Plano only switches while the session's cumulative switch spend stays within `max_overhead_pct`% of what - staying put would have cost. This is a routing concern and works whether or not - prompt caching is on. + staying put would have cost. This is a routing concern and is self-sufficient: + it needs no `prompt_caching` config (it derives sessions and prices warm + anchors at cached rates on its own). --- @@ -48,8 +49,10 @@ model_metrics_sources: openai-gpt-4o: openai/gpt-4o anthropic-claude-4.6-sonnet: anthropic/claude-sonnet-4-6 -prompt_caching: - enabled: true # automatic caching + session affinity (separate concern) +# OPTIONAL for the routing budget — see below. Needed for §4 (real caching on +# marker-based models when Plano proxies the request). +# prompt_caching: +# enabled: true routing: routing_budget: # no default — presence turns it on @@ -63,8 +66,12 @@ no aliases needed — its keys already match `provider/model` routing names). Th demo models are priced identically on both feeds, so every number in this guide holds either way. -The routing budget lives under `routing` and is independent of prompt caching — it -applies whether or not `prompt_caching.enabled` is set. +The routing budget is fully self-sufficient: configuring it turns on implicit +session derivation and prices warm anchors at cached rates on its own — +`prompt_caching` is **not** required. Enable `prompt_caching` for what it adds: +injecting provider cache-control markers when Plano proxies the request (without +markers, marker-based models like `anthropic/*` never actually cache — see §4) +and session affinity when no budget is configured. @@ -121,9 +128,12 @@ The model listener comes up on **:12000** (per `config.yaml`). ## 4. See caching in action (single model) -Send the same large system prompt across several turns. With caching enabled, -Plano derives an implicit session from the stable prefix and pins the model, so -turns 2+ read the prefix from the provider cache. +This section needs `prompt_caching: { enabled: true }` (uncomment it in +`config.yaml`): the model here is Anthropic-family, which only caches when the +request carries cache-control markers, and it's Plano proxying the request — +so Plano must inject them. Send the same large system prompt across several +turns. Plano derives an implicit session from the stable prefix and pins the +model, so turns 2+ read the prefix from the provider cache. ```bash curl -s localhost:12000/v1/chat/completions \ @@ -353,8 +363,9 @@ the caching-ON/OFF comparison: - **Baseline (no caching):** send requests with header `X-Plano-Cache: off` (disables implicit pinning + marker injection per request), or run with -`prompt_caching.enabled: false`. -- **Treatment (caching on):** default config in this folder. +`prompt_caching` absent/disabled. +- **Treatment (caching on):** the config in this folder with the +`prompt_caching` block uncommented. Compare, over an identical multi-turn eval set: @@ -400,6 +411,7 @@ curl -s localhost:12000/v1/chat/completions \ - Caching **never** changes which model routing selects — the router still makes the quality call; the overhead cap only vetoes a switch that the session can't afford. -- The routing budget is independent of prompt caching (it lives under `routing`) and -is fully opt-in with **no baked-in cap**: configuring it without a `max_overhead_pct` -(or without a cost source) fails startup with a clear message. +- The routing budget is independent of prompt caching (it lives under `routing`, +needs no `prompt_caching` config, and always prices warm anchors at cached rates) +and is fully opt-in with **no baked-in cap**: configuring it without a +`max_overhead_pct` (or without a cost source) fails startup with a clear message. diff --git a/demos/llm_routing/routing_budget/config.yaml b/demos/llm_routing/routing_budget/config.yaml index b66e3a7b..2853f70f 100644 --- a/demos/llm_routing/routing_budget/config.yaml +++ b/demos/llm_routing/routing_budget/config.yaml @@ -46,12 +46,13 @@ model_metrics_sources: # provider: models.dev # refresh_interval: 86400 -# Automatic prompt caching (opt-in). Keeps a conversation pinned to the same -# model so the upstream provider's prompt cache stays warm across turns, and -# auto-injects cache-control markers where the provider needs them. This is a -# separate concern from the routing budget below. -prompt_caching: - enabled: true +# Automatic prompt caching (opt-in). OPTIONAL for the routing budget: the budget +# derives sessions and prices warm anchors at cached rates on its own. Enable this +# only for what it adds — injecting cache-control markers on the full-proxy path +# (required for the provider cache to be real on marker-based models like +# anthropic/*; see GUIDE.md §4) and session affinity without a budget. +# prompt_caching: +# enabled: true routing: # Per-session cost gate on model switching. Independent of prompt caching: it