feat(routing): decouple budget pricing from prompt_caching (#996)

* feat(routing): budget prices cache-aware regardless of prompt_caching flag

* docs: prompt_caching is optional for the routing budget

config/plano_config_schema.yaml

@@ -547,8 +547,10 @@ properties:
       routing_budget:
         type: object
         description: >
-          Per-session cost gate on model switching. Independent of prompt caching — it
+          Per-session cost gate on model switching. Self-sufficient and independent of
-          applies whenever configured (presence of this block turns it on). The default
+          prompt_caching — it applies whenever configured (presence of this block turns
+          it on), derives implicit sessions on its own, and always prices warm anchors
+          at their cached input rate (provider caches are assumed real). The default
           posture is to stick to the model a session is warm on. When routing proposes a
           different model while that session's provider cache is plausibly still warm
           (inferred from the idle gap vs. the provider's cache window), the actual

crates/brightstaff/src/handlers/llm/mod.rs

@@ -408,7 +408,6 @@ async fn llm_chat_inner(
             context_tokens,
             candidate_model: &candidate_model,
             candidate_route: candidate_route.as_deref(),
-            caching_enabled: prompt_caching.enabled,
         },
     )
     .await;

crates/brightstaff/src/handlers/llm/session_router.rs

@@ -13,10 +13,11 @@
 //!   baseline (what staying on the session's `default_model` would have cost — priced
 //!   independently of the current anchor, which drifts as switches happen). An
 //!   outright-cheaper switch is free but never reduces that spend. The promise: this
-//!   conversation bills at most `max_overhead_pct`% above never-switching. With prompt
+//!   conversation bills at most `max_overhead_pct`% above never-switching. A warm
-//!   caching the anchor is warm, so staying is priced at its cached rate and the switch
+//!   anchor is priced at its *cached* rate and the switch pays the cache-loss delta:
-//!   pays the cache-loss delta; without caching there is no warm cache, so both the
+//!   provider caches are assumed real whenever the budget is active (most providers
-//!   baseline and the switch cost are priced at the plain uncached input rate.
+//!   cache automatically), independent of `prompt_caching.enabled`, which only
+//!   controls Plano's own marker injection and caching-without-budget affinity.
 //!
 //! The default posture is to stick. Quality and cost stay separate: the router decides
 //! whether a switch *improves quality*; the overhead cap decides whether it is *affordable*.
@@ -108,11 +109,6 @@ pub struct RouteFacts<'a> {
     /// The model the quality router picked for this request.
     pub candidate_model: &'a str,
     pub candidate_route: Option<&'a str>,
-    /// Whether automatic prompt caching is enabled for this instance. Selects how the
-    /// switch-cost gate prices context reads: with caching the anchor is warm (cached
-    /// rate, plus warm-token credit for A→B→A returns); without it there is no warm
-    /// cache, so both sides are priced at their plain uncached input rate.
-    pub caching_enabled: bool,
 }
 
 /// The routing decision plus the session state to carry into the response side.
@@ -288,7 +284,6 @@ pub async fn route(
                         context_tokens,
                         &session_default,
                         cfg.cache_read_discount,
-                        facts.caching_enabled,
                     )
                     .await
                     .unwrap_or(0.0),
@@ -309,21 +304,17 @@ pub async fn route(
                 ceiling_opt = Some(ceiling);
                 // Credit any context the candidate still has cached from an earlier visit
                 // this session: a return to a still-warm model re-reads only the tokens
-                // appended since, not the whole context (the A→B→A case). Only meaningful
+                // appended since, not the whole context (the A→B→A case).
-                // with prompt caching — without it there is no cache to return to.
+                candidate_warm_tokens = b
-                candidate_warm_tokens = if facts.caching_enabled {
+                    .history
-                    b.history
+                    .iter()
-                        .iter()
+                    .find(|v| v.model == facts.candidate_model)
-                        .find(|v| v.model == facts.candidate_model)
+                    .filter(|v| {
-                        .filter(|v| {
+                        now.duration_since(v.last_used).unwrap_or(Duration::MAX)
-                            now.duration_since(v.last_used).unwrap_or(Duration::MAX)
+                            <= warmth_window(&capability_for_model(facts.candidate_model))
-                                <= warmth_window(&capability_for_model(facts.candidate_model))
+                    })
-                        })
+                    .map(|v| v.cached_tokens.min(context_tokens))
-                        .map(|v| v.cached_tokens.min(context_tokens))
+                    .unwrap_or(0);
-                        .unwrap_or(0)
-                } else {
-                    0
-                };
                 match orchestrator
                     .estimate_switch_cost_in_usd(
                         context_tokens,
@@ -331,7 +322,6 @@ pub async fn route(
                         facts.candidate_model,
                         candidate_warm_tokens,
                         cfg.cache_read_discount,
-                        facts.caching_enabled,
                     )
                     .await
                 {
@@ -717,7 +707,6 @@ mod tests {
             context_tokens: 0,
             candidate_model: candidate,
             candidate_route: None,
-            caching_enabled: true,
         }
     }
 
@@ -805,52 +794,12 @@ mod tests {
             context_tokens: 0,
             candidate_model: "openai/pricey",
             candidate_route: None,
-            caching_enabled: true,
         };
         let d = route(&orch, Some(&st), facts).await;
         assert_eq!(d.model, "openai/pricey");
         assert!(!d.warm);
     }
 
-    // With prompt caching OFF, the anchor holds no warm cache: the switch cost is priced
-    // against both models' *uncached* input rates and the never-switch baseline grows at
-    // the default model's uncached rate — no cache-loss penalty, no warm-token credit.
-    #[tokio::test]
-    async fn caching_off_prices_switch_against_uncached_rates() {
-        let orch = orch_with_rates();
-        // Warm on `expensive` (uncached 3.0/M). Seed baseline $1.00.
-        seed_warm_binding(&orch, 1.0, 0.0, 30).await;
-        let st = routing_budget(25.0);
-        let facts = RouteFacts {
-            session_id: Some("s1"),
-            tenant_id: None,
-            prefix_hash: Some(1),
-            context_tokens: 0,
-            candidate_model: "openai/pricey",
-            candidate_route: None,
-            caching_enabled: false,
-        };
-        let d = route(&orch, Some(&st), facts).await;
-
-        assert_eq!(d.model, "openai/pricey");
-        assert!(d.warm);
-        assert_eq!(d.switches, 1);
-        // Baseline grows at the default's *uncached* rate: 100k x 3.0/M = $0.30 -> $1.30.
-        assert!(
-            (d.baseline_usd - 1.30).abs() < 1e-6,
-            "baseline {} != 1.30 (uncached default rate)",
-            d.baseline_usd
-        );
-        // Switch cost = 100k x (pricey 5.0 - expensive 3.0)/1M = $0.20 (both uncached),
-        // well under the 25% x $1.30 = $0.325 ceiling. Contrast the caching-on cost of
-        // 100k x (5.0 - cached 0.3)/1M = $0.47, which would be vetoed here.
-        assert!(
-            (d.switch_spend_usd - 0.20).abs() < 1e-6,
-            "spend {} != 0.20 (uncached-vs-uncached switch cost)",
-            d.switch_spend_usd
-        );
-    }
-
     #[tokio::test]
     async fn baseline_priced_on_default_not_anchor() {
         // A session that started on `expensive` (default) but has since switched to
@@ -959,7 +908,6 @@ mod tests {
             context_tokens: 100_000,
             candidate_model: candidate,
             candidate_route: None,
-            caching_enabled: true,
         };
 
         // Turn 1 — cold start: no binding yet, the candidate is honored and becomes both

crates/brightstaff/src/handlers/routing_service.rs

@@ -239,7 +239,6 @@ async fn routing_decision_inner(
                     context_tokens,
                     candidate_model: &candidate_model,
                     candidate_route: result.route_name.as_deref(),
-                    caching_enabled: prompt_caching.enabled,
                 },
             )
             .await;

crates/brightstaff/src/router/orchestrator.rs

@@ -213,13 +213,12 @@ impl OrchestratorService {
     /// Estimate the input-cost (USD) of switching a session from `anchor_model`
     /// (the model that handled the latest request) to `candidate_model`.
     ///
-    /// When `caching_enabled`, the anchor is warm, so staying re-reads the context at its
+    /// The anchor is warm (this is only called for warm sessions), so staying re-reads the
-    /// *cached* rate and `candidate_warm_tokens` (context the candidate still holds from an
+    /// context at its *cached* rate, while `candidate_warm_tokens` (context the candidate
-    /// earlier visit this session) re-read at the candidate's cached rate. When caching is
+    /// still holds from an earlier visit this session) re-read at the candidate's cached
-    /// off there is no warm cache on either side: staying re-reads at the anchor's *plain*
+    /// rate. Provider caches are assumed real whenever the routing budget is active —
-    /// (uncached) rate and the candidate re-ingests the whole context uncached (no
+    /// most providers cache automatically, with or without Plano's marker injection
-    /// warm-token credit), so the cost collapses to
+    /// (`prompt_caching.enabled` is not consulted here).
-    /// `context_tokens x (candidate_uncached_rate - anchor_uncached_rate) / 1M`.
     ///
     /// Fetches per-model rates from the configured cost feed; returns `None` when pricing
     /// is missing for either side so the caller can fail open (switch freely) rather than
@@ -232,22 +231,13 @@ impl OrchestratorService {
         candidate_model: &str,
         candidate_warm_tokens: u64,
         cache_read_discount: f64,
-        caching_enabled: bool,
     ) -> Option<f64> {
         let anchor = self.model_rates(anchor_model).await?;
         let candidate = self.model_rates(candidate_model).await?;
-        let (anchor_read_rate, candidate_warm_tokens) = if caching_enabled {
-            (
-                anchor.cached_input_rate(cache_read_discount),
-                candidate_warm_tokens,
-            )
-        } else {
-            (anchor.input_per_million, 0)
-        };
         Some(switch_cost_in_usd(
             context_tokens,
             candidate_warm_tokens,
-            anchor_read_rate,
+            anchor.cached_input_rate(cache_read_discount),
             candidate.input_per_million,
             candidate.cached_input_rate(cache_read_discount),
         ))
@@ -256,26 +246,19 @@ impl OrchestratorService {
     /// This turn's contribution to the session's *never-switch* baseline: the USD cost of
     /// reading `context_tokens` on `model` — the session's `default_model`, i.e. what it
     /// would have paid by never switching (not the possibly-drifted current anchor). Priced
-    /// at the cached input rate when `caching_enabled` (the never-switch path stays warm),
+    /// at the cached input rate: the never-switch path stays warm, and provider caches are
-    /// at the plain uncached rate otherwise (nothing is cached, so every turn re-reads the
+    /// assumed real whenever the routing budget is active. Summed across turns, this is
-    /// full context). Summed across turns, this is the denominator the percentage overhead
+    /// the denominator the percentage overhead cap is measured against. `None` when the
-    /// cap is measured against. `None` when the model has no pricing (the caller then can't
+    /// model has no pricing (the caller then can't grow the baseline this turn).
-    /// grow the baseline this turn).
     pub async fn context_read_cost_in_usd(
         &self,
         context_tokens: u64,
         model: &str,
         cache_read_discount: f64,
-        caching_enabled: bool,
     ) -> Option<f64> {
         let rates = self.model_rates(model).await?;
         let context_millions = context_tokens as f64 / TOKENS_PER_MILLION;
-        let rate = if caching_enabled {
+        Some(context_millions * rates.cached_input_rate(cache_read_discount))
-            rates.cached_input_rate(cache_read_discount)
-        } else {
-            rates.input_per_million
-        };
-        Some(context_millions * rate)
     }
 
     // ---- LLM routing ----

crates/common/src/configuration.rs

@@ -33,9 +33,10 @@ pub struct Routing {
     pub session_ttl_seconds: Option<u64>,
     pub session_max_entries: Option<usize>,
     pub session_cache: Option<SessionCacheConfig>,
-    /// Cost gate on model switching within a session. Independent of prompt caching:
+    /// Cost gate on model switching within a session. Self-sufficient and independent
-    /// this is a routing decision that applies whenever it is configured, whether or
+    /// of prompt caching: presence of this block turns it on, implicit sessions are
-    /// not `prompt_caching` is enabled. Presence of this block turns it on.
+    /// derived on its own, and warm anchors are always priced at cached rates —
+    /// `prompt_caching` only controls marker injection and affinity-without-budget.
     pub routing_budget: Option<RoutingBudget>,
 }
 

demos/llm_routing/routing_budget/GUIDE.md

@@ -9,8 +9,9 @@ There are two independent behaviors to observe:
 2. **Routing budget** — the router still runs every turn, but when it proposes a
   *different* model while the session's cache is plausibly warm, Plano only switches
   while the session's cumulative switch spend stays within `max_overhead_pct`% of what
-  staying put would have cost. This is a routing concern and works whether or not
+  staying put would have cost. This is a routing concern and is self-sufficient:
-  prompt caching is on.
+  it needs no `prompt_caching` config (it derives sessions and prices warm
+  anchors at cached rates on its own).
 
 ---
 
@@ -48,8 +49,10 @@ model_metrics_sources:
       openai-gpt-4o: openai/gpt-4o
       anthropic-claude-4.6-sonnet: anthropic/claude-sonnet-4-6
 
-prompt_caching:
+# OPTIONAL for the routing budget — see below. Needed for §4 (real caching on
-  enabled: true                   # automatic caching + session affinity (separate concern)
+# marker-based models when Plano proxies the request).
+# prompt_caching:
+#   enabled: true
 
 routing:
   routing_budget:                 # no default — presence turns it on
@@ -63,8 +66,12 @@ no aliases needed — its keys already match `provider/model` routing names). Th
 demo models are priced identically on both feeds, so every number in this guide
 holds either way.
 
-The routing budget lives under `routing` and is independent of prompt caching — it
+The routing budget is fully self-sufficient: configuring it turns on implicit
-applies whether or not `prompt_caching.enabled` is set.
+session derivation and prices warm anchors at cached rates on its own —
+`prompt_caching` is **not** required. Enable `prompt_caching` for what it adds:
+injecting provider cache-control markers when Plano proxies the request (without
+markers, marker-based models like `anthropic/*` never actually cache — see §4)
+and session affinity when no budget is configured.
 
 
 
@@ -121,9 +128,12 @@ The model listener comes up on **:12000** (per `config.yaml`).
 
 ## 4. See caching in action (single model)
 
-Send the same large system prompt across several turns. With caching enabled,
+This section needs `prompt_caching: { enabled: true }` (uncomment it in
-Plano derives an implicit session from the stable prefix and pins the model, so
+`config.yaml`): the model here is Anthropic-family, which only caches when the
-turns 2+ read the prefix from the provider cache.
+request carries cache-control markers, and it's Plano proxying the request —
+so Plano must inject them. Send the same large system prompt across several
+turns. Plano derives an implicit session from the stable prefix and pins the
+model, so turns 2+ read the prefix from the provider cache.
 
 ```bash
 curl -s localhost:12000/v1/chat/completions \
@@ -353,8 +363,9 @@ the caching-ON/OFF comparison:
 
 - **Baseline (no caching):** send requests with header `X-Plano-Cache: off`
 (disables implicit pinning + marker injection per request), or run with
-`prompt_caching.enabled: false`.
+`prompt_caching` absent/disabled.
-- **Treatment (caching on):** default config in this folder.
+- **Treatment (caching on):** the config in this folder with the
+`prompt_caching` block uncommented.
 
 Compare, over an identical multi-turn eval set:
 
@@ -400,6 +411,7 @@ curl -s localhost:12000/v1/chat/completions \
 
 - Caching **never** changes which model routing selects — the router still makes
 the quality call; the overhead cap only vetoes a switch that the session can't afford.
-- The routing budget is independent of prompt caching (it lives under `routing`) and
+- The routing budget is independent of prompt caching (it lives under `routing`,
-is fully opt-in with **no baked-in cap**: configuring it without a `max_overhead_pct`
+needs no `prompt_caching` config, and always prices warm anchors at cached rates)
-(or without a cost source) fails startup with a clear message.
+and is fully opt-in with **no baked-in cap**: configuring it without a
+`max_overhead_pct` (or without a cost source) fails startup with a clear message.

demos/llm_routing/routing_budget/config.yaml

@@ -46,12 +46,13 @@ model_metrics_sources:
   #   provider: models.dev
   #   refresh_interval: 86400
 
-# Automatic prompt caching (opt-in). Keeps a conversation pinned to the same
+# Automatic prompt caching (opt-in). OPTIONAL for the routing budget: the budget
-# model so the upstream provider's prompt cache stays warm across turns, and
+# derives sessions and prices warm anchors at cached rates on its own. Enable this
-# auto-injects cache-control markers where the provider needs them. This is a
+# only for what it adds — injecting cache-control markers on the full-proxy path
-# separate concern from the routing budget below.
+# (required for the provider cache to be real on marker-based models like
-prompt_caching:
+# anthropic/*; see GUIDE.md §4) and session affinity without a budget.
-  enabled: true
+# prompt_caching:
+#   enabled: true
 
 routing:
   # Per-session cost gate on model switching. Independent of prompt caching: it