fix DO pricing URL, model_providers name validation, cost_metrics demo endpoint

demos/llm_routing/model_routing_service/config.yaml

@@ -34,13 +34,12 @@ routing_preferences:
       prefer: fastest
 
 model_metrics_sources:
-  - type: digitalocean_pricing
-    refresh_interval: 3600
+  - type: cost_metrics
+    url: http://localhost:8080/costs
+    refresh_interval: 300
 
   - type: prometheus_metrics
     url: http://localhost:9090
     query: model_latency_p95_seconds
     refresh_interval: 60
 
-tracing:
-  random_sampling: 100

demos/llm_routing/model_routing_service/demo.sh

@@ -8,9 +8,12 @@ echo ""
 echo "This demo shows how to use the /routing/v1/* endpoints to get"
 echo "routing decisions without actually proxying the request to an LLM."
 echo ""
+echo "The response includes a ranked 'models' list — use models[0] as the"
+echo "primary and fall back to models[1] on 429/5xx errors."
+echo ""
 
-# --- Example 1: OpenAI Chat Completions format ---
-echo "--- 1. Code generation query (OpenAI format) ---"
+# --- Example 1: Code generation (ranked by fastest) ---
+echo "--- 1. Code generation query (prefer: fastest) ---"
 echo ""
 curl -s "$PLANO_URL/routing/v1/chat/completions" \
   -H "Content-Type: application/json" \
@@ -22,8 +25,8 @@ curl -s "$PLANO_URL/routing/v1/chat/completions" \
   }' | python3 -m json.tool
 echo ""
 
-# --- Example 2: Complex reasoning query ---
-echo "--- 2. Complex reasoning query (OpenAI format) ---"
+# --- Example 2: Complex reasoning (ranked by cheapest) ---
+echo "--- 2. Complex reasoning query (prefer: cheapest) ---"
 echo ""
 curl -s "$PLANO_URL/routing/v1/chat/completions" \
   -H "Content-Type: application/json" \
@@ -36,7 +39,7 @@ curl -s "$PLANO_URL/routing/v1/chat/completions" \
 echo ""
 
 # --- Example 3: Simple query (no routing match) ---
-echo "--- 3. Simple query - no routing match (OpenAI format) ---"
+echo "--- 3. Simple query - no routing match (falls back to request model) ---"
 echo ""
 curl -s "$PLANO_URL/routing/v1/chat/completions" \
   -H "Content-Type: application/json" \
@@ -62,8 +65,31 @@ curl -s "$PLANO_URL/routing/v1/messages" \
   }' | python3 -m json.tool
 echo ""
 
-# --- Example 5: Inline routing policy in request body ---
-echo "--- 5. Inline routing_policy (no config needed) ---"
+# --- Example 5: Inline routing_preferences with prefer:cheapest ---
+echo "--- 5. Inline routing_preferences (prefer: cheapest) ---"
+echo "    models[] will be sorted by ascending cost from DigitalOcean pricing"
+echo ""
+curl -s "$PLANO_URL/routing/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [
+      {"role": "user", "content": "Summarize the key differences between TCP and UDP"}
+    ],
+    "routing_preferences": [
+      {
+        "name": "general",
+        "description": "general questions, explanations, and summaries",
+        "models": ["openai/gpt-4o", "openai/gpt-4o-mini"],
+        "selection_policy": {"prefer": "cheapest"}
+      }
+    ]
+  }' | python3 -m json.tool
+echo ""
+
+# --- Example 6: Inline routing_preferences with prefer:fastest ---
+echo "--- 6. Inline routing_preferences (prefer: fastest) ---"
+echo "    models[] will be sorted by ascending P95 latency from Prometheus"
 echo ""
 curl -s "$PLANO_URL/routing/v1/chat/completions" \
   -H "Content-Type: application/json" \
@@ -72,46 +98,12 @@ curl -s "$PLANO_URL/routing/v1/chat/completions" \
     "messages": [
       {"role": "user", "content": "Write a quicksort implementation in Go"}
     ],
-    "routing_policy": [
+    "routing_preferences": [
       {
-        "model": "openai/gpt-4o",
-        "routing_preferences": [
-          {"name": "coding", "description": "code generation, writing functions, debugging"}
-        ]
-      },
-      {
-        "model": "openai/gpt-4o-mini",
-        "routing_preferences": [
-          {"name": "general", "description": "general questions, simple lookups, casual conversation"}
-        ]
-      }
-    ]
-  }' | python3 -m json.tool
-echo ""
-
-# --- Example 6: Inline routing policy with Anthropic format ---
-echo "--- 6. Inline routing_policy (Anthropic format) ---"
-echo ""
-curl -s "$PLANO_URL/routing/v1/messages" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "gpt-4o-mini",
-    "max_tokens": 1024,
-    "messages": [
-      {"role": "user", "content": "What is the weather like today?"}
-    ],
-    "routing_policy": [
-      {
-        "model": "openai/gpt-4o",
-        "routing_preferences": [
-          {"name": "coding", "description": "code generation, writing functions, debugging"}
-        ]
-      },
-      {
-        "model": "openai/gpt-4o-mini",
-        "routing_preferences": [
-          {"name": "general", "description": "general questions, simple lookups, casual conversation"}
-        ]
+        "name": "coding",
+        "description": "code generation, writing functions, debugging",
+        "models": ["anthropic/claude-sonnet-4-20250514", "openai/gpt-4o", "openai/gpt-4o-mini"],
+        "selection_policy": {"prefer": "fastest"}
       }
     ]
   }' | python3 -m json.tool

demos/llm_routing/model_routing_service/metrics_server.py

@@ -1,10 +1,14 @@
 """
-Minimal Prometheus metrics server for demo purposes.
-Exposes mock P95 latency data for model routing.
+Demo metrics server.
+
+Exposes two endpoints:
+  GET /metrics  — Prometheus text format, P95 latency per model (scraped by Prometheus)
+  GET /costs    — JSON cost data per model, compatible with cost_metrics source
 """
+import json
 from http.server import HTTPServer, BaseHTTPRequestHandler
 
-METRICS = """\
+PROMETHEUS_METRICS = """\
 # HELP model_latency_p95_seconds P95 request latency in seconds per model
 # TYPE model_latency_p95_seconds gauge
 model_latency_p95_seconds{model_name="anthropic/claude-sonnet-4-20250514"} 0.85
@@ -12,13 +16,27 @@ model_latency_p95_seconds{model_name="openai/gpt-4o"} 1.20
 model_latency_p95_seconds{model_name="openai/gpt-4o-mini"} 0.40
 """.encode()
 
+COST_DATA = {
+    "anthropic/claude-sonnet-4-20250514": {"input_per_million": 3.0, "output_per_million": 15.0},
+    "openai/gpt-4o":                      {"input_per_million": 5.0, "output_per_million": 20.0},
+    "openai/gpt-4o-mini":                 {"input_per_million": 0.15, "output_per_million": 0.6},
+}
+
 
 class MetricsHandler(BaseHTTPRequestHandler):
     def do_GET(self):
-        self.send_response(200)
-        self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-        self.end_headers()
-        self.wfile.write(METRICS)
+        if self.path == "/costs":
+            body = json.dumps(COST_DATA).encode()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            self.wfile.write(body)
+        else:
+            # /metrics and everything else → Prometheus format
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(PROMETHEUS_METRICS)
 
     def log_message(self, fmt, *args):
         pass  # suppress access logs
@@ -26,5 +44,5 @@ class MetricsHandler(BaseHTTPRequestHandler):
 
 if __name__ == "__main__":
     server = HTTPServer(("", 8080), MetricsHandler)
-    print("metrics server listening on :8080", flush=True)
+    print("metrics server listening on :8080 (/metrics, /costs)", flush=True)
     server.serve_forever()