add support for claude

arch/arch_config_schema.yaml

@@ -70,8 +70,11 @@ properties:
         provider_interface:
           type: string
           enum:
-            - openai
+            - claude
+            - deepseek
+            - groq
             - mistral
+            - openai
         access_key:
           type: string
         model:

arch/envoy.template.yaml

@@ -451,38 +451,14 @@ static_resources:
                       "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
 
   clusters:
-    - name: openai
-      connect_timeout: 0.5s
-      type: LOGICAL_DNS
-      dns_lookup_family: V4_ONLY
-      lb_policy: ROUND_ROBIN
-      load_assignment:
-        cluster_name: openai
-        endpoints:
-          - lb_endpoints:
-              - endpoint:
-                  address:
-                    socket_address:
-                      address: api.openai.com
-                      port_value: 443
-                  hostname: "api.openai.com"
-      transport_socket:
-        name: envoy.transport_sockets.tls
-        typed_config:
-          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
-          sni: api.openai.com
-          common_tls_context:
-            tls_params:
-              tls_minimum_protocol_version: TLSv1_2
-              tls_maximum_protocol_version: TLSv1_3
 
-    - name: anthropic
+    - name: claude
       connect_timeout: 0.5s
       type: LOGICAL_DNS
       dns_lookup_family: V4_ONLY
       lb_policy: ROUND_ROBIN
       load_assignment:
-        cluster_name: anthropic
+        cluster_name: claude
         endpoints:
           - lb_endpoints:
               - endpoint:
@@ -501,6 +477,31 @@ static_resources:
               tls_minimum_protocol_version: TLSv1_2
               tls_maximum_protocol_version: TLSv1_3
 
+    - name: deepseek
+      connect_timeout: 0.5s
+      type: LOGICAL_DNS
+      dns_lookup_family: V4_ONLY
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: deepseek
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: api.deepseek.com
+                      port_value: 443
+                  hostname: "api.deepseek.com"
+      transport_socket:
+        name: envoy.transport_sockets.tls
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+          sni: api.deepseek.com
+          common_tls_context:
+            tls_params:
+              tls_minimum_protocol_version: TLSv1_2
+              tls_maximum_protocol_version: TLSv1_3
+
     - name: gemini
       connect_timeout: 0.5s
       type: LOGICAL_DNS
@@ -571,6 +572,32 @@ static_resources:
         typed_config:
           "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
           sni: api.mistral.ai
+
+    - name: openai
+      connect_timeout: 0.5s
+      type: LOGICAL_DNS
+      dns_lookup_family: V4_ONLY
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: openai
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: api.openai.com
+                      port_value: 443
+                  hostname: "api.openai.com"
+      transport_socket:
+        name: envoy.transport_sockets.tls
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+          sni: api.openai.com
+          common_tls_context:
+            tls_params:
+              tls_minimum_protocol_version: TLSv1_2
+              tls_maximum_protocol_version: TLSv1_3
+
     {% for internal_cluster in ["arch_fc", "model_server"] %}
     - name: {{ internal_cluster }}
       connect_timeout: 0.5s

crates/common/src/api/open_ai.rs

@@ -330,6 +330,7 @@ impl TryFrom<&str> for ChatCompletionStreamResponseServerEvents {
         let response_chunks: VecDeque<ChatCompletionStreamResponse> = value
             .lines()
             .filter(|line| line.starts_with("data: "))
+            .filter(|line| !line.starts_with(r#"data: {"type": "ping"}"#))
             .map(|line| line.get(6..).unwrap())
             .filter(|data_chunk| *data_chunk != "[DONE]")
             .map(serde_json::from_str::<ChatCompletionStreamResponse>)
@@ -677,4 +678,37 @@ data: [DONE]
             "Hello! How can I assist you today?"
         );
     }
+
+    #[test]
+    fn stream_chunk_parse_claude() {
+        const CHUNK_RESPONSE: &str = r#"data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"role":"assistant"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: {"type": "ping"}
+
+data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"content":"Hello!"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"content":" How can I assist you today? Whether"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"content":" you have a question, need information"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"content":", or just want to chat about"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"content":" something, I'm here to help. What woul"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{"content":"d you like to talk about?"}}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: {"id":"msg_01DZDMxYSgq8aPQxMQoBv6Kb","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"created":1747685264,"model":"claude-3-7-sonnet-latest","object":"chat.completion.chunk"}
+
+data: [DONE]
+"#;
+
+        let sever_events: ChatCompletionStreamResponseServerEvents =
+            ChatCompletionStreamResponseServerEvents::try_from(CHUNK_RESPONSE).unwrap();
+        assert_eq!(sever_events.events.len(), 8);
+
+        assert_eq!(
+            sever_events.to_string(),
+            "Hello! How can I assist you today? Whether you have a question, need information, or just want to chat about something, I'm here to help. What would you like to talk about?"
+        );
+    }
 }

crates/common/src/configuration.rs

@@ -143,19 +143,28 @@ pub struct EmbeddingProviver {
     pub model: String,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
 pub enum LlmProviderType {
-    #[serde(rename = "openai")]
+    #[serde(rename = "claude")]
-    OpenAI,
+    Claude,
+    #[serde(rename = "deepseek")]
+    Deepseek,
+    #[serde(rename = "groq")]
+    Groq,
     #[serde(rename = "mistral")]
     Mistral,
+    #[serde(rename = "openai")]
+    OpenAI,
 }
 
 impl Display for LlmProviderType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            LlmProviderType::OpenAI => write!(f, "openai"),
+            LlmProviderType::Claude => write!(f, "claude"),
+            LlmProviderType::Deepseek => write!(f, "deepseek"),
+            LlmProviderType::Groq => write!(f, "groq"),
             LlmProviderType::Mistral => write!(f, "mistral"),
+            LlmProviderType::OpenAI => write!(f, "openai"),
         }
     }
 }
@@ -175,6 +184,23 @@ pub struct LlmProvider {
     pub usage: Option<String>,
 }
 
+impl Default for LlmProvider {
+    fn default() -> Self {
+        Self {
+            name: "openai".to_string(),
+            provider_interface: LlmProviderType::OpenAI,
+            access_key: None,
+            model: None,
+            default: Some(true),
+            stream: Some(false),
+            endpoint: None,
+            port: None,
+            rate_limits: None,
+            usage: None,
+        }
+    }
+}
+
 impl Display for LlmProvider {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.name)

crates/llm_gateway/src/stream_context.rs

@@ -89,15 +89,7 @@ impl StreamContext {
             provider_hint,
         ));
 
-        // Check if we need to modify the path based on the provider's base_url
+        if self.llm_provider.as_ref().unwrap().provider_interface == LlmProviderType::Groq {
-        let needs_openai_prefix = self
-            .llm_provider
-            .as_ref()
-            .and_then(|provider| provider.endpoint.as_ref())
-            .map(|url| url.contains("api.groq.com"))
-            .unwrap_or(false);
-
-        if needs_openai_prefix {
             if let Some(path) = self.get_http_request_header(":path") {
                 if path.starts_with("/v1/") {
                     let new_path = format!("/openai{}", path);
@@ -221,14 +213,7 @@ impl HttpContext for StreamContext {
             self.llm_provider = Some(Rc::new(LlmProvider {
                 name: routing_header_value.to_string(),
                 provider_interface: LlmProviderType::OpenAI,
-                access_key: None,
+                ..Default::default()
-                endpoint: None,
-                model: None,
-                default: None,
-                stream: None,
-                port: None,
-                rate_limits: None,
-                usage: None,
             }));
         } else {
             self.select_llm_provider();
@@ -539,6 +524,9 @@ impl HttpContext for StreamContext {
             }
             streaming_chunk
         } else {
+            if body_size == 0 {
+                return Action::Continue;
+            }
             debug!("non streaming response bytes read: 0:{}", body_size);
             match self.get_http_response_body(0, body_size) {
                 Some(body) => body,

demos/use_cases/llm_routing/arch_config.yaml

@@ -12,7 +12,6 @@ llm_providers:
     access_key: $OPENAI_API_KEY
     provider_interface: openai
     model: gpt-4o-mini
-    default: true
 
   - name: gpt-4o
     access_key: $OPENAI_API_KEY
@@ -26,20 +25,19 @@ llm_providers:
 
   - name: claude-sonnet
     access_key: $ANTHROPY_API_KEY
-    provider_interface: anthropic
+    provider_interface: claude
     model: claude-3-7-sonnet-latest
+    default: true
 
   - name: deepseek
     access_key: $DEEPSEEK_API_KEY
     provider_interface: deepseek
     model: deepseek-reasoner
-    base_url: https://api.deepseek.com/
 
   - name: groq
     access_key: $GROQ_API_KEY
     provider_interface: groq
     model: llama-3.1-8b-instant
-    base_url: https://api.groq.com
 
 tracing:
   random_sampling: 100