bring back agent routing

arch/Dockerfile

@@ -1,4 +1,4 @@
-# build filter using rust toolchain
+# build docker iamge for arch gateway
 FROM rust:1.82.0 as builder
 RUN rustup -v target add wasm32-wasip1
 WORKDIR /arch

arch/Dockerfile.brightstaff

@@ -1,20 +0,0 @@
-# build filter using rust toolchain
-FROM rust:1.82.0 as builder
-WORKDIR /arch
-COPY crates .
-
-RUN cd brightstaff && cargo build --release
-
-# copy built filter into envoy image
-FROM docker.io/envoyproxy/envoy:v1.32-latest as envoy
-
-#Build config generator, so that we have a single build image for both Rust and Python
-FROM python:3.12-slim as arch
-
-RUN apt-get update && apt-get install -y gettext-base curl && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-COPY --from=builder /arch/target/release/brightstaff /app/brightstaff
-
-WORKDIR /app
-
-ENTRYPOINT ["sh","-c", "./brightstaff 2>&1 | tee /var/log/brightstaff.log"]

crates/llm_gateway/src/stream_context.rs

@@ -317,6 +317,10 @@ impl HttpContext for StreamContext {
                 }
             };
 
+        for message in deserialized_body.messages.iter_mut() {
+            message.model = None;
+        }
+
         self.user_message = deserialized_body
             .messages
             .iter()
@@ -324,17 +328,43 @@ impl HttpContext for StreamContext {
             .last()
             .cloned();
 
+        let model_name = match self.llm_provider.as_ref() {
+            Some(llm_provider) => llm_provider.model.as_ref(),
+            None => None,
+        };
+
+        let use_agent_orchestrator = match self.overrides.as_ref() {
+            Some(overrides) => overrides.use_agent_orchestrator.unwrap_or_default(),
+            None => false,
+        };
+
         let model_requested = deserialized_body.model.clone();
+        if deserialized_body.model.is_empty() || deserialized_body.model.to_lowercase() == "none" {
+            deserialized_body.model = match model_name {
+                Some(model_name) => model_name.clone(),
+                None => {
+                    if use_agent_orchestrator {
+                        "agent_orchestrator".to_string()
+                    } else {
+                        self.send_server_error(
+                          ServerError::BadRequest {
+                              why: format!("No model specified in request and couldn't determine model name from arch_config. Model name in req: {}, arch_config, provider: {}, model: {:?}", deserialized_body.model, self.llm_provider().name, self.llm_provider().model).to_string(),
+                          },
+                          Some(StatusCode::BAD_REQUEST),
+                      );
+                        return Action::Continue;
+                    }
+                }
+            }
+        }
 
         info!(
             "on_http_request_body: provider: {}, model requested: {}, model selected: {}",
             self.llm_provider().name,
             model_requested,
-            self.llm_provider().model.as_ref().unwrap_or(&String::new())
+            model_name.unwrap_or(&"None".to_string()),
         );
 
-        deserialized_body.model = self.llm_provider().model.clone().unwrap();
-
         let chat_completion_request_str = serde_json::to_string(&deserialized_body).unwrap();
 
         debug!(