diff --git a/arch/Dockerfile b/arch/Dockerfile index 7eab78b2..42e20755 100644 --- a/arch/Dockerfile +++ b/arch/Dockerfile @@ -1,4 +1,4 @@ -# build filter using rust toolchain +# build docker iamge for arch gateway FROM rust:1.82.0 as builder RUN rustup -v target add wasm32-wasip1 WORKDIR /arch diff --git a/arch/Dockerfile.brightstaff b/arch/Dockerfile.brightstaff deleted file mode 100644 index c0de5dc9..00000000 --- a/arch/Dockerfile.brightstaff +++ /dev/null @@ -1,20 +0,0 @@ -# build filter using rust toolchain -FROM rust:1.82.0 as builder -WORKDIR /arch -COPY crates . - -RUN cd brightstaff && cargo build --release - -# copy built filter into envoy image -FROM docker.io/envoyproxy/envoy:v1.32-latest as envoy - -#Build config generator, so that we have a single build image for both Rust and Python -FROM python:3.12-slim as arch - -RUN apt-get update && apt-get install -y gettext-base curl && apt-get clean && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /arch/target/release/brightstaff /app/brightstaff - -WORKDIR /app - -ENTRYPOINT ["sh","-c", "./brightstaff 2>&1 | tee /var/log/brightstaff.log"] diff --git a/crates/llm_gateway/src/stream_context.rs b/crates/llm_gateway/src/stream_context.rs index aa77dd34..8fc92787 100644 --- a/crates/llm_gateway/src/stream_context.rs +++ b/crates/llm_gateway/src/stream_context.rs @@ -317,6 +317,10 @@ impl HttpContext for StreamContext { } }; + for message in deserialized_body.messages.iter_mut() { + message.model = None; + } + self.user_message = deserialized_body .messages .iter() @@ -324,17 +328,43 @@ impl HttpContext for StreamContext { .last() .cloned(); + let model_name = match self.llm_provider.as_ref() { + Some(llm_provider) => llm_provider.model.as_ref(), + None => None, + }; + + let use_agent_orchestrator = match self.overrides.as_ref() { + Some(overrides) => overrides.use_agent_orchestrator.unwrap_or_default(), + None => false, + }; + let model_requested = deserialized_body.model.clone(); + if deserialized_body.model.is_empty() || deserialized_body.model.to_lowercase() == "none" { + deserialized_body.model = match model_name { + Some(model_name) => model_name.clone(), + None => { + if use_agent_orchestrator { + "agent_orchestrator".to_string() + } else { + self.send_server_error( + ServerError::BadRequest { + why: format!("No model specified in request and couldn't determine model name from arch_config. Model name in req: {}, arch_config, provider: {}, model: {:?}", deserialized_body.model, self.llm_provider().name, self.llm_provider().model).to_string(), + }, + Some(StatusCode::BAD_REQUEST), + ); + return Action::Continue; + } + } + } + } info!( "on_http_request_body: provider: {}, model requested: {}, model selected: {}", self.llm_provider().name, model_requested, - self.llm_provider().model.as_ref().unwrap_or(&String::new()) + model_name.unwrap_or(&"None".to_string()), ); - deserialized_body.model = self.llm_provider().model.clone().unwrap(); - let chat_completion_request_str = serde_json::to_string(&deserialized_body).unwrap(); debug!(