diff --git a/config/envoy.template.yaml b/config/envoy.template.yaml index 40980167..7415bba5 100644 --- a/config/envoy.template.yaml +++ b/config/envoy.template.yaml @@ -257,7 +257,7 @@ static_resources: route: auto_host_rewrite: true cluster: bright_staff - timeout: 300s + timeout: {{ listener.timeout | default('300s') }} {% for cluster_name, cluster in plano_clusters.items() %} - match: prefix: "/" @@ -268,7 +268,7 @@ static_resources: route: auto_host_rewrite: true cluster: {{ cluster_name }} - timeout: 300s + timeout: {{ listener.timeout | default('300s') }} {% endfor %} http_filters: - name: envoy.filters.http.router diff --git a/crates/prompt_gateway/src/http_context.rs b/crates/prompt_gateway/src/http_context.rs index e33b2a4c..2b0ca433 100644 --- a/crates/prompt_gateway/src/http_context.rs +++ b/crates/prompt_gateway/src/http_context.rs @@ -205,11 +205,7 @@ impl HttpContext for StreamContext { info!("on_http_request_body: sending request to model server"); debug!("request body: {}", json_data); - let timeout_ms = if let Some(overrides) = self.overrides.as_ref() { - overrides.upstream_timeout_ms.unwrap_or(MODEL_SERVER_REQUEST_TIMEOUT_MS) - } else { - MODEL_SERVER_REQUEST_TIMEOUT_MS - }; + let timeout_ms = MODEL_SERVER_REQUEST_TIMEOUT_MS; let timeout_str = timeout_ms.to_string(); let mut headers = vec![ diff --git a/crates/prompt_gateway/src/stream_context.rs b/crates/prompt_gateway/src/stream_context.rs index 173fcaf5..6166e09b 100644 --- a/crates/prompt_gateway/src/stream_context.rs +++ b/crates/prompt_gateway/src/stream_context.rs @@ -171,13 +171,7 @@ impl StreamContext { callout_context.request_body.messages.clone(), ); let arch_messages_json = serde_json::to_string(¶ms).unwrap(); - let timeout_ms = if let Some(overrides) = self.overrides.as_ref() { - overrides - .upstream_timeout_ms - .unwrap_or(DEFAULT_TARGET_REQUEST_TIMEOUT_MS) - } else { - DEFAULT_TARGET_REQUEST_TIMEOUT_MS - }; + let timeout_ms = DEFAULT_TARGET_REQUEST_TIMEOUT_MS; let timeout_str = timeout_ms.to_string(); let mut headers = vec![ @@ -429,11 +423,7 @@ impl StreamContext { debug!("on_http_call_response: api call body {:?}", api_call_body); - let timeout_ms = if let Some(overrides) = self.overrides.as_ref() { - overrides.upstream_timeout_ms.unwrap_or(API_REQUEST_TIMEOUT_MS) - } else { - API_REQUEST_TIMEOUT_MS - }; + let timeout_ms = API_REQUEST_TIMEOUT_MS; let timeout_str = timeout_ms.to_string(); let http_method_str = http_method.to_string();