apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-05-27 14:17:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: make upstream timeout configurable, default to 300s

Browse source
This commit is contained in:
Syed Hashmi 2026-03-02 19:15:35 -06:00
parent 198c912202
commit 0c7b999770
6 changed files with 34 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crates/prompt_gateway/src/http_context.rs
View file

@ -205,7 +205,12 @@ impl HttpContext for StreamContext {
info!("on_http_request_body: sending request to model server");
debug!("request body: {}", json_data);
let timeout_str = MODEL_SERVER_REQUEST_TIMEOUT_MS.to_string();
let timeout_ms = if let Some(overrides) = self.overrides.as_ref() {
overrides.upstream_timeout_ms.unwrap_or(MODEL_SERVER_REQUEST_TIMEOUT_MS)
} else {
MODEL_SERVER_REQUEST_TIMEOUT_MS
};
let timeout_str = timeout_ms.to_string();
let mut headers = vec![
(ARCH_UPSTREAM_HOST_HEADER, MODEL_SERVER_NAME),
@ -230,7 +235,7 @@ impl HttpContext for StreamContext {
headers,
Some(json_data.as_bytes()),
vec![],
Duration::from_secs(5),
Duration::from_millis(timeout_ms),
);
if let Some(content) = self.user_prompt.as_ref().unwrap().content.as_ref() {

20
crates/prompt_gateway/src/stream_context.rs
View file

@ -171,7 +171,14 @@ impl StreamContext {
callout_context.request_body.messages.clone(),
);
let arch_messages_json = serde_json::to_string(&params).unwrap();
let timeout_str = DEFAULT_TARGET_REQUEST_TIMEOUT_MS.to_string();
let timeout_ms = if let Some(overrides) = self.overrides.as_ref() {
overrides
.upstream_timeout_ms
.unwrap_or(DEFAULT_TARGET_REQUEST_TIMEOUT_MS)
} else {
DEFAULT_TARGET_REQUEST_TIMEOUT_MS
};
let timeout_str = timeout_ms.to_string();
let mut headers = vec![
(":method", "POST"),
@ -193,7 +200,7 @@ impl StreamContext {
headers,
Some(arch_messages_json.as_bytes()),
vec![],
Duration::from_secs(5),
Duration::from_millis(timeout_ms),
);
callout_context.response_handler_type = ResponseHandlerType::DefaultTarget;
callout_context.prompt_target_name = Some(default_prompt_target.name.clone());
@ -422,7 +429,12 @@ impl StreamContext {
debug!("on_http_call_response: api call body {:?}", api_call_body);
let timeout_str = API_REQUEST_TIMEOUT_MS.to_string();
let timeout_ms = if let Some(overrides) = self.overrides.as_ref() {
overrides.upstream_timeout_ms.unwrap_or(API_REQUEST_TIMEOUT_MS)
} else {
API_REQUEST_TIMEOUT_MS
};
let timeout_str = timeout_ms.to_string();
let http_method_str = http_method.to_string();
let mut headers: HashMap<_, _> = [
@ -457,7 +469,7 @@ impl StreamContext {
headers.into_iter().collect(),
api_call_body.as_deref().map(|s| s.as_bytes()),
vec![],
Duration::from_secs(5),
Duration::from_millis(timeout_ms),
);
info!(