apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

send latency numbers from model_server as metadata

Browse source
This commit is contained in:
Adil Hafeez 2024-12-10 16:01:17 -08:00
parent 14625e2a1d
commit 60dfb18018
4 changed files with 9 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

3
crates/prompt_gateway/src/stream_context.rs
View file

@ -124,7 +124,8 @@ impl StreamContext {
let arch_fc_response: ChatCompletionsResponse = match serde_json::from_str(&body_str) {
Ok(arch_fc_response) => arch_fc_response,
Err(e) => {
warn!("error deserializing archfc response: {}", e);
warn!("error deserializing archfc response: {}, body: {}", e, body_str
);
return self.send_server_error(ServerError::Deserialization(e), None);
}
};

2
model_server/pyproject.toml
View file

@ -37,7 +37,7 @@ opentelemetry-instrumentation-fastapi = "^0.49b0"
overrides = "^7.7.0"
[tool.poetry.scripts]
archgw_modelserver = "src.cli:start_server"
archgw_modelserver = "src.cli:run_server"
[build-system]
requires = ["poetry-core>=1.0.0"]

1
model_server/src/core/model_utils.py
View file

@ -30,6 +30,7 @@ class ChatCompletionResponse(BaseModel):
created: Optional[str] = ""
choices: List[Choice]
model: str
metadata: Optional[Dict[str, str]] = {}
class GuardRequest(BaseModel):

9
model_server/src/main.py
View file

@ -67,11 +67,12 @@ async def function_calling(req: ChatMessage, res: Response):
"Arch-Function"
].chat_completion(req)
function_latency = time.perf_counter() - function_start_time
return {
"response": function_calling_response,
"intent_latency": round(intent_latency * 1000, 3),
"function_latency": round(function_latency * 1000, 3),
function_calling_response.metadata = {
"intent_latency": str(round(intent_latency * 1000, 3)),
"function_latency": str(round(function_latency * 1000, 3)),
}
return function_calling_response
except Exception as e:
# [TODO] Review: update how to collect debugging outputs
# logger.error(f"Error in chat_completion from `Arch-Function`: {e}")