nomyo-ai/nomyo-router
3
2
Fork 0
Code Issues 1 Pull requests 7 Projects Releases 10 Packages 1 Wiki Activity Actions

Add files via upload

Browse source 
Increasing timeout for heavy load situatuions
This commit is contained in:
Alpha Nerd 2025-09-01 09:01:41 +02:00 • committed by GitHub
parent 295ace0401
commit 64549b4e1c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
router.py
View file

@ -71,7 +71,7 @@ async def fetch_available_models(endpoint: str) -> Set[str]:
set is returned. set is returned.
""" """
try: try:
async with httpx.AsyncClient(timeout=1.0) as client: async with httpx.AsyncClient(timeout=5.0) as client:
if "/v1" in endpoint: if "/v1" in endpoint:
resp = await client.get(f"{endpoint}/models") resp = await client.get(f"{endpoint}/models")
else: else:
@ -118,12 +118,12 @@ async def fetch_endpoint_details(endpoint: str, route: str, detail: str) -> List
resp = await client.get(f"{endpoint}{route}") resp = await client.get(f"{endpoint}{route}")
resp.raise_for_status() resp.raise_for_status()
data = resp.json() data = resp.json()
detail = data.get(detail) detail = data.get(detail, [])
return detail return detail
except Exception as e: except Exception as e:
# If anything goes wrong we cannot reply details # If anything goes wrong we cannot reply details
print(e) print(e)
return {detail: ["N/A"]} return {detail: []}
def ep2base(ep): def ep2base(ep):
if "/v1" in ep: if "/v1" in ep:
@ -795,7 +795,7 @@ async def ps_proxy(request: Request):
for modellist in loaded_models: for modellist in loaded_models:
models['models'] += modellist models['models'] += modellist
# 25. Return a JSONResponse with deduplicated currently deployed models # 2. Return a JSONResponse with deduplicated currently deployed models
return JSONResponse( return JSONResponse(
content={"models": dedupe_on_keys(models['models'], ['digest'])}, content={"models": dedupe_on_keys(models['models'], ['digest'])},
status_code=200, status_code=200,