Add Docker support

Adds comprehensive docker support
This commit is contained in:
YetheSamartaka 2025-11-07 13:59:16 +01:00
parent 20f4d1ac96
commit 9a4bcb6f97
5 changed files with 243 additions and 24 deletions

109
router.py
View file

@ -9,6 +9,7 @@ license: AGPL
import json, time, asyncio, yaml, ollama, openai, os, re, aiohttp, ssl, datetime, random, base64, io
from pathlib import Path
from typing import Dict, Set, List, Optional
from urllib.parse import urlparse
from fastapi import FastAPI, Request, HTTPException
from fastapi_sse import sse_handler
from fastapi.staticfiles import StaticFiles
@ -86,8 +87,20 @@ class Config(BaseSettings):
return cls(**cleaned)
return cls()
def _config_path_from_env() -> Path:
"""
Resolve the configuration file path. Defaults to `config.yaml`
in the current working directory unless NOMYO_ROUTER_CONFIG_PATH
is set.
"""
candidate = os.getenv("NOMYO_ROUTER_CONFIG_PATH")
if candidate:
return Path(candidate).expanduser()
return Path("config.yaml")
# Create the global config object it will be overwritten on startup
config = Config()
config = Config.from_yaml(_config_path_from_env())
# -------------------------------------------------------------
# 2. FastAPI application
@ -123,6 +136,47 @@ async def _ensure_success(resp: aiohttp.ClientResponse) -> None:
text = await resp.text()
raise HTTPException(status_code=resp.status, detail=text)
def _format_connection_issue(url: str, error: Exception) -> str:
"""
Provide a human-friendly error string for connection failures so operators
know which endpoint and address failed from inside the container.
"""
parsed = urlparse(url)
host_hint = parsed.hostname or ""
port_hint = parsed.port or ""
if isinstance(error, aiohttp.ClientConnectorError):
resolved_host = getattr(error, "host", host_hint) or host_hint or "?"
resolved_port = getattr(error, "port", port_hint) or port_hint or "?"
parts = [
f"Failed to connect to {url} (resolved: {resolved_host}:{resolved_port}).",
"Ensure the endpoint address is reachable from within the container.",
]
if resolved_host in {"localhost", "127.0.0.1"}:
parts.append(
"Inside Docker, 'localhost' refers to the container itself; use "
"'host.docker.internal' or a Docker network alias if the service "
"runs on the host machine."
)
os_error = getattr(error, "os_error", None)
if isinstance(os_error, OSError):
errno = getattr(os_error, "errno", None)
strerror = os_error.strerror or str(os_error)
if errno is not None or strerror:
parts.append(f"OS error [{errno}]: {strerror}.")
elif os_error:
parts.append(f"OS error: {os_error}.")
parts.append(f"Original error: {error}.")
return " ".join(parts)
if isinstance(error, asyncio.TimeoutError):
return (
f"Timed out waiting for {url}. "
"The remote endpoint may be offline or slow to respond."
)
return f"Error while contacting {url}: {error}"
def is_ext_openai_endpoint(endpoint: str) -> bool:
if "/v1" not in endpoint:
return False
@ -192,7 +246,8 @@ class fetch:
return models
except Exception as e:
# Treat any error as if the endpoint offers no models
print(f"[fetch.available_models] {endpoint} error: {e}")
message = _format_connection_issue(endpoint_url, e)
print(f"[fetch.available_models] {message}")
_error_cache[endpoint] = time.time()
return set()
@ -212,8 +267,10 @@ class fetch:
# {"models": [{"name": "model1"}, {"name": "model2"}]}
models = {m.get("name") for m in data.get("models", []) if m.get("name")}
return models
except Exception:
except Exception as e:
# If anything goes wrong we simply assume the endpoint has no models
message = _format_connection_issue(f"{endpoint}/api/ps", e)
print(f"[fetch.loaded_models] {message}")
return set()
async def endpoint_details(endpoint: str, route: str, detail: str, api_key: Optional[str] = None) -> List[dict]:
@ -226,15 +283,17 @@ class fetch:
if api_key is not None:
headers = {"Authorization": "Bearer " + api_key}
request_url = f"{endpoint}{route}"
try:
async with client.get(f"{endpoint}{route}", headers=headers) as resp:
async with client.get(request_url, headers=headers) as resp:
await _ensure_success(resp)
data = await resp.json()
detail = data.get(detail, [])
return detail
except Exception as e:
# If anything goes wrong we cannot reply details
print(e)
message = _format_connection_issue(request_url, e)
print(f"[fetch.endpoint_details] {message}")
return []
def ep2base(ep):
@ -1269,23 +1328,25 @@ async def config_proxy(request: Request):
which endpoints are being proxied.
"""
async def check_endpoint(url: str):
client: aiohttp.ClientSession = app_state["session"]
headers = None
if "/v1" in url:
headers = {"Authorization": "Bearer " + config.api_keys[url]}
target_url = f"{url}/models"
else:
target_url = f"{url}/api/version"
try:
client: aiohttp.ClientSession = app_state["session"]
if "/v1" in url:
headers = {"Authorization": "Bearer " + config.api_keys[url]}
async with client.get(f"{url}/models", headers=headers) as resp:
await _ensure_success(resp)
data = await resp.json()
else:
async with client.get(f"{url}/api/version") as resp:
await _ensure_success(resp)
data = await resp.json()
async with client.get(target_url, headers=headers) as resp:
await _ensure_success(resp)
data = await resp.json()
if "/v1" in url:
return {"url": url, "status": "ok", "version": "latest"}
else:
return {"url": url, "status": "ok", "version": data.get("version")}
except Exception as e:
return {"url": url, "status": "error", "detail": str(e)}
detail = _format_connection_issue(target_url, e)
return {"url": url, "status": "error", "detail": detail}
results = await asyncio.gather(*[check_endpoint(ep) for ep in config.endpoints])
return {"endpoints": results}
@ -1664,9 +1725,19 @@ async def usage_stream(request: Request):
async def startup_event() -> None:
global config
# Load YAML config (or use defaults if not present)
config = Config.from_yaml(Path("config.yaml"))
print(f"Loaded configuration:\n endpoints={config.endpoints},\n "
f"max_concurrent_connections={config.max_concurrent_connections}")
config_path = _config_path_from_env()
config = Config.from_yaml(config_path)
if config_path.exists():
print(
f"Loaded configuration from {config_path}:\n"
f" endpoints={config.endpoints},\n"
f" max_concurrent_connections={config.max_concurrent_connections}"
)
else:
print(
f"No configuration file found at {config_path}. "
"Falling back to default settings."
)
ssl_context = ssl.create_default_context()
connector = aiohttp.TCPConnector(limit=0, limit_per_host=512, ssl=ssl_context)