86 lines
No EOL
4 KiB
YAML
86 lines
No EOL
4 KiB
YAML
# config.yaml
|
||
# Ollama endpoints local + remote
|
||
endpoints:
|
||
- http://localhost:11434
|
||
- http://192.168.0.51:11434
|
||
- http://192.168.0.52:11434
|
||
# External OpenAI-compatible endpoints (will NOT be queried for /api/ps /api/ps_details)
|
||
- https://api.openai.com/v1
|
||
|
||
# llama-server endpoints (OpenAI-compatible with /v1/models status info)
|
||
# These endpoints will be queried for /api/tags, /api/ps, /api/ps_details
|
||
# and included in the model selection pool for inference routing
|
||
llama_server_endpoints:
|
||
- http://localhost:8080/v1
|
||
- http://192.168.0.33:8081/v1
|
||
|
||
# Maximum concurrent connections *per endpoint‑model pair* (equals to OLLAMA_NUM_PARALLEL)
|
||
max_concurrent_connections: 2
|
||
|
||
# Optional router-level API key that gates router/API/web UI access (leave empty to disable)
|
||
nomyo-router-api-key: ""
|
||
|
||
# API keys for remote endpoints
|
||
# Set an environment variable like OPENAI_KEY
|
||
# Confirm endpoints are exactly as in endpoints block
|
||
api_keys:
|
||
"http://192.168.0.50:11434": "ollama"
|
||
"http://192.168.0.51:11434": "ollama"
|
||
"http://192.168.0.52:11434": "ollama"
|
||
"https://api.openai.com/v1": "${OPENAI_KEY}"
|
||
"http://localhost:8080/v1": "llama-server" # Optional API key for llama-server - depends on llama_server config
|
||
"http://192.168.0.33:8081/v1": "llama-server"
|
||
|
||
# -------------------------------------------------------------
|
||
# Semantic LLM Cache (optional — disabled by default)
|
||
# Caches LLM responses to cut costs and latency on repeated or
|
||
# semantically similar prompts.
|
||
# Cached routes: /api/chat /api/generate /v1/chat/completions /v1/completions
|
||
# MOE requests (moe-* model prefix) always bypass the cache.
|
||
# -------------------------------------------------------------
|
||
# cache_enabled: false
|
||
|
||
# Backend — where cached responses are stored:
|
||
# memory → in-process LRU (lost on restart, not shared across replicas) [default]
|
||
# sqlite → persistent file-based (single instance, survives restart)
|
||
# redis → distributed (shared across replicas, requires Redis)
|
||
# cache_backend: memory
|
||
|
||
# Cosine similarity threshold for a cache hit:
|
||
# 1.0 → exact match only (works on any image variant)
|
||
# <1.0 → semantic matching (requires the :semantic Docker image tag)
|
||
# cache_similarity: 1.0
|
||
|
||
# Response TTL in seconds. Remove the key or set to null to cache forever.
|
||
# cache_ttl: 3600
|
||
|
||
# SQLite backend: path to the cache database file
|
||
# cache_db_path: llm_cache.db
|
||
|
||
# Redis backend: connection URL
|
||
# cache_redis_url: redis://localhost:6379/0
|
||
|
||
# Weight of the BM25-weighted chat-history embedding vs last-user-message embedding.
|
||
# 0.3 = 30% history context signal, 70% question signal.
|
||
# Only relevant when cache_similarity < 1.0.
|
||
# cache_history_weight: 0.3
|
||
|
||
# -------------------------------------------------------------
|
||
# aissurance.eu compliance plugin (optional — disabled by default)
|
||
# Sends signed AI-infrastructure evidence (model/endpoint inventory +
|
||
# aggregate telemetry — never prompt/completion content) to aissurance.eu.
|
||
# All traffic is outbound. See doc/compliance.md for the full reference.
|
||
# -------------------------------------------------------------
|
||
# compliance:
|
||
# enabled: true
|
||
# server_url: "https://www.aissurance.eu/api/v1/discovery/receive"
|
||
# api_key: "${AISSURANCE_KEY}" # tenant secret (separate from nomyo-router-api-key)
|
||
# tenant_id: "${AISSURANCE_TENANT_ID}" # optional if embedded in the key
|
||
# router_id: "router-prod-001" # defaults to the machine hostname
|
||
# polling_interval: 300 # seconds between discovery snapshots
|
||
# health_interval: 3600 # seconds between health/keepalive calls
|
||
# batch_size: 50 # max models per payload (413 → auto-split)
|
||
# max_retry_attempts: 10 # in-cycle send retries before buffering
|
||
# retry_backoff_base: 2 # exponential backoff base (seconds)
|
||
# buffer_dir: "./compliance-buffer" # encrypted offline retry storage
|
||
# max_buffer_payloads: 100 # oldest dropped first when full |