Add dynamic vision model list from OpenRouter with combobox selector

surfsense_backend/app/config/vision_model_list_fallback.json

@@ -0,0 +1,23 @@
+[
+  {"value": "gpt-4o", "label": "GPT-4o", "provider": "OPENAI", "context_window": "128K"},
+  {"value": "gpt-4o-mini", "label": "GPT-4o Mini", "provider": "OPENAI", "context_window": "128K"},
+  {"value": "gpt-4-turbo", "label": "GPT-4 Turbo", "provider": "OPENAI", "context_window": "128K"},
+  {"value": "claude-sonnet-4-20250514", "label": "Claude Sonnet 4", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-7-sonnet-20250219", "label": "Claude 3.7 Sonnet", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-5-sonnet-20241022", "label": "Claude 3.5 Sonnet", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-opus-20240229", "label": "Claude 3 Opus", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-haiku-20240307", "label": "Claude 3 Haiku", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "gemini-2.5-flash", "label": "Gemini 2.5 Flash", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-2.5-pro", "label": "Gemini 2.5 Pro", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-2.0-flash", "label": "Gemini 2.0 Flash", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-1.5-pro", "label": "Gemini 1.5 Pro", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-1.5-flash", "label": "Gemini 1.5 Flash", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "pixtral-large-latest", "label": "Pixtral Large", "provider": "MISTRAL", "context_window": "128K"},
+  {"value": "pixtral-12b-2409", "label": "Pixtral 12B", "provider": "MISTRAL", "context_window": "128K"},
+  {"value": "grok-2-vision-1212", "label": "Grok 2 Vision", "provider": "XAI", "context_window": "32K"},
+  {"value": "llava", "label": "LLaVA", "provider": "OLLAMA"},
+  {"value": "bakllava", "label": "BakLLaVA", "provider": "OLLAMA"},
+  {"value": "llava-llama3", "label": "LLaVA Llama 3", "provider": "OLLAMA"},
+  {"value": "llama-4-scout-17b-16e-instruct", "label": "Llama 4 Scout 17B", "provider": "GROQ", "context_window": "128K"},
+  {"value": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "label": "Llama 4 Scout 17B", "provider": "TOGETHER_AI", "context_window": "128K"}
+]

surfsense_backend/app/routes/vision_llm_routes.py

@@ -1,6 +1,7 @@
 import logging
 
 from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -17,6 +18,7 @@ from app.schemas import (
     VisionLLMConfigRead,
     VisionLLMConfigUpdate,
 )
+from app.services.vision_model_list_service import get_vision_model_list
 from app.users import current_active_user
 from app.utils.rbac import check_permission
 
@@ -24,6 +26,32 @@ router = APIRouter()
 logger = logging.getLogger(__name__)
 
 
+# =============================================================================
+# Vision Model Catalogue (from OpenRouter, filtered for image-input models)
+# =============================================================================
+
+
+class VisionModelListItem(BaseModel):
+    value: str
+    label: str
+    provider: str
+    context_window: str | None = None
+
+
+@router.get("/vision-models", response_model=list[VisionModelListItem])
+async def list_vision_models(
+    user: User = Depends(current_active_user),
+):
+    """Return vision-capable models sourced from OpenRouter (filtered by image input)."""
+    try:
+        return await get_vision_model_list()
+    except Exception as e:
+        logger.exception("Failed to fetch vision model list")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to fetch vision model list: {e!s}"
+        ) from e
+
+
 # =============================================================================
 # Global Vision LLM Configs (from YAML)
 # =============================================================================

surfsense_backend/app/services/vision_model_list_service.py

@@ -0,0 +1,132 @@
+"""
+Service for fetching and caching the vision-capable model list.
+
+Reuses the same OpenRouter public API and local fallback as the LLM model
+list service, but filters for models that accept image input.
+"""
+
+import json
+import logging
+import time
+from pathlib import Path
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+OPENROUTER_API_URL = "https://openrouter.ai/api/v1/models"
+FALLBACK_FILE = Path(__file__).parent.parent / "config" / "vision_model_list_fallback.json"
+CACHE_TTL_SECONDS = 86400  # 24 hours
+
+_cache: list[dict] | None = None
+_cache_timestamp: float = 0
+
+OPENROUTER_SLUG_TO_VISION_PROVIDER: dict[str, str] = {
+    "openai": "OPENAI",
+    "anthropic": "ANTHROPIC",
+    "google": "GOOGLE",
+    "mistralai": "MISTRAL",
+    "x-ai": "XAI",
+}
+
+
+def _format_context_length(length: int | None) -> str | None:
+    if not length:
+        return None
+    if length >= 1_000_000:
+        return f"{length / 1_000_000:g}M"
+    if length >= 1_000:
+        return f"{length / 1_000:g}K"
+    return str(length)
+
+
+async def _fetch_from_openrouter() -> list[dict] | None:
+    try:
+        async with httpx.AsyncClient(timeout=15) as client:
+            response = await client.get(OPENROUTER_API_URL)
+            response.raise_for_status()
+            data = response.json()
+            return data.get("data", [])
+    except Exception as e:
+        logger.warning("Failed to fetch from OpenRouter API for vision models: %s", e)
+        return None
+
+
+def _load_fallback() -> list[dict]:
+    try:
+        with open(FALLBACK_FILE, encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        logger.error("Failed to load vision model fallback list: %s", e)
+        return []
+
+
+def _is_vision_model(model: dict) -> bool:
+    """Return True if the model accepts image input and outputs text."""
+    arch = model.get("architecture", {})
+    input_mods = arch.get("input_modalities", [])
+    output_mods = arch.get("output_modalities", [])
+    return "image" in input_mods and "text" in output_mods
+
+
+def _process_vision_models(raw_models: list[dict]) -> list[dict]:
+    processed: list[dict] = []
+
+    for model in raw_models:
+        model_id: str = model.get("id", "")
+        name: str = model.get("name", "")
+        context_length = model.get("context_length")
+
+        if "/" not in model_id:
+            continue
+
+        if not _is_vision_model(model):
+            continue
+
+        provider_slug, model_name = model_id.split("/", 1)
+        context_window = _format_context_length(context_length)
+
+        processed.append(
+            {
+                "value": model_id,
+                "label": name,
+                "provider": "OPENROUTER",
+                "context_window": context_window,
+            }
+        )
+
+        native_provider = OPENROUTER_SLUG_TO_VISION_PROVIDER.get(provider_slug)
+        if native_provider:
+            if native_provider == "GOOGLE" and not model_name.startswith("gemini-"):
+                continue
+
+            processed.append(
+                {
+                    "value": model_name,
+                    "label": name,
+                    "provider": native_provider,
+                    "context_window": context_window,
+                }
+            )
+
+    return processed
+
+
+async def get_vision_model_list() -> list[dict]:
+    global _cache, _cache_timestamp
+
+    if _cache is not None and (time.time() - _cache_timestamp) < CACHE_TTL_SECONDS:
+        return _cache
+
+    raw_models = await _fetch_from_openrouter()
+
+    if raw_models is None:
+        logger.info("Using fallback vision model list")
+        return _load_fallback()
+
+    processed = _process_vision_models(raw_models)
+
+    _cache = processed
+    _cache_timestamp = time.time()
+
+    return processed